Adding upstream version 7.0.14-dfsg.upstream/7.0.14-dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

65 files changed, 177558 insertions, 0 deletions

diff --git a/src/VBox/VMM/VMMAll/APICAll.cpp b/src/VBox/VMM/VMMAll/APICAll.cpp
new file mode 100644
index 00000000..6041a843
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/APICAll.cpp
@@ -0,0 +1,3655 @@
+/* $Id: APICAll.cpp $ */
+/** @file
+ * APIC - Advanced Programmable Interrupt Controller - All Contexts.
+ */
+
+/*
+ * Copyright (C) 2016-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DEV_APIC
+#define VMCPU_INCL_CPUM_GST_CTX /* for macOS hack */
+#include "APICInternal.h"
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vmcpuset.h>
+#ifdef IN_RING0
+# include <VBox/vmm/gvmm.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static void apicSetInterruptFF(PVMCPUCC pVCpu, PDMAPICIRQ enmType);
+static void apicStopTimer(PVMCPUCC pVCpu);
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+/** An ordered array of valid LVT masks. */
+static const uint32_t g_au32LvtValidMasks[] =
+{
+    XAPIC_LVT_TIMER_VALID,
+    XAPIC_LVT_THERMAL_VALID,
+    XAPIC_LVT_PERF_VALID,
+    XAPIC_LVT_LINT_VALID,   /* LINT0 */
+    XAPIC_LVT_LINT_VALID,   /* LINT1 */
+    XAPIC_LVT_ERROR_VALID
+};
+#endif
+
+#if 0
+/** @todo CMCI */
+static const uint32_t g_au32LvtExtValidMask[] =
+{
+    XAPIC_LVT_CMCI_VALID
+};
+#endif
+
+
+/**
+ * Checks if a vector is set in an APIC 256-bit sparse register.
+ *
+ * @returns true if the specified vector is set, false otherwise.
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   uVector         The vector to check if set.
+ */
+DECLINLINE(bool) apicTestVectorInReg(const volatile XAPIC256BITREG *pApicReg, uint8_t uVector)
+{
+    const volatile uint8_t *pbBitmap = (const volatile uint8_t *)&pApicReg->u[0];
+    return ASMBitTest(pbBitmap, (XAPIC_REG256_VECTOR_OFF(uVector) << 3) + XAPIC_REG256_VECTOR_BIT(uVector));
+}
+
+
+/**
+ * Sets the vector in an APIC 256-bit sparse register.
+ *
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   uVector         The vector to set.
+ */
+DECLINLINE(void) apicSetVectorInReg(volatile XAPIC256BITREG *pApicReg, uint8_t uVector)
+{
+    volatile uint8_t *pbBitmap = (volatile uint8_t *)&pApicReg->u[0];
+    ASMAtomicBitSet(pbBitmap, (XAPIC_REG256_VECTOR_OFF(uVector) << 3) + XAPIC_REG256_VECTOR_BIT(uVector));
+}
+
+
+/**
+ * Clears the vector in an APIC 256-bit sparse register.
+ *
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   uVector         The vector to clear.
+ */
+DECLINLINE(void) apicClearVectorInReg(volatile XAPIC256BITREG *pApicReg, uint8_t uVector)
+{
+    volatile uint8_t *pbBitmap = (volatile uint8_t *)&pApicReg->u[0];
+    ASMAtomicBitClear(pbBitmap, (XAPIC_REG256_VECTOR_OFF(uVector) << 3) + XAPIC_REG256_VECTOR_BIT(uVector));
+}
+
+
+#if 0 /* unused */
+/**
+ * Checks if a vector is set in an APIC Pending-Interrupt Bitmap (PIB).
+ *
+ * @returns true if the specified vector is set, false otherwise.
+ * @param   pvPib           Opaque pointer to the PIB.
+ * @param   uVector         The vector to check if set.
+ */
+DECLINLINE(bool) apicTestVectorInPib(volatile void *pvPib, uint8_t uVector)
+{
+    return ASMBitTest(pvPib, uVector);
+}
+#endif /* unused */
+
+
+/**
+ * Atomically sets the PIB notification bit.
+ *
+ * @returns non-zero if the bit was already set, 0 otherwise.
+ * @param   pApicPib        Pointer to the PIB.
+ */
+DECLINLINE(uint32_t) apicSetNotificationBitInPib(PAPICPIB pApicPib)
+{
+    return ASMAtomicXchgU32(&pApicPib->fOutstandingNotification, RT_BIT_32(31));
+}
+
+
+/**
+ * Atomically tests and clears the PIB notification bit.
+ *
+ * @returns non-zero if the bit was already set, 0 otherwise.
+ * @param   pApicPib        Pointer to the PIB.
+ */
+DECLINLINE(uint32_t) apicClearNotificationBitInPib(PAPICPIB pApicPib)
+{
+    return ASMAtomicXchgU32(&pApicPib->fOutstandingNotification, UINT32_C(0));
+}
+
+
+/**
+ * Sets the vector in an APIC Pending-Interrupt Bitmap (PIB).
+ *
+ * @param   pvPib           Opaque pointer to the PIB.
+ * @param   uVector         The vector to set.
+ */
+DECLINLINE(void) apicSetVectorInPib(volatile void *pvPib, uint8_t uVector)
+{
+    ASMAtomicBitSet(pvPib, uVector);
+}
+
+#if 0 /* unused */
+/**
+ * Clears the vector in an APIC Pending-Interrupt Bitmap (PIB).
+ *
+ * @param   pvPib           Opaque pointer to the PIB.
+ * @param   uVector         The vector to clear.
+ */
+DECLINLINE(void) apicClearVectorInPib(volatile void *pvPib, uint8_t uVector)
+{
+    ASMAtomicBitClear(pvPib, uVector);
+}
+#endif /* unused */
+
+#if 0 /* unused */
+/**
+ * Atomically OR's a fragment (32 vectors) into an APIC 256-bit sparse
+ * register.
+ *
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   idxFragment     The index of the 32-bit fragment in @a
+ *                          pApicReg.
+ * @param   u32Fragment     The 32-bit vector fragment to OR.
+ */
+DECLINLINE(void) apicOrVectorsToReg(volatile XAPIC256BITREG *pApicReg, size_t idxFragment, uint32_t u32Fragment)
+{
+    Assert(idxFragment < RT_ELEMENTS(pApicReg->u));
+    ASMAtomicOrU32(&pApicReg->u[idxFragment].u32Reg, u32Fragment);
+}
+#endif /* unused */
+
+
+#if 0 /* unused */
+/**
+ * Atomically AND's a fragment (32 vectors) into an APIC
+ * 256-bit sparse register.
+ *
+ * @param   pApicReg        The APIC 256-bit spare register.
+ * @param   idxFragment     The index of the 32-bit fragment in @a
+ *                          pApicReg.
+ * @param   u32Fragment     The 32-bit vector fragment to AND.
+ */
+DECLINLINE(void) apicAndVectorsToReg(volatile XAPIC256BITREG *pApicReg, size_t idxFragment, uint32_t u32Fragment)
+{
+    Assert(idxFragment < RT_ELEMENTS(pApicReg->u));
+    ASMAtomicAndU32(&pApicReg->u[idxFragment].u32Reg, u32Fragment);
+}
+#endif /* unused */
+
+
+/**
+ * Reports and returns appropriate error code for invalid MSR accesses.
+ *
+ * @returns VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u32Reg          The MSR being accessed.
+ * @param   enmAccess       The invalid-access type.
+ */
+static int apicMsrAccessError(PVMCPUCC pVCpu, uint32_t u32Reg, APICMSRACCESS enmAccess)
+{
+    static struct
+    {
+        const char *pszBefore;   /* The error message before printing the MSR index */
+        const char *pszAfter;    /* The error message after printing the MSR index */
+    } const s_aAccess[] =
+    {
+        /* enmAccess  pszBefore                        pszAfter */
+        /* 0 */     { "read MSR",                      " while not in x2APIC mode"    },
+        /* 1 */     { "write MSR",                     " while not in x2APIC mode"    },
+        /* 2 */     { "read reserved/unknown MSR",     ""                             },
+        /* 3 */     { "write reserved/unknown MSR",    ""                             },
+        /* 4 */     { "read write-only MSR",           ""                             },
+        /* 5 */     { "write read-only MSR",           ""                             },
+        /* 6 */     { "read reserved bits of MSR",     ""                             },
+        /* 7 */     { "write reserved bits of MSR",    ""                             },
+        /* 8 */     { "write an invalid value to MSR", ""                             },
+        /* 9 */     { "write MSR",                     " disallowed by configuration" },
+        /* 10 */    { "read MSR",                      " disallowed by configuration" },
+    };
+    AssertCompile(RT_ELEMENTS(s_aAccess) == APICMSRACCESS_COUNT);
+
+    size_t const i = enmAccess;
+    Assert(i < RT_ELEMENTS(s_aAccess));
+    if (pVCpu->apic.s.cLogMaxAccessError++ < 5)
+        LogRel(("APIC%u: Attempt to %s (%#x)%s -> #GP(0)\n", pVCpu->idCpu, s_aAccess[i].pszBefore, u32Reg, s_aAccess[i].pszAfter));
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * Gets the descriptive APIC mode.
+ *
+ * @returns The name.
+ * @param   enmMode     The xAPIC mode.
+ */
+const char *apicGetModeName(APICMODE enmMode)
+{
+    switch (enmMode)
+    {
+        case APICMODE_DISABLED:  return "Disabled";
+        case APICMODE_XAPIC:     return "xAPIC";
+        case APICMODE_X2APIC:    return "x2APIC";
+        default:                 break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the descriptive destination format name.
+ *
+ * @returns The destination format name.
+ * @param   enmDestFormat       The destination format.
+ */
+const char *apicGetDestFormatName(XAPICDESTFORMAT enmDestFormat)
+{
+    switch (enmDestFormat)
+    {
+        case XAPICDESTFORMAT_FLAT:      return "Flat";
+        case XAPICDESTFORMAT_CLUSTER:   return "Cluster";
+        default:                        break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the descriptive delivery mode name.
+ *
+ * @returns The delivery mode name.
+ * @param   enmDeliveryMode     The delivery mode.
+ */
+const char *apicGetDeliveryModeName(XAPICDELIVERYMODE enmDeliveryMode)
+{
+    switch (enmDeliveryMode)
+    {
+        case XAPICDELIVERYMODE_FIXED:        return "Fixed";
+        case XAPICDELIVERYMODE_LOWEST_PRIO:  return "Lowest-priority";
+        case XAPICDELIVERYMODE_SMI:          return "SMI";
+        case XAPICDELIVERYMODE_NMI:          return "NMI";
+        case XAPICDELIVERYMODE_INIT:         return "INIT";
+        case XAPICDELIVERYMODE_STARTUP:      return "SIPI";
+        case XAPICDELIVERYMODE_EXTINT:       return "ExtINT";
+        default:                             break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the descriptive destination mode name.
+ *
+ * @returns The destination mode name.
+ * @param   enmDestMode     The destination mode.
+ */
+const char *apicGetDestModeName(XAPICDESTMODE enmDestMode)
+{
+    switch (enmDestMode)
+    {
+        case XAPICDESTMODE_PHYSICAL:  return "Physical";
+        case XAPICDESTMODE_LOGICAL:   return "Logical";
+        default:                      break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the descriptive trigger mode name.
+ *
+ * @returns The trigger mode name.
+ * @param   enmTriggerMode     The trigger mode.
+ */
+const char *apicGetTriggerModeName(XAPICTRIGGERMODE enmTriggerMode)
+{
+    switch (enmTriggerMode)
+    {
+        case XAPICTRIGGERMODE_EDGE:     return "Edge";
+        case XAPICTRIGGERMODE_LEVEL:    return "Level";
+        default:                        break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the destination shorthand name.
+ *
+ * @returns The destination shorthand name.
+ * @param   enmDestShorthand     The destination shorthand.
+ */
+const char *apicGetDestShorthandName(XAPICDESTSHORTHAND enmDestShorthand)
+{
+    switch (enmDestShorthand)
+    {
+        case XAPICDESTSHORTHAND_NONE:           return "None";
+        case XAPICDESTSHORTHAND_SELF:           return "Self";
+        case XAPIDDESTSHORTHAND_ALL_INCL_SELF:  return "All including self";
+        case XAPICDESTSHORTHAND_ALL_EXCL_SELF:  return "All excluding self";
+        default:                                break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the timer mode name.
+ *
+ * @returns The timer mode name.
+ * @param   enmTimerMode         The timer mode.
+ */
+const char *apicGetTimerModeName(XAPICTIMERMODE enmTimerMode)
+{
+    switch (enmTimerMode)
+    {
+        case XAPICTIMERMODE_ONESHOT:        return "One-shot";
+        case XAPICTIMERMODE_PERIODIC:       return "Periodic";
+        case XAPICTIMERMODE_TSC_DEADLINE:   return "TSC deadline";
+        default:                            break;
+    }
+    return "Invalid";
+}
+
+
+/**
+ * Gets the APIC mode given the base MSR value.
+ *
+ * @returns The APIC mode.
+ * @param   uApicBaseMsr        The APIC Base MSR value.
+ */
+APICMODE apicGetMode(uint64_t uApicBaseMsr)
+{
+    uint32_t const uMode   = (uApicBaseMsr >> 10) & UINT64_C(3);
+    APICMODE const enmMode = (APICMODE)uMode;
+#ifdef VBOX_STRICT
+    /* Paranoia. */
+    switch (uMode)
+    {
+        case APICMODE_DISABLED:
+        case APICMODE_INVALID:
+        case APICMODE_XAPIC:
+        case APICMODE_X2APIC:
+            break;
+        default:
+            AssertMsgFailed(("Invalid mode"));
+    }
+#endif
+    return enmMode;
+}
+
+
+/**
+ * Returns whether the APIC is hardware enabled or not.
+ *
+ * @returns true if enabled, false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) APICIsEnabled(PCVMCPUCC pVCpu)
+{
+    PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    return RT_BOOL(pApicCpu->uApicBaseMsr & MSR_IA32_APICBASE_EN);
+}
+
+
+/**
+ * Finds the most significant set bit in an APIC 256-bit sparse register.
+ *
+ * @returns @a rcNotFound if no bit was set, 0-255 otherwise.
+ * @param   pReg            The APIC 256-bit sparse register.
+ * @param   rcNotFound      What to return when no bit is set.
+ */
+static int apicGetHighestSetBitInReg(volatile const XAPIC256BITREG *pReg, int rcNotFound)
+{
+    ssize_t const  cFragments     = RT_ELEMENTS(pReg->u);
+    unsigned const uFragmentShift = 5;
+    AssertCompile(1 << uFragmentShift == sizeof(pReg->u[0].u32Reg) * 8);
+    for (ssize_t i = cFragments - 1; i >= 0; i--)
+    {
+        uint32_t const uFragment = pReg->u[i].u32Reg;
+        if (uFragment)
+        {
+            unsigned idxSetBit = ASMBitLastSetU32(uFragment);
+            --idxSetBit;
+            idxSetBit |= i << uFragmentShift;
+            return idxSetBit;
+        }
+    }
+    return rcNotFound;
+}
+
+
+/**
+ * Reads a 32-bit register at a specified offset.
+ *
+ * @returns The value at the specified offset.
+ * @param   pXApicPage      The xAPIC page.
+ * @param   offReg          The offset of the register being read.
+ */
+DECLINLINE(uint32_t) apicReadRaw32(PCXAPICPAGE pXApicPage, uint16_t offReg)
+{
+    Assert(offReg < sizeof(*pXApicPage) - sizeof(uint32_t));
+    uint8_t  const *pbXApic =  (const uint8_t *)pXApicPage;
+    uint32_t const  uValue  = *(const uint32_t *)(pbXApic + offReg);
+    return uValue;
+}
+
+
+/**
+ * Writes a 32-bit register at a specified offset.
+ *
+ * @param   pXApicPage      The xAPIC page.
+ * @param   offReg          The offset of the register being written.
+ * @param   uReg            The value of the register.
+ */
+DECLINLINE(void) apicWriteRaw32(PXAPICPAGE pXApicPage, uint16_t offReg, uint32_t uReg)
+{
+    Assert(offReg < sizeof(*pXApicPage) - sizeof(uint32_t));
+    uint8_t *pbXApic = (uint8_t *)pXApicPage;
+    *(uint32_t *)(pbXApic + offReg) = uReg;
+}
+
+
+/**
+ * Sets an error in the internal ESR of the specified APIC.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uError          The error.
+ * @thread  Any.
+ */
+DECLINLINE(void) apicSetError(PVMCPUCC pVCpu, uint32_t uError)
+{
+    PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    ASMAtomicOrU32(&pApicCpu->uEsrInternal, uError);
+}
+
+
+/**
+ * Clears all errors in the internal ESR.
+ *
+ * @returns The value of the internal ESR before clearing.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+DECLINLINE(uint32_t) apicClearAllErrors(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    return ASMAtomicXchgU32(&pApicCpu->uEsrInternal, 0);
+}
+
+
+/**
+ * Signals the guest if a pending interrupt is ready to be serviced.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static void apicSignalNextPendingIntr(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    if (pXApicPage->svr.u.fApicSoftwareEnable)
+    {
+        int const irrv = apicGetHighestSetBitInReg(&pXApicPage->irr, -1 /* rcNotFound */);
+        if (irrv >= 0)
+        {
+            Assert(irrv <= (int)UINT8_MAX);
+            uint8_t const uVector = irrv;
+            int const isrv        = apicGetHighestSetBitInReg(&pXApicPage->isr, 0 /* rcNotFound */);
+            Assert(isrv <= (int)UINT8_MAX);
+            uint8_t const uIsrVec = isrv;
+
+            /* uIsrVect reflects the highest interrupt vector currently serviced (i.e. in ISR),
+             * or zero if there's none. We want to report a pending interrupt only if IRR > ISR but
+             * regardless of TPR. Hence we can't look at the PPR value, since that also reflects TPR.
+             * NB: The APIC emulation will know when ISR changes, but not necessarily when TPR does.
+             */
+            if (XAPIC_PPR_GET_PP(uVector) > XAPIC_PPR_GET_PP(uIsrVec))
+            {
+                Log2(("APIC%u: apicSignalNextPendingIntr: Signalling pending interrupt. uVector=%#x\n", pVCpu->idCpu, uVector));
+                apicSetInterruptFF(pVCpu, PDMAPICIRQ_HARDWARE);
+            }
+            else
+                Log2(("APIC%u: apicSignalNextPendingIntr: Nothing to signal yet. uVector=%#x uIsrVec=%#x\n", pVCpu->idCpu, uVector, uIsrVec));
+        }
+    }
+    else
+    {
+        Log2(("APIC%u: apicSignalNextPendingIntr: APIC software-disabled, clearing pending interrupt\n", pVCpu->idCpu));
+        apicClearInterruptFF(pVCpu, PDMAPICIRQ_HARDWARE);
+    }
+}
+
+
+/**
+ * Sets the Spurious-Interrupt Vector Register (SVR).
+ *
+ * @returns VINF_SUCCESS or VERR_CPUM_RAISE_GP_0.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uSvr            The SVR value.
+ */
+static int apicSetSvr(PVMCPUCC pVCpu, uint32_t uSvr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    uint32_t   uValidMask = XAPIC_SVR_VALID;
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    if (pXApicPage->version.u.fEoiBroadcastSupression)
+        uValidMask |= XAPIC_SVR_SUPRESS_EOI_BROADCAST;
+
+    if (   XAPIC_IN_X2APIC_MODE(pVCpu)
+        && (uSvr & ~uValidMask))
+        return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_SVR, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    Log2(("APIC%u: apicSetSvr: uSvr=%#RX32\n", pVCpu->idCpu, uSvr));
+    apicWriteRaw32(pXApicPage, XAPIC_OFF_SVR, uSvr);
+    if (!pXApicPage->svr.u.fApicSoftwareEnable)
+    {
+        /** @todo CMCI. */
+        pXApicPage->lvt_timer.u.u1Mask   = 1;
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+        pXApicPage->lvt_thermal.u.u1Mask = 1;
+#endif
+        pXApicPage->lvt_perf.u.u1Mask    = 1;
+        pXApicPage->lvt_lint0.u.u1Mask   = 1;
+        pXApicPage->lvt_lint1.u.u1Mask   = 1;
+        pXApicPage->lvt_error.u.u1Mask   = 1;
+    }
+
+    apicSignalNextPendingIntr(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sends an interrupt to one or more APICs.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure, can be
+ *                              NULL if the source of the interrupt is not an
+ *                              APIC (for e.g. a bus).
+ * @param   uVector             The interrupt vector.
+ * @param   enmTriggerMode      The trigger mode.
+ * @param   enmDeliveryMode     The delivery mode.
+ * @param   pDestCpuSet         The destination CPU set.
+ * @param   pfIntrAccepted      Where to store whether this interrupt was
+ *                              accepted by the target APIC(s) or not.
+ *                              Optional, can be NULL.
+ * @param   uSrcTag             The interrupt source tag (debugging).
+ * @param   rcRZ                The return code if the operation cannot be
+ *                              performed in the current context.
+ */
+static VBOXSTRICTRC apicSendIntr(PVMCC pVM, PVMCPUCC pVCpu, uint8_t uVector, XAPICTRIGGERMODE enmTriggerMode,
+                                 XAPICDELIVERYMODE enmDeliveryMode, PCVMCPUSET pDestCpuSet, bool *pfIntrAccepted,
+                                 uint32_t uSrcTag, int rcRZ)
+{
+    AssertCompile(sizeof(pVM->apic.s) <= sizeof(pVM->apic.padding));
+    AssertCompile(sizeof(pVCpu->apic.s) <= sizeof(pVCpu->apic.padding));
+#ifdef IN_RING0
+    AssertCompile(sizeof(pVM->apicr0.s) <= sizeof(pVM->apicr0.padding));
+#endif
+    VBOXSTRICTRC  rcStrict  = VINF_SUCCESS;
+    VMCPUID const cCpus     = pVM->cCpus;
+    bool          fAccepted = false;
+    switch (enmDeliveryMode)
+    {
+        case XAPICDELIVERYMODE_FIXED:
+        {
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    PVMCPUCC pItVCpu = pVM->CTX_SUFF(apCpus)[idCpu];
+                    if (APICIsEnabled(pItVCpu))
+                        fAccepted = apicPostInterrupt(pItVCpu, uVector, enmTriggerMode, uSrcTag);
+                }
+            break;
+        }
+
+        case XAPICDELIVERYMODE_LOWEST_PRIO:
+        {
+            VMCPUID const idCpu = VMCPUSET_FIND_FIRST_PRESENT(pDestCpuSet);
+            AssertMsgBreak(idCpu < pVM->cCpus, ("APIC: apicSendIntr: No CPU found for lowest-priority delivery mode! idCpu=%u\n", idCpu));
+            PVMCPUCC pVCpuDst = pVM->CTX_SUFF(apCpus)[idCpu];
+            if (APICIsEnabled(pVCpuDst))
+                fAccepted = apicPostInterrupt(pVCpuDst, uVector, enmTriggerMode, uSrcTag);
+            else
+                AssertMsgFailed(("APIC: apicSendIntr: Target APIC not enabled in lowest-priority delivery mode! idCpu=%u\n", idCpu));
+            break;
+        }
+
+        case XAPICDELIVERYMODE_SMI:
+        {
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    Log2(("APIC: apicSendIntr: Raising SMI on VCPU%u\n", idCpu));
+                    apicSetInterruptFF(pVM->CTX_SUFF(apCpus)[idCpu], PDMAPICIRQ_SMI);
+                    fAccepted = true;
+                }
+            break;
+        }
+
+        case XAPICDELIVERYMODE_NMI:
+        {
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    PVMCPUCC pItVCpu = pVM->CTX_SUFF(apCpus)[idCpu];
+                    if (APICIsEnabled(pItVCpu))
+                    {
+                        Log2(("APIC: apicSendIntr: Raising NMI on VCPU%u\n", idCpu));
+                        apicSetInterruptFF(pItVCpu, PDMAPICIRQ_NMI);
+                        fAccepted = true;
+                    }
+                }
+            break;
+        }
+
+        case XAPICDELIVERYMODE_INIT:
+        {
+#ifdef IN_RING3
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    Log2(("APIC: apicSendIntr: Issuing INIT to VCPU%u\n", idCpu));
+                    VMMR3SendInitIpi(pVM, idCpu);
+                    fAccepted = true;
+                }
+#else
+            /* We need to return to ring-3 to deliver the INIT. */
+            rcStrict = rcRZ;
+            fAccepted = true;
+#endif
+            break;
+        }
+
+        case XAPICDELIVERYMODE_STARTUP:
+        {
+#ifdef IN_RING3
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    Log2(("APIC: apicSendIntr: Issuing SIPI to VCPU%u\n", idCpu));
+                    VMMR3SendStartupIpi(pVM, idCpu, uVector);
+                    fAccepted = true;
+                }
+#else
+            /* We need to return to ring-3 to deliver the SIPI. */
+            rcStrict = rcRZ;
+            fAccepted = true;
+            Log2(("APIC: apicSendIntr: SIPI issued, returning to RZ. rc=%Rrc\n", rcRZ));
+#endif
+            break;
+        }
+
+        case XAPICDELIVERYMODE_EXTINT:
+        {
+            for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+                if (VMCPUSET_IS_PRESENT(pDestCpuSet, idCpu))
+                {
+                    Log2(("APIC: apicSendIntr: Raising EXTINT on VCPU%u\n", idCpu));
+                    apicSetInterruptFF(pVM->CTX_SUFF(apCpus)[idCpu], PDMAPICIRQ_EXTINT);
+                    fAccepted = true;
+                }
+            break;
+        }
+
+        default:
+        {
+            AssertMsgFailed(("APIC: apicSendIntr: Unsupported delivery mode %#x (%s)\n", enmDeliveryMode,
+                             apicGetDeliveryModeName(enmDeliveryMode)));
+            break;
+        }
+    }
+
+    /*
+     * If an illegal vector is programmed, set the 'send illegal vector' error here if the
+     * interrupt is being sent by an APIC.
+     *
+     * The 'receive illegal vector' will be set on the target APIC when the interrupt
+     * gets generated, see apicPostInterrupt().
+     *
+     * See Intel spec. 10.5.3 "Error Handling".
+     */
+    if (   rcStrict != rcRZ
+        && pVCpu)
+    {
+        /*
+         * Flag only errors when the delivery mode is fixed and not others.
+         *
+         * Ubuntu 10.04-3 amd64 live CD with 2 VCPUs gets upset as it sends an SIPI to the
+         * 2nd VCPU with vector 6 and checks the ESR for no errors, see @bugref{8245#c86}.
+         */
+        /** @todo The spec says this for LVT, but not explcitly for ICR-lo
+         *        but it probably is true. */
+        if (enmDeliveryMode == XAPICDELIVERYMODE_FIXED)
+        {
+            if (RT_UNLIKELY(uVector <= XAPIC_ILLEGAL_VECTOR_END))
+                apicSetError(pVCpu, XAPIC_ESR_SEND_ILLEGAL_VECTOR);
+        }
+    }
+
+    if (pfIntrAccepted)
+        *pfIntrAccepted = fAccepted;
+
+    return rcStrict;
+}
+
+
+/**
+ * Checks if this APIC belongs to a logical destination.
+ *
+ * @returns true if the APIC belongs to the logical
+ *          destination, false otherwise.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   fDest                   The destination mask.
+ *
+ * @thread  Any.
+ */
+static bool apicIsLogicalDest(PVMCPUCC pVCpu, uint32_t fDest)
+{
+    if (XAPIC_IN_X2APIC_MODE(pVCpu))
+    {
+        /*
+         * Flat logical mode is not supported in x2APIC mode.
+         * In clustered logical mode, the 32-bit logical ID in the LDR is interpreted as follows:
+         *    - High 16 bits is the cluster ID.
+         *    - Low 16 bits: each bit represents a unique APIC within the cluster.
+         */
+        PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpu);
+        uint32_t const u32Ldr    = pX2ApicPage->ldr.u32LogicalApicId;
+        if (X2APIC_LDR_GET_CLUSTER_ID(u32Ldr) == (fDest & X2APIC_LDR_CLUSTER_ID))
+            return RT_BOOL(u32Ldr & fDest & X2APIC_LDR_LOGICAL_ID);
+        return false;
+    }
+
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    /*
+     * In both flat and clustered logical mode, a destination mask of all set bits indicates a broadcast.
+     * See AMD spec. 16.6.1 "Receiving System and IPI Interrupts".
+     */
+    Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+    if ((fDest & XAPIC_LDR_FLAT_LOGICAL_ID) == XAPIC_LDR_FLAT_LOGICAL_ID)
+        return true;
+
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    XAPICDESTFORMAT enmDestFormat = (XAPICDESTFORMAT)pXApicPage->dfr.u.u4Model;
+    if (enmDestFormat == XAPICDESTFORMAT_FLAT)
+    {
+        /* The destination mask is interpreted as a bitmap of 8 unique logical APIC IDs. */
+        uint8_t const u8Ldr = pXApicPage->ldr.u.u8LogicalApicId;
+        return RT_BOOL(u8Ldr & fDest & XAPIC_LDR_FLAT_LOGICAL_ID);
+    }
+
+    /*
+     * In clustered logical mode, the 8-bit logical ID in the LDR is interpreted as follows:
+     *    - High 4 bits is the cluster ID.
+     *    - Low 4 bits: each bit represents a unique APIC within the cluster.
+     */
+    Assert(enmDestFormat == XAPICDESTFORMAT_CLUSTER);
+    uint8_t const u8Ldr = pXApicPage->ldr.u.u8LogicalApicId;
+    if (XAPIC_LDR_CLUSTERED_GET_CLUSTER_ID(u8Ldr) == (fDest & XAPIC_LDR_CLUSTERED_CLUSTER_ID))
+        return RT_BOOL(u8Ldr & fDest & XAPIC_LDR_CLUSTERED_LOGICAL_ID);
+    return false;
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+}
+
+
+/**
+ * Figures out the set of destination CPUs for a given destination mode, format
+ * and delivery mode setting.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fDestMask       The destination mask.
+ * @param   fBroadcastMask  The broadcast mask.
+ * @param   enmDestMode     The destination mode.
+ * @param   enmDeliveryMode The delivery mode.
+ * @param   pDestCpuSet     The destination CPU set to update.
+ */
+static void apicGetDestCpuSet(PVMCC pVM, uint32_t fDestMask, uint32_t fBroadcastMask, XAPICDESTMODE enmDestMode,
+                              XAPICDELIVERYMODE enmDeliveryMode, PVMCPUSET pDestCpuSet)
+{
+    VMCPUSET_EMPTY(pDestCpuSet);
+
+    /*
+     * Physical destination mode only supports either a broadcast or a single target.
+     *    - Broadcast with lowest-priority delivery mode is not supported[1], we deliver it
+     *      as a regular broadcast like in fixed delivery mode.
+     *    - For a single target, lowest-priority delivery mode makes no sense. We deliver
+     *      to the target like in fixed delivery mode.
+     *
+     * [1] See Intel spec. 10.6.2.1 "Physical Destination Mode".
+     */
+    if (   enmDestMode == XAPICDESTMODE_PHYSICAL
+        && enmDeliveryMode == XAPICDELIVERYMODE_LOWEST_PRIO)
+    {
+        AssertMsgFailed(("APIC: Lowest-priority delivery using physical destination mode!"));
+        enmDeliveryMode = XAPICDELIVERYMODE_FIXED;
+    }
+
+    uint32_t const cCpus = pVM->cCpus;
+    if (enmDeliveryMode == XAPICDELIVERYMODE_LOWEST_PRIO)
+    {
+        Assert(enmDestMode == XAPICDESTMODE_LOGICAL);
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+        VMCPUID idCpuLowestTpr = NIL_VMCPUID;
+        uint8_t u8LowestTpr    = UINT8_C(0xff);
+        for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+        {
+            PVMCPUCC pVCpuDst = pVM->CTX_SUFF(apCpus)[idCpu];
+            if (apicIsLogicalDest(pVCpuDst, fDestMask))
+            {
+                PCXAPICPAGE   pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpuDst);
+                uint8_t const u8Tpr      = pXApicPage->tpr.u8Tpr;         /* PAV */
+
+                /*
+                 * If there is a tie for lowest priority, the local APIC with the highest ID is chosen.
+                 * Hence the use of "<=" in the check below.
+                 * See AMD spec. 16.6.2 "Lowest Priority Messages and Arbitration".
+                 */
+                if (u8Tpr <= u8LowestTpr)
+                {
+                    u8LowestTpr    = u8Tpr;
+                    idCpuLowestTpr = idCpu;
+                }
+            }
+        }
+        if (idCpuLowestTpr != NIL_VMCPUID)
+            VMCPUSET_ADD(pDestCpuSet, idCpuLowestTpr);
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+        return;
+    }
+
+    /*
+     * x2APIC:
+     *    - In both physical and logical destination mode, a destination mask of 0xffffffff implies a broadcast[1].
+     * xAPIC:
+     *    - In physical destination mode, a destination mask of 0xff implies a broadcast[2].
+     *    - In both flat and clustered logical mode, a destination mask of 0xff implies a broadcast[3].
+     *
+     * [1] See Intel spec. 10.12.9 "ICR Operation in x2APIC Mode".
+     * [2] See Intel spec. 10.6.2.1 "Physical Destination Mode".
+     * [2] See AMD spec. 16.6.1 "Receiving System and IPI Interrupts".
+     */
+    if ((fDestMask & fBroadcastMask) == fBroadcastMask)
+    {
+        VMCPUSET_FILL(pDestCpuSet);
+        return;
+    }
+
+    if (enmDestMode == XAPICDESTMODE_PHYSICAL)
+    {
+        /* The destination mask is interpreted as the physical APIC ID of a single target. */
+#if 1
+        /* Since our physical APIC ID is read-only to software, set the corresponding bit in the CPU set. */
+        if (RT_LIKELY(fDestMask < cCpus))
+            VMCPUSET_ADD(pDestCpuSet, fDestMask);
+#else
+        /* The physical APIC ID may not match our VCPU ID, search through the list of targets. */
+        for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+        {
+            PVMCPUCC pVCpuDst = &pVM->aCpus[idCpu];
+            if (XAPIC_IN_X2APIC_MODE(pVCpuDst))
+            {
+                PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpuDst);
+                if (pX2ApicPage->id.u32ApicId == fDestMask)
+                    VMCPUSET_ADD(pDestCpuSet, pVCpuDst->idCpu);
+            }
+            else
+            {
+                PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpuDst);
+                if (pXApicPage->id.u8ApicId == (uint8_t)fDestMask)
+                    VMCPUSET_ADD(pDestCpuSet, pVCpuDst->idCpu);
+            }
+        }
+#endif
+    }
+    else
+    {
+        Assert(enmDestMode == XAPICDESTMODE_LOGICAL);
+
+        /* A destination mask of all 0's implies no target APICs (since it's interpreted as a bitmap or partial bitmap). */
+        if (RT_UNLIKELY(!fDestMask))
+            return;
+
+        /* The destination mask is interpreted as a bitmap of software-programmable logical APIC ID of the target APICs. */
+        for (VMCPUID idCpu = 0; idCpu < cCpus; idCpu++)
+        {
+            PVMCPUCC pVCpuDst = pVM->CTX_SUFF(apCpus)[idCpu];
+            if (apicIsLogicalDest(pVCpuDst, fDestMask))
+                VMCPUSET_ADD(pDestCpuSet, pVCpuDst->idCpu);
+        }
+    }
+}
+
+
+/**
+ * Sends an Interprocessor Interrupt (IPI) using values from the Interrupt
+ * Command Register (ICR).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   rcRZ            The return code if the operation cannot be
+ *                          performed in the current context.
+ */
+DECLINLINE(VBOXSTRICTRC) apicSendIpi(PVMCPUCC pVCpu, int rcRZ)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    XAPICDELIVERYMODE const  enmDeliveryMode  = (XAPICDELIVERYMODE)pXApicPage->icr_lo.u.u3DeliveryMode;
+    XAPICDESTMODE const      enmDestMode      = (XAPICDESTMODE)pXApicPage->icr_lo.u.u1DestMode;
+    XAPICINITLEVEL const     enmInitLevel     = (XAPICINITLEVEL)pXApicPage->icr_lo.u.u1Level;
+    XAPICTRIGGERMODE const   enmTriggerMode   = (XAPICTRIGGERMODE)pXApicPage->icr_lo.u.u1TriggerMode;
+    XAPICDESTSHORTHAND const enmDestShorthand = (XAPICDESTSHORTHAND)pXApicPage->icr_lo.u.u2DestShorthand;
+    uint8_t const            uVector          = pXApicPage->icr_lo.u.u8Vector;
+
+    PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu);
+    uint32_t const fDest    = XAPIC_IN_X2APIC_MODE(pVCpu) ? pX2ApicPage->icr_hi.u32IcrHi : pXApicPage->icr_hi.u.u8Dest;
+    Log5(("apicSendIpi: delivery=%u mode=%u init=%u trigger=%u short=%u vector=%#x fDest=%#x\n",
+          enmDeliveryMode, enmDestMode, enmInitLevel, enmTriggerMode, enmDestShorthand, uVector, fDest));
+
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    /*
+     * INIT Level De-assert is not support on Pentium 4 and Xeon processors.
+     * Apparently, this also applies to NMI, SMI, lowest-priority and fixed delivery modes,
+     * see @bugref{8245#c116}.
+     *
+     * See AMD spec. 16.5 "Interprocessor Interrupts (IPI)" for a table of valid ICR combinations.
+     */
+    if (   enmTriggerMode  == XAPICTRIGGERMODE_LEVEL
+        && enmInitLevel    == XAPICINITLEVEL_DEASSERT
+        && (   enmDeliveryMode == XAPICDELIVERYMODE_FIXED
+            || enmDeliveryMode == XAPICDELIVERYMODE_LOWEST_PRIO
+            || enmDeliveryMode == XAPICDELIVERYMODE_SMI
+            || enmDeliveryMode == XAPICDELIVERYMODE_NMI
+            || enmDeliveryMode == XAPICDELIVERYMODE_INIT))
+    {
+        Log2(("APIC%u: %s level de-assert unsupported, ignoring!\n", pVCpu->idCpu, apicGetDeliveryModeName(enmDeliveryMode)));
+        return VINF_SUCCESS;
+    }
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+
+    /*
+     * The destination and delivery modes are ignored/by-passed when a destination shorthand is specified.
+     * See Intel spec. 10.6.2.3 "Broadcast/Self Delivery Mode".
+     */
+    VMCPUSET DestCpuSet;
+    switch (enmDestShorthand)
+    {
+        case XAPICDESTSHORTHAND_NONE:
+        {
+            PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+            uint32_t const fBroadcastMask = XAPIC_IN_X2APIC_MODE(pVCpu) ? X2APIC_ID_BROADCAST_MASK : XAPIC_ID_BROADCAST_MASK;
+            apicGetDestCpuSet(pVM, fDest, fBroadcastMask, enmDestMode, enmDeliveryMode, &DestCpuSet);
+            break;
+        }
+
+        case XAPICDESTSHORTHAND_SELF:
+        {
+            VMCPUSET_EMPTY(&DestCpuSet);
+            VMCPUSET_ADD(&DestCpuSet, pVCpu->idCpu);
+            break;
+        }
+
+        case XAPIDDESTSHORTHAND_ALL_INCL_SELF:
+        {
+            VMCPUSET_FILL(&DestCpuSet);
+            break;
+        }
+
+        case XAPICDESTSHORTHAND_ALL_EXCL_SELF:
+        {
+            VMCPUSET_FILL(&DestCpuSet);
+            VMCPUSET_DEL(&DestCpuSet, pVCpu->idCpu);
+            break;
+        }
+    }
+
+    return apicSendIntr(pVCpu->CTX_SUFF(pVM), pVCpu, uVector, enmTriggerMode, enmDeliveryMode, &DestCpuSet,
+                        NULL /* pfIntrAccepted */, 0 /* uSrcTag */, rcRZ);
+}
+
+
+/**
+ * Sets the Interrupt Command Register (ICR) high dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uIcrHi          The ICR high dword.
+ */
+static VBOXSTRICTRC apicSetIcrHi(PVMCPUCC pVCpu, uint32_t uIcrHi)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    pXApicPage->icr_hi.all.u32IcrHi = uIcrHi & XAPIC_ICR_HI_DEST;
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatIcrHiWrite);
+    Log2(("APIC%u: apicSetIcrHi: uIcrHi=%#RX32\n", pVCpu->idCpu, pXApicPage->icr_hi.all.u32IcrHi));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the Interrupt Command Register (ICR) low dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uIcrLo          The ICR low dword.
+ * @param   rcRZ            The return code if the operation cannot be performed
+ *                          in the current context.
+ * @param   fUpdateStat     Whether to update the ICR low write statistics
+ *                          counter.
+ */
+static VBOXSTRICTRC apicSetIcrLo(PVMCPUCC pVCpu, uint32_t uIcrLo, int rcRZ, bool fUpdateStat)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PXAPICPAGE pXApicPage  = VMCPU_TO_XAPICPAGE(pVCpu);
+    pXApicPage->icr_lo.all.u32IcrLo = uIcrLo & XAPIC_ICR_LO_WR_VALID;
+    Log2(("APIC%u: apicSetIcrLo: uIcrLo=%#RX32\n", pVCpu->idCpu, pXApicPage->icr_lo.all.u32IcrLo));
+
+    if (fUpdateStat)
+        STAM_COUNTER_INC(&pVCpu->apic.s.StatIcrLoWrite);
+    RT_NOREF(fUpdateStat);
+
+    return apicSendIpi(pVCpu, rcRZ);
+}
+
+
+/**
+ * Sets the Interrupt Command Register (ICR).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u64Icr          The ICR (High and Low combined).
+ * @param   rcRZ            The return code if the operation cannot be performed
+ *                          in the current context.
+ *
+ * @remarks This function is used by both x2APIC interface and the Hyper-V
+ *          interface, see APICHvSetIcr. The Hyper-V spec isn't clear what
+ *          happens when invalid bits are set. For the time being, it will
+ *          \#GP like a regular x2APIC access.
+ */
+static VBOXSTRICTRC apicSetIcr(PVMCPUCC pVCpu, uint64_t u64Icr, int rcRZ)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /* Validate. */
+    uint32_t const uLo = RT_LO_U32(u64Icr);
+    if (RT_LIKELY(!(uLo & ~XAPIC_ICR_LO_WR_VALID)))
+    {
+        /* Update high dword first, then update the low dword which sends the IPI. */
+        PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu);
+        pX2ApicPage->icr_hi.u32IcrHi = RT_HI_U32(u64Icr);
+        STAM_COUNTER_INC(&pVCpu->apic.s.StatIcrFullWrite);
+        return apicSetIcrLo(pVCpu, uLo, rcRZ, false /* fUpdateStat */);
+    }
+    return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_ICR, APICMSRACCESS_WRITE_RSVD_BITS);
+}
+
+
+/**
+ * Sets the Error Status Register (ESR).
+ *
+ * @returns VINF_SUCCESS or VERR_CPUM_RAISE_GP_0.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uEsr            The ESR value.
+ */
+static int apicSetEsr(PVMCPUCC pVCpu, uint32_t uEsr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    Log2(("APIC%u: apicSetEsr: uEsr=%#RX32\n", pVCpu->idCpu, uEsr));
+
+    if (   XAPIC_IN_X2APIC_MODE(pVCpu)
+        && (uEsr & ~XAPIC_ESR_WO_VALID))
+        return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_ESR, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    /*
+     * Writes to the ESR causes the internal state to be updated in the register,
+     * clearing the original state. See AMD spec. 16.4.6 "APIC Error Interrupts".
+     */
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    pXApicPage->esr.all.u32Errors = apicClearAllErrors(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Updates the Processor Priority Register (PPR).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static void apicUpdatePpr(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /* See Intel spec 10.8.3.1 "Task and Processor Priorities". */
+    PXAPICPAGE    pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    uint8_t const uIsrv      = apicGetHighestSetBitInReg(&pXApicPage->isr, 0 /* rcNotFound */);
+    uint8_t       uPpr;
+    if (XAPIC_TPR_GET_TP(pXApicPage->tpr.u8Tpr) >= XAPIC_PPR_GET_PP(uIsrv))
+        uPpr = pXApicPage->tpr.u8Tpr;
+    else
+        uPpr = XAPIC_PPR_GET_PP(uIsrv);
+    pXApicPage->ppr.u8Ppr = uPpr;
+}
+
+
+/**
+ * Gets the Processor Priority Register (PPR).
+ *
+ * @returns The PPR value.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static uint8_t apicGetPpr(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatTprRead);
+
+    /*
+     * With virtualized APIC registers or with TPR virtualization, the hardware may
+     * update ISR/TPR transparently. We thus re-calculate the PPR which may be out of sync.
+     * See Intel spec. 29.2.2 "Virtual-Interrupt Delivery".
+     *
+     * In all other instances, whenever the TPR or ISR changes, we need to update the PPR
+     * as well (e.g. like we do manually in apicR3InitIpi and by calling apicUpdatePpr).
+     */
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    if (pApic->fVirtApicRegsEnabled)        /** @todo re-think this */
+        apicUpdatePpr(pVCpu);
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    return pXApicPage->ppr.u8Ppr;
+}
+
+
+/**
+ * Sets the Task Priority Register (TPR).
+ *
+ * @returns VINF_SUCCESS or VERR_CPUM_RAISE_GP_0.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   uTpr                    The TPR value.
+ * @param   fForceX2ApicBehaviour   Pretend the APIC is in x2APIC mode during
+ *                                  this write.
+ */
+static int apicSetTprEx(PVMCPUCC pVCpu, uint32_t uTpr, bool fForceX2ApicBehaviour)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    Log2(("APIC%u: apicSetTprEx: uTpr=%#RX32\n", pVCpu->idCpu, uTpr));
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatTprWrite);
+
+    bool const fX2ApicMode = XAPIC_IN_X2APIC_MODE(pVCpu) || fForceX2ApicBehaviour;
+    if (   fX2ApicMode
+        && (uTpr & ~XAPIC_TPR_VALID))
+        return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_TPR, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    pXApicPage->tpr.u8Tpr = uTpr;
+    apicUpdatePpr(pVCpu);
+    apicSignalNextPendingIntr(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the End-Of-Interrupt (EOI) register.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   uEoi                    The EOI value.
+ * @param   fForceX2ApicBehaviour   Pretend the APIC is in x2APIC mode during
+ *                                  this write.
+ */
+static VBOXSTRICTRC apicSetEoi(PVMCPUCC pVCpu, uint32_t uEoi, bool fForceX2ApicBehaviour)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    Log2(("APIC%u: apicSetEoi: uEoi=%#RX32\n", pVCpu->idCpu, uEoi));
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatEoiWrite);
+
+    bool const fX2ApicMode = XAPIC_IN_X2APIC_MODE(pVCpu) || fForceX2ApicBehaviour;
+    if (   fX2ApicMode
+        && (uEoi & ~XAPIC_EOI_WO_VALID))
+        return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_EOI, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    int isrv = apicGetHighestSetBitInReg(&pXApicPage->isr, -1 /* rcNotFound */);
+    if (isrv >= 0)
+    {
+        /*
+         * Broadcast the EOI to the I/O APIC(s).
+         *
+         * We'll handle the EOI broadcast first as there is tiny chance we get rescheduled to
+         * ring-3 due to contention on the I/O APIC lock. This way we don't mess with the rest
+         * of the APIC state and simply restart the EOI write operation from ring-3.
+         */
+        Assert(isrv <= (int)UINT8_MAX);
+        uint8_t const uVector      = isrv;
+        bool const fLevelTriggered = apicTestVectorInReg(&pXApicPage->tmr, uVector);
+        if (fLevelTriggered)
+        {
+            PDMIoApicBroadcastEoi(pVCpu->CTX_SUFF(pVM), uVector);
+
+            /*
+             * Clear the vector from the TMR.
+             *
+             * The broadcast to I/O APIC can re-trigger new interrupts to arrive via the bus. However,
+             * APICUpdatePendingInterrupts() which updates TMR can only be done from EMT which we
+             * currently are on, so no possibility of concurrent updates.
+             */
+            apicClearVectorInReg(&pXApicPage->tmr, uVector);
+
+            /*
+             * Clear the remote IRR bit for level-triggered, fixed mode LINT0 interrupt.
+             * The LINT1 pin does not support level-triggered interrupts.
+             * See Intel spec. 10.5.1 "Local Vector Table".
+             */
+            uint32_t const uLvtLint0 = pXApicPage->lvt_lint0.all.u32LvtLint0;
+            if (   XAPIC_LVT_GET_REMOTE_IRR(uLvtLint0)
+                && XAPIC_LVT_GET_VECTOR(uLvtLint0) == uVector
+                && XAPIC_LVT_GET_DELIVERY_MODE(uLvtLint0) == XAPICDELIVERYMODE_FIXED)
+            {
+                ASMAtomicAndU32((volatile uint32_t *)&pXApicPage->lvt_lint0.all.u32LvtLint0, ~XAPIC_LVT_REMOTE_IRR);
+                Log2(("APIC%u: apicSetEoi: Cleared remote-IRR for LINT0. uVector=%#x\n", pVCpu->idCpu, uVector));
+            }
+
+            Log2(("APIC%u: apicSetEoi: Cleared level triggered interrupt from TMR. uVector=%#x\n", pVCpu->idCpu, uVector));
+        }
+
+        /*
+         * Mark interrupt as serviced, update the PPR and signal pending interrupts.
+         */
+        Log2(("APIC%u: apicSetEoi: Clearing interrupt from ISR. uVector=%#x\n", pVCpu->idCpu, uVector));
+        apicClearVectorInReg(&pXApicPage->isr, uVector);
+        apicUpdatePpr(pVCpu);
+        apicSignalNextPendingIntr(pVCpu);
+    }
+    else
+    {
+#ifdef DEBUG_ramshankar
+        /** @todo Figure out if this is done intentionally by guests or is a bug
+         *   in our emulation. Happened with Win10 SMP VM during reboot after
+         *   installation of guest additions with 3D support. */
+        AssertMsgFailed(("APIC%u: apicSetEoi: Failed to find any ISR bit\n", pVCpu->idCpu));
+#endif
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the Logical Destination Register (LDR).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uLdr            The LDR value.
+ *
+ * @remarks LDR is read-only in x2APIC mode.
+ */
+static VBOXSTRICTRC apicSetLdr(PVMCPUCC pVCpu, uint32_t uLdr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    Assert(!XAPIC_IN_X2APIC_MODE(pVCpu) || pApic->fHyperVCompatMode); RT_NOREF_PV(pApic);
+
+    Log2(("APIC%u: apicSetLdr: uLdr=%#RX32\n", pVCpu->idCpu, uLdr));
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    apicWriteRaw32(pXApicPage, XAPIC_OFF_LDR, uLdr & XAPIC_LDR_VALID);
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatLdrWrite);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the Destination Format Register (DFR).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uDfr            The DFR value.
+ *
+ * @remarks DFR is not available in x2APIC mode.
+ */
+static VBOXSTRICTRC apicSetDfr(PVMCPUCC pVCpu, uint32_t uDfr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+
+    uDfr &= XAPIC_DFR_VALID;
+    uDfr |= XAPIC_DFR_RSVD_MB1;
+
+    Log2(("APIC%u: apicSetDfr: uDfr=%#RX32\n", pVCpu->idCpu, uDfr));
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    apicWriteRaw32(pXApicPage, XAPIC_OFF_DFR, uDfr);
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatDfrWrite);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the Timer Divide Configuration Register (DCR).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uTimerDcr       The timer DCR value.
+ */
+static VBOXSTRICTRC apicSetTimerDcr(PVMCPUCC pVCpu, uint32_t uTimerDcr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    if (   XAPIC_IN_X2APIC_MODE(pVCpu)
+        && (uTimerDcr & ~XAPIC_TIMER_DCR_VALID))
+        return apicMsrAccessError(pVCpu, MSR_IA32_X2APIC_TIMER_DCR, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    Log2(("APIC%u: apicSetTimerDcr: uTimerDcr=%#RX32\n", pVCpu->idCpu, uTimerDcr));
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    apicWriteRaw32(pXApicPage, XAPIC_OFF_TIMER_DCR, uTimerDcr);
+    STAM_COUNTER_INC(&pVCpu->apic.s.StatDcrWrite);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the timer's Current Count Register (CCR).
+ *
+ * @returns VBox status code.
+ * @param   pDevIns         The device instance.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   rcBusy          The busy return code for the timer critical section.
+ * @param   puValue         Where to store the LVT timer CCR.
+ */
+static VBOXSTRICTRC apicGetTimerCcr(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, int rcBusy, uint32_t *puValue)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(puValue);
+
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    *puValue = 0;
+
+    /* In TSC-deadline mode, CCR returns 0, see Intel spec. 10.5.4.1 "TSC-Deadline Mode". */
+    if (pXApicPage->lvt_timer.u.u2TimerMode == XAPIC_TIMER_MODE_TSC_DEADLINE)
+        return VINF_SUCCESS;
+
+    /* If the initial-count register is 0, CCR returns 0 as it cannot exceed the ICR. */
+    uint32_t const uInitialCount = pXApicPage->timer_icr.u32InitialCount;
+    if (!uInitialCount)
+        return VINF_SUCCESS;
+
+    /*
+     * Reading the virtual-sync clock requires locking its timer because it's not
+     * a simple atomic operation, see tmVirtualSyncGetEx().
+     *
+     * We also need to lock before reading the timer CCR, see apicR3TimerCallback().
+     */
+    PCAPICCPU       pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    TMTIMERHANDLE   hTimer   = pApicCpu->hTimer;
+
+    VBOXSTRICTRC rc = PDMDevHlpTimerLockClock(pDevIns, hTimer, rcBusy);
+    if (rc == VINF_SUCCESS)
+    {
+        /* If the current-count register is 0, it implies the timer expired. */
+        uint32_t const uCurrentCount = pXApicPage->timer_ccr.u32CurrentCount;
+        if (uCurrentCount)
+        {
+            uint64_t const cTicksElapsed = PDMDevHlpTimerGet(pDevIns, hTimer) - pApicCpu->u64TimerInitial;
+            PDMDevHlpTimerUnlockClock(pDevIns, hTimer);
+            uint8_t  const uTimerShift   = apicGetTimerShift(pXApicPage);
+            uint64_t const uDelta        = cTicksElapsed >> uTimerShift;
+            if (uInitialCount > uDelta)
+                *puValue = uInitialCount - uDelta;
+        }
+        else
+            PDMDevHlpTimerUnlockClock(pDevIns, hTimer);
+    }
+    return rc;
+}
+
+
+/**
+ * Sets the timer's Initial-Count Register (ICR).
+ *
+ * @returns Strict VBox status code.
+ * @param   pDevIns         The device instance.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   rcBusy          The busy return code for the timer critical section.
+ * @param   uInitialCount   The timer ICR.
+ */
+static VBOXSTRICTRC apicSetTimerIcr(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, int rcBusy, uint32_t uInitialCount)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PAPIC      pApic      = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    PAPICCPU   pApicCpu   = VMCPU_TO_APICCPU(pVCpu);
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+
+    Log2(("APIC%u: apicSetTimerIcr: uInitialCount=%#RX32\n", pVCpu->idCpu, uInitialCount));
+    STAM_COUNTER_INC(&pApicCpu->StatTimerIcrWrite);
+
+    /* In TSC-deadline mode, timer ICR writes are ignored, see Intel spec. 10.5.4.1 "TSC-Deadline Mode". */
+    if (   pApic->fSupportsTscDeadline
+        && pXApicPage->lvt_timer.u.u2TimerMode == XAPIC_TIMER_MODE_TSC_DEADLINE)
+        return VINF_SUCCESS;
+
+    /*
+     * The timer CCR may be modified by apicR3TimerCallback() in parallel,
+     * so obtain the lock -before- updating it here to be consistent with the
+     * timer ICR. We rely on CCR being consistent in apicGetTimerCcr().
+     */
+    TMTIMERHANDLE hTimer = pApicCpu->hTimer;
+    VBOXSTRICTRC rc = PDMDevHlpTimerLockClock(pDevIns, hTimer, rcBusy);
+    if (rc == VINF_SUCCESS)
+    {
+        pXApicPage->timer_icr.u32InitialCount = uInitialCount;
+        pXApicPage->timer_ccr.u32CurrentCount = uInitialCount;
+        if (uInitialCount)
+            apicStartTimer(pVCpu, uInitialCount);
+        else
+            apicStopTimer(pVCpu);
+        PDMDevHlpTimerUnlockClock(pDevIns, hTimer);
+    }
+    return rc;
+}
+
+
+/**
+ * Sets an LVT entry.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   offLvt          The LVT entry offset in the xAPIC page.
+ * @param   uLvt            The LVT value to set.
+ */
+static VBOXSTRICTRC apicSetLvtEntry(PVMCPUCC pVCpu, uint16_t offLvt, uint32_t uLvt)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    AssertMsg(   offLvt == XAPIC_OFF_LVT_TIMER
+              || offLvt == XAPIC_OFF_LVT_THERMAL
+              || offLvt == XAPIC_OFF_LVT_PERF
+              || offLvt == XAPIC_OFF_LVT_LINT0
+              || offLvt == XAPIC_OFF_LVT_LINT1
+              || offLvt == XAPIC_OFF_LVT_ERROR,
+             ("APIC%u: apicSetLvtEntry: invalid offset, offLvt=%#RX16, uLvt=%#RX32\n", pVCpu->idCpu, offLvt, uLvt));
+
+    /*
+     * If TSC-deadline mode isn't support, ignore the bit in xAPIC mode
+     * and raise #GP(0) in x2APIC mode.
+     */
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    if (offLvt == XAPIC_OFF_LVT_TIMER)
+    {
+        STAM_COUNTER_INC(&pVCpu->apic.s.StatLvtTimerWrite);
+        if (   !pApic->fSupportsTscDeadline
+            && (uLvt & XAPIC_LVT_TIMER_TSCDEADLINE))
+        {
+            if (XAPIC_IN_X2APIC_MODE(pVCpu))
+                return apicMsrAccessError(pVCpu, XAPIC_GET_X2APIC_MSR(offLvt), APICMSRACCESS_WRITE_RSVD_BITS);
+            uLvt &= ~XAPIC_LVT_TIMER_TSCDEADLINE;
+            /** @todo TSC-deadline timer mode transition */
+        }
+    }
+
+    /*
+     * Validate rest of the LVT bits.
+     */
+    uint16_t const idxLvt = (offLvt - XAPIC_OFF_LVT_START) >> 4;
+    AssertReturn(idxLvt < RT_ELEMENTS(g_au32LvtValidMasks), VERR_OUT_OF_RANGE);
+
+    /*
+     * For x2APIC, disallow setting of invalid/reserved bits.
+     * For xAPIC, mask out invalid/reserved bits (i.e. ignore them).
+     */
+    if (   XAPIC_IN_X2APIC_MODE(pVCpu)
+        && (uLvt & ~g_au32LvtValidMasks[idxLvt]))
+        return apicMsrAccessError(pVCpu, XAPIC_GET_X2APIC_MSR(offLvt), APICMSRACCESS_WRITE_RSVD_BITS);
+
+    uLvt &= g_au32LvtValidMasks[idxLvt];
+
+    /*
+     * In the software-disabled state, LVT mask-bit must remain set and attempts to clear the mask
+     * bit must be ignored. See Intel spec. 10.4.7.2 "Local APIC State After It Has Been Software Disabled".
+     */
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    if (!pXApicPage->svr.u.fApicSoftwareEnable)
+        uLvt |= XAPIC_LVT_MASK;
+
+    /*
+     * It is unclear whether we should signal a 'send illegal vector' error here and ignore updating
+     * the LVT entry when the delivery mode is 'fixed'[1] or update it in addition to signalling the
+     * error or not signal the error at all. For now, we'll allow setting illegal vectors into the LVT
+     * but set the 'send illegal vector' error here. The 'receive illegal vector' error will be set if
+     * the interrupt for the vector happens to be generated, see apicPostInterrupt().
+     *
+     * [1] See Intel spec. 10.5.2 "Valid Interrupt Vectors".
+     */
+    if (RT_UNLIKELY(   XAPIC_LVT_GET_VECTOR(uLvt) <= XAPIC_ILLEGAL_VECTOR_END
+                    && XAPIC_LVT_GET_DELIVERY_MODE(uLvt) == XAPICDELIVERYMODE_FIXED))
+        apicSetError(pVCpu, XAPIC_ESR_SEND_ILLEGAL_VECTOR);
+
+    Log2(("APIC%u: apicSetLvtEntry: offLvt=%#RX16 uLvt=%#RX32\n", pVCpu->idCpu, offLvt, uLvt));
+
+    apicWriteRaw32(pXApicPage, offLvt, uLvt);
+    return VINF_SUCCESS;
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif  /* XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4 */
+}
+
+
+#if 0
+/**
+ * Sets an LVT entry in the extended LVT range.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   offLvt          The LVT entry offset in the xAPIC page.
+ * @param   uValue          The LVT value to set.
+ */
+static int apicSetLvtExtEntry(PVMCPUCC pVCpu, uint16_t offLvt, uint32_t uLvt)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    AssertMsg(offLvt == XAPIC_OFF_CMCI, ("APIC%u: apicSetLvt1Entry: invalid offset %#RX16\n", pVCpu->idCpu, offLvt));
+
+    /** @todo support CMCI. */
+    return VERR_NOT_IMPLEMENTED;
+}
+#endif
+
+
+/**
+ * Hints TM about the APIC timer frequency.
+ *
+ * @param   pDevIns         The device instance.
+ * @param   pApicCpu        The APIC CPU state.
+ * @param   uInitialCount   The new initial count.
+ * @param   uTimerShift     The new timer shift.
+ * @thread  Any.
+ */
+void apicHintTimerFreq(PPDMDEVINS pDevIns, PAPICCPU pApicCpu, uint32_t uInitialCount, uint8_t uTimerShift)
+{
+    Assert(pApicCpu);
+
+    if (   pApicCpu->uHintedTimerInitialCount != uInitialCount
+        || pApicCpu->uHintedTimerShift        != uTimerShift)
+    {
+        uint32_t uHz;
+        if (uInitialCount)
+        {
+            uint64_t cTicksPerPeriod = (uint64_t)uInitialCount << uTimerShift;
+            uHz = PDMDevHlpTimerGetFreq(pDevIns, pApicCpu->hTimer) / cTicksPerPeriod;
+        }
+        else
+            uHz = 0;
+
+        PDMDevHlpTimerSetFrequencyHint(pDevIns, pApicCpu->hTimer, uHz);
+        pApicCpu->uHintedTimerInitialCount = uInitialCount;
+        pApicCpu->uHintedTimerShift = uTimerShift;
+    }
+}
+
+
+/**
+ * Gets the Interrupt Command Register (ICR), without performing any interface
+ * checks.
+ *
+ * @returns The ICR value.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+DECLINLINE(uint64_t) apicGetIcrNoCheck(PVMCPUCC pVCpu)
+{
+    PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpu);
+    uint64_t const uHi  = pX2ApicPage->icr_hi.u32IcrHi;
+    uint64_t const uLo  = pX2ApicPage->icr_lo.all.u32IcrLo;
+    uint64_t const uIcr = RT_MAKE_U64(uLo, uHi);
+    return uIcr;
+}
+
+
+/**
+ * Reads an APIC register.
+ *
+ * @returns VBox status code.
+ * @param   pDevIns         The device instance.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   offReg          The offset of the register being read.
+ * @param   puValue         Where to store the register value.
+ */
+DECLINLINE(VBOXSTRICTRC) apicReadRegister(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, uint16_t offReg, uint32_t *puValue)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(offReg <= XAPIC_OFF_MAX_VALID);
+
+    PXAPICPAGE   pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    uint32_t     uValue = 0;
+    VBOXSTRICTRC rc = VINF_SUCCESS;
+    switch (offReg)
+    {
+        case XAPIC_OFF_ID:
+        case XAPIC_OFF_VERSION:
+        case XAPIC_OFF_TPR:
+        case XAPIC_OFF_EOI:
+        case XAPIC_OFF_RRD:
+        case XAPIC_OFF_LDR:
+        case XAPIC_OFF_DFR:
+        case XAPIC_OFF_SVR:
+        case XAPIC_OFF_ISR0:    case XAPIC_OFF_ISR1:    case XAPIC_OFF_ISR2:    case XAPIC_OFF_ISR3:
+        case XAPIC_OFF_ISR4:    case XAPIC_OFF_ISR5:    case XAPIC_OFF_ISR6:    case XAPIC_OFF_ISR7:
+        case XAPIC_OFF_TMR0:    case XAPIC_OFF_TMR1:    case XAPIC_OFF_TMR2:    case XAPIC_OFF_TMR3:
+        case XAPIC_OFF_TMR4:    case XAPIC_OFF_TMR5:    case XAPIC_OFF_TMR6:    case XAPIC_OFF_TMR7:
+        case XAPIC_OFF_IRR0:    case XAPIC_OFF_IRR1:    case XAPIC_OFF_IRR2:    case XAPIC_OFF_IRR3:
+        case XAPIC_OFF_IRR4:    case XAPIC_OFF_IRR5:    case XAPIC_OFF_IRR6:    case XAPIC_OFF_IRR7:
+        case XAPIC_OFF_ESR:
+        case XAPIC_OFF_ICR_LO:
+        case XAPIC_OFF_ICR_HI:
+        case XAPIC_OFF_LVT_TIMER:
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+        case XAPIC_OFF_LVT_THERMAL:
+#endif
+        case XAPIC_OFF_LVT_PERF:
+        case XAPIC_OFF_LVT_LINT0:
+        case XAPIC_OFF_LVT_LINT1:
+        case XAPIC_OFF_LVT_ERROR:
+        case XAPIC_OFF_TIMER_ICR:
+        case XAPIC_OFF_TIMER_DCR:
+        {
+            Assert(   !XAPIC_IN_X2APIC_MODE(pVCpu)
+                   || (   offReg != XAPIC_OFF_DFR
+                       && offReg != XAPIC_OFF_ICR_HI
+                       && offReg != XAPIC_OFF_EOI));
+            uValue = apicReadRaw32(pXApicPage, offReg);
+            Log2(("APIC%u: apicReadRegister: offReg=%#x uValue=%#x\n", pVCpu->idCpu, offReg, uValue));
+            break;
+        }
+
+        case XAPIC_OFF_PPR:
+        {
+            uValue = apicGetPpr(pVCpu);
+            break;
+        }
+
+        case XAPIC_OFF_TIMER_CCR:
+        {
+            Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+            rc = apicGetTimerCcr(pDevIns, pVCpu, VINF_IOM_R3_MMIO_READ, &uValue);
+            break;
+        }
+
+        case XAPIC_OFF_APR:
+        {
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+            /* Unsupported on Pentium 4 and Xeon CPUs, invalid in x2APIC mode. */
+            Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+            break;
+        }
+
+        default:
+        {
+            Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+            rc = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS, "VCPU[%u]: offReg=%#RX16\n", pVCpu->idCpu, offReg);
+            apicSetError(pVCpu, XAPIC_ESR_ILLEGAL_REG_ADDRESS);
+            break;
+        }
+    }
+
+    *puValue = uValue;
+    return rc;
+}
+
+
+/**
+ * Writes an APIC register.
+ *
+ * @returns Strict VBox status code.
+ * @param   pDevIns         The device instance.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   offReg          The offset of the register being written.
+ * @param   uValue          The register value.
+ */
+DECLINLINE(VBOXSTRICTRC) apicWriteRegister(PPDMDEVINS pDevIns, PVMCPUCC pVCpu, uint16_t offReg, uint32_t uValue)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(offReg <= XAPIC_OFF_MAX_VALID);
+    Assert(!XAPIC_IN_X2APIC_MODE(pVCpu));
+
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    switch (offReg)
+    {
+        case XAPIC_OFF_TPR:
+        {
+            rcStrict = apicSetTprEx(pVCpu, uValue, false /* fForceX2ApicBehaviour */);
+            break;
+        }
+
+        case XAPIC_OFF_LVT_TIMER:
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+        case XAPIC_OFF_LVT_THERMAL:
+#endif
+        case XAPIC_OFF_LVT_PERF:
+        case XAPIC_OFF_LVT_LINT0:
+        case XAPIC_OFF_LVT_LINT1:
+        case XAPIC_OFF_LVT_ERROR:
+        {
+            rcStrict = apicSetLvtEntry(pVCpu, offReg, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_TIMER_ICR:
+        {
+            rcStrict = apicSetTimerIcr(pDevIns, pVCpu, VINF_IOM_R3_MMIO_WRITE, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_EOI:
+        {
+            rcStrict = apicSetEoi(pVCpu, uValue, false /* fForceX2ApicBehaviour */);
+            break;
+        }
+
+        case XAPIC_OFF_LDR:
+        {
+            rcStrict = apicSetLdr(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_DFR:
+        {
+            rcStrict = apicSetDfr(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_SVR:
+        {
+            rcStrict = apicSetSvr(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_ICR_LO:
+        {
+            rcStrict = apicSetIcrLo(pVCpu, uValue, VINF_IOM_R3_MMIO_WRITE, true /* fUpdateStat */);
+            break;
+        }
+
+        case XAPIC_OFF_ICR_HI:
+        {
+            rcStrict = apicSetIcrHi(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_TIMER_DCR:
+        {
+            rcStrict = apicSetTimerDcr(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_ESR:
+        {
+            rcStrict = apicSetEsr(pVCpu, uValue);
+            break;
+        }
+
+        case XAPIC_OFF_APR:
+        case XAPIC_OFF_RRD:
+        {
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+            /* Unsupported on Pentium 4 and Xeon CPUs but writes do -not- set an illegal register access error. */
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+            break;
+        }
+
+        /* Read-only, write ignored: */
+        case XAPIC_OFF_VERSION:
+        case XAPIC_OFF_ID:
+            break;
+
+        /* Unavailable/reserved in xAPIC mode: */
+        case X2APIC_OFF_SELF_IPI:
+        /* Read-only registers: */
+        case XAPIC_OFF_PPR:
+        case XAPIC_OFF_ISR0:    case XAPIC_OFF_ISR1:    case XAPIC_OFF_ISR2:    case XAPIC_OFF_ISR3:
+        case XAPIC_OFF_ISR4:    case XAPIC_OFF_ISR5:    case XAPIC_OFF_ISR6:    case XAPIC_OFF_ISR7:
+        case XAPIC_OFF_TMR0:    case XAPIC_OFF_TMR1:    case XAPIC_OFF_TMR2:    case XAPIC_OFF_TMR3:
+        case XAPIC_OFF_TMR4:    case XAPIC_OFF_TMR5:    case XAPIC_OFF_TMR6:    case XAPIC_OFF_TMR7:
+        case XAPIC_OFF_IRR0:    case XAPIC_OFF_IRR1:    case XAPIC_OFF_IRR2:    case XAPIC_OFF_IRR3:
+        case XAPIC_OFF_IRR4:    case XAPIC_OFF_IRR5:    case XAPIC_OFF_IRR6:    case XAPIC_OFF_IRR7:
+        case XAPIC_OFF_TIMER_CCR:
+        default:
+        {
+            rcStrict = PDMDevHlpDBGFStop(pDevIns, RT_SRC_POS, "APIC%u: offReg=%#RX16\n", pVCpu->idCpu, offReg);
+            apicSetError(pVCpu, XAPIC_ESR_ILLEGAL_REG_ADDRESS);
+            break;
+        }
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Reads an APIC MSR.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u32Reg          The MSR being read.
+ * @param   pu64Value       Where to store the read value.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICReadMsr(PVMCPUCC pVCpu, uint32_t u32Reg, uint64_t *pu64Value)
+{
+    /*
+     * Validate.
+     */
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(u32Reg >= MSR_IA32_X2APIC_ID && u32Reg <= MSR_IA32_X2APIC_SELF_IPI);
+    Assert(pu64Value);
+
+    /*
+     * Is the APIC enabled?
+     */
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    if (APICIsEnabled(pVCpu))
+    { /* likely */ }
+    else
+        return apicMsrAccessError(pVCpu, u32Reg, pApic->enmMaxMode == PDMAPICMODE_NONE ?
+                                                 APICMSRACCESS_READ_DISALLOWED_CONFIG : APICMSRACCESS_READ_RSVD_OR_UNKNOWN);
+
+#ifndef IN_RING3
+    if (pApic->CTXALLMID(f,Enabled))
+    { /* likely */}
+    else
+        return VINF_CPUM_R3_MSR_READ;
+#endif
+
+    STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMsrRead));
+
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    if (RT_LIKELY(   XAPIC_IN_X2APIC_MODE(pVCpu)
+                  || pApic->fHyperVCompatMode))
+    {
+        switch (u32Reg)
+        {
+            /* Special handling for x2APIC: */
+            case MSR_IA32_X2APIC_ICR:
+            {
+                *pu64Value = apicGetIcrNoCheck(pVCpu);
+                break;
+            }
+
+            /* Special handling, compatible with xAPIC: */
+            case MSR_IA32_X2APIC_TIMER_CCR:
+            {
+                uint32_t uValue;
+                rcStrict = apicGetTimerCcr(VMCPU_TO_DEVINS(pVCpu), pVCpu, VINF_CPUM_R3_MSR_READ, &uValue);
+                *pu64Value = uValue;
+                break;
+            }
+
+            /* Special handling, compatible with xAPIC: */
+            case MSR_IA32_X2APIC_PPR:
+            {
+                *pu64Value = apicGetPpr(pVCpu);
+                break;
+            }
+
+            /* Raw read, compatible with xAPIC: */
+            case MSR_IA32_X2APIC_ID:
+            {
+                STAM_COUNTER_INC(&pVCpu->apic.s.StatIdMsrRead);
+                /* Horrible macOS hack (sample rdmsr addres: 0008:ffffff801686f21a). */
+                if (   !pApic->fMacOSWorkaround
+                    || pVCpu->cpum.GstCtx.cs.Sel != 8
+                    || pVCpu->cpum.GstCtx.rip < UINT64_C(0xffffff8000000000))
+                { /* likely */ }
+                else
+                {
+                    PCX2APICPAGE pX2ApicPage = VMCPU_TO_CX2APICPAGE(pVCpu);
+                    uint32_t const idApic = pX2ApicPage->id.u32ApicId;
+                    *pu64Value = (idApic << 24) | idApic;
+                    Log(("APIC: Applying macOS hack to MSR_IA32_X2APIC_ID: %#RX64\n", *pu64Value));
+                    break;
+                }
+                RT_FALL_THRU();
+            }
+            case MSR_IA32_X2APIC_VERSION:
+            case MSR_IA32_X2APIC_TPR:
+            case MSR_IA32_X2APIC_LDR:
+            case MSR_IA32_X2APIC_SVR:
+            case MSR_IA32_X2APIC_ISR0:  case MSR_IA32_X2APIC_ISR1:  case MSR_IA32_X2APIC_ISR2:  case MSR_IA32_X2APIC_ISR3:
+            case MSR_IA32_X2APIC_ISR4:  case MSR_IA32_X2APIC_ISR5:  case MSR_IA32_X2APIC_ISR6:  case MSR_IA32_X2APIC_ISR7:
+            case MSR_IA32_X2APIC_TMR0:  case MSR_IA32_X2APIC_TMR1:  case MSR_IA32_X2APIC_TMR2:  case MSR_IA32_X2APIC_TMR3:
+            case MSR_IA32_X2APIC_TMR4:  case MSR_IA32_X2APIC_TMR5:  case MSR_IA32_X2APIC_TMR6:  case MSR_IA32_X2APIC_TMR7:
+            case MSR_IA32_X2APIC_IRR0:  case MSR_IA32_X2APIC_IRR1:  case MSR_IA32_X2APIC_IRR2:  case MSR_IA32_X2APIC_IRR3:
+            case MSR_IA32_X2APIC_IRR4:  case MSR_IA32_X2APIC_IRR5:  case MSR_IA32_X2APIC_IRR6:  case MSR_IA32_X2APIC_IRR7:
+            case MSR_IA32_X2APIC_ESR:
+            case MSR_IA32_X2APIC_LVT_TIMER:
+            case MSR_IA32_X2APIC_LVT_THERMAL:
+            case MSR_IA32_X2APIC_LVT_PERF:
+            case MSR_IA32_X2APIC_LVT_LINT0:
+            case MSR_IA32_X2APIC_LVT_LINT1:
+            case MSR_IA32_X2APIC_LVT_ERROR:
+            case MSR_IA32_X2APIC_TIMER_ICR:
+            case MSR_IA32_X2APIC_TIMER_DCR:
+            {
+                PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+                uint16_t const offReg = X2APIC_GET_XAPIC_OFF(u32Reg);
+                *pu64Value = apicReadRaw32(pXApicPage, offReg);
+                break;
+            }
+
+            /* Write-only MSRs: */
+            case MSR_IA32_X2APIC_SELF_IPI:
+            case MSR_IA32_X2APIC_EOI:
+            {
+                rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_READ_WRITE_ONLY);
+                break;
+            }
+
+            /*
+             * Windows guest using Hyper-V x2APIC MSR compatibility mode tries to read the "high"
+             * LDR bits, which is quite absurd (as it's a 32-bit register) using this invalid MSR
+             * index (0x80E), see @bugref{8382#c175}.
+             */
+            case MSR_IA32_X2APIC_LDR + 1:
+            {
+                if (pApic->fHyperVCompatMode)
+                    *pu64Value = 0;
+                else
+                    rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_READ_RSVD_OR_UNKNOWN);
+                break;
+            }
+
+            /* Reserved MSRs: */
+            case MSR_IA32_X2APIC_LVT_CMCI:
+            default:
+            {
+                rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_READ_RSVD_OR_UNKNOWN);
+                break;
+            }
+        }
+    }
+    else
+        rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_INVALID_READ_MODE);
+
+    return rcStrict;
+}
+
+
+/**
+ * Writes an APIC MSR.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u32Reg          The MSR being written.
+ * @param   u64Value        The value to write.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICWriteMsr(PVMCPUCC pVCpu, uint32_t u32Reg, uint64_t u64Value)
+{
+    /*
+     * Validate.
+     */
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(u32Reg >= MSR_IA32_X2APIC_ID && u32Reg <= MSR_IA32_X2APIC_SELF_IPI);
+
+    /*
+     * Is the APIC enabled?
+     */
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    if (APICIsEnabled(pVCpu))
+    { /* likely */ }
+    else
+        return apicMsrAccessError(pVCpu, u32Reg, pApic->enmMaxMode == PDMAPICMODE_NONE ?
+                                  APICMSRACCESS_WRITE_DISALLOWED_CONFIG : APICMSRACCESS_WRITE_RSVD_OR_UNKNOWN);
+
+#ifndef IN_RING3
+    if (pApic->CTXALLMID(f,Enabled))
+    { /* likely */ }
+    else
+        return VINF_CPUM_R3_MSR_WRITE;
+#endif
+
+    STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMsrWrite));
+
+    /*
+     * In x2APIC mode, we need to raise #GP(0) for writes to reserved bits, unlike MMIO
+     * accesses where they are ignored. Hence, we need to validate each register before
+     * invoking the generic/xAPIC write functions.
+     *
+     * Bits 63:32 of all registers except the ICR are reserved, we'll handle this common
+     * case first and handle validating the remaining bits on a per-register basis.
+     * See Intel spec. 10.12.1.2 "x2APIC Register Address Space".
+     */
+    if (   u32Reg != MSR_IA32_X2APIC_ICR
+        && RT_HI_U32(u64Value))
+        return apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_RSVD_BITS);
+
+    uint32_t     u32Value = RT_LO_U32(u64Value);
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    if (RT_LIKELY(   XAPIC_IN_X2APIC_MODE(pVCpu)
+                  || pApic->fHyperVCompatMode))
+    {
+        switch (u32Reg)
+        {
+            case MSR_IA32_X2APIC_TPR:
+            {
+                rcStrict = apicSetTprEx(pVCpu, u32Value, false /* fForceX2ApicBehaviour */);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_ICR:
+            {
+                rcStrict = apicSetIcr(pVCpu, u64Value, VINF_CPUM_R3_MSR_WRITE);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_SVR:
+            {
+                rcStrict = apicSetSvr(pVCpu, u32Value);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_ESR:
+            {
+                rcStrict = apicSetEsr(pVCpu, u32Value);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_TIMER_DCR:
+            {
+                rcStrict = apicSetTimerDcr(pVCpu, u32Value);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_LVT_TIMER:
+            case MSR_IA32_X2APIC_LVT_THERMAL:
+            case MSR_IA32_X2APIC_LVT_PERF:
+            case MSR_IA32_X2APIC_LVT_LINT0:
+            case MSR_IA32_X2APIC_LVT_LINT1:
+            case MSR_IA32_X2APIC_LVT_ERROR:
+            {
+                rcStrict = apicSetLvtEntry(pVCpu, X2APIC_GET_XAPIC_OFF(u32Reg), u32Value);
+                break;
+            }
+
+            case MSR_IA32_X2APIC_TIMER_ICR:
+            {
+                rcStrict = apicSetTimerIcr(VMCPU_TO_DEVINS(pVCpu), pVCpu, VINF_CPUM_R3_MSR_WRITE, u32Value);
+                break;
+            }
+
+            /* Write-only MSRs: */
+            case MSR_IA32_X2APIC_SELF_IPI:
+            {
+                uint8_t const uVector = XAPIC_SELF_IPI_GET_VECTOR(u32Value);
+                apicPostInterrupt(pVCpu, uVector, XAPICTRIGGERMODE_EDGE, 0 /* uSrcTag */);
+                rcStrict = VINF_SUCCESS;
+                break;
+            }
+
+            case MSR_IA32_X2APIC_EOI:
+            {
+                rcStrict = apicSetEoi(pVCpu, u32Value, false /* fForceX2ApicBehaviour */);
+                break;
+            }
+
+            /*
+             * Windows guest using Hyper-V x2APIC MSR compatibility mode tries to write the "high"
+             * LDR bits, which is quite absurd (as it's a 32-bit register) using this invalid MSR
+             * index (0x80E). The write value was 0xffffffff on a Windows 8.1 64-bit guest. We can
+             * safely ignore this nonsense, See @bugref{8382#c7}.
+             */
+            case MSR_IA32_X2APIC_LDR + 1:
+            {
+                if (pApic->fHyperVCompatMode)
+                    rcStrict = VINF_SUCCESS;
+                else
+                    rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_RSVD_OR_UNKNOWN);
+                break;
+            }
+
+            /* Special-treament (read-only normally, but not with Hyper-V) */
+            case MSR_IA32_X2APIC_LDR:
+            {
+                if (pApic->fHyperVCompatMode)
+                {
+                    rcStrict = apicSetLdr(pVCpu, u32Value);
+                    break;
+                }
+            }
+            RT_FALL_THRU();
+            /* Read-only MSRs: */
+            case MSR_IA32_X2APIC_ID:
+            case MSR_IA32_X2APIC_VERSION:
+            case MSR_IA32_X2APIC_PPR:
+            case MSR_IA32_X2APIC_ISR0:  case MSR_IA32_X2APIC_ISR1:  case MSR_IA32_X2APIC_ISR2:  case MSR_IA32_X2APIC_ISR3:
+            case MSR_IA32_X2APIC_ISR4:  case MSR_IA32_X2APIC_ISR5:  case MSR_IA32_X2APIC_ISR6:  case MSR_IA32_X2APIC_ISR7:
+            case MSR_IA32_X2APIC_TMR0:  case MSR_IA32_X2APIC_TMR1:  case MSR_IA32_X2APIC_TMR2:  case MSR_IA32_X2APIC_TMR3:
+            case MSR_IA32_X2APIC_TMR4:  case MSR_IA32_X2APIC_TMR5:  case MSR_IA32_X2APIC_TMR6:  case MSR_IA32_X2APIC_TMR7:
+            case MSR_IA32_X2APIC_IRR0:  case MSR_IA32_X2APIC_IRR1:  case MSR_IA32_X2APIC_IRR2:  case MSR_IA32_X2APIC_IRR3:
+            case MSR_IA32_X2APIC_IRR4:  case MSR_IA32_X2APIC_IRR5:  case MSR_IA32_X2APIC_IRR6:  case MSR_IA32_X2APIC_IRR7:
+            case MSR_IA32_X2APIC_TIMER_CCR:
+            {
+                rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_READ_ONLY);
+                break;
+            }
+
+            /* Reserved MSRs: */
+            case MSR_IA32_X2APIC_LVT_CMCI:
+            default:
+            {
+                rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_WRITE_RSVD_OR_UNKNOWN);
+                break;
+            }
+        }
+    }
+    else
+        rcStrict = apicMsrAccessError(pVCpu, u32Reg, APICMSRACCESS_INVALID_WRITE_MODE);
+
+    return rcStrict;
+}
+
+
+/**
+ * Resets the APIC base MSR.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+static void apicResetBaseMsr(PVMCPUCC pVCpu)
+{
+    /*
+     * Initialize the APIC base MSR. The APIC enable-bit is set upon power-up or reset[1].
+     *
+     * A Reset (in xAPIC and x2APIC mode) brings up the local APIC in xAPIC mode.
+     * An INIT IPI does -not- cause a transition between xAPIC and x2APIC mode[2].
+     *
+     * [1] See AMD spec. 14.1.3 "Processor Initialization State"
+     * [2] See Intel spec. 10.12.5.1 "x2APIC States".
+     */
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+
+    /* Construct. */
+    PAPICCPU pApicCpu     = VMCPU_TO_APICCPU(pVCpu);
+    PAPIC    pApic        = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    uint64_t uApicBaseMsr = MSR_IA32_APICBASE_ADDR;
+    if (pVCpu->idCpu == 0)
+        uApicBaseMsr |= MSR_IA32_APICBASE_BSP;
+
+    /* If the VM was configured with no APIC, don't enable xAPIC mode, obviously. */
+    if (pApic->enmMaxMode != PDMAPICMODE_NONE)
+    {
+        uApicBaseMsr |= MSR_IA32_APICBASE_EN;
+
+        /*
+         * While coming out of a reset the APIC is enabled and in xAPIC mode. If software had previously
+         * disabled the APIC (which results in the CPUID bit being cleared as well) we re-enable it here.
+         * See Intel spec. 10.12.5.1 "x2APIC States".
+         */
+        if (CPUMSetGuestCpuIdPerCpuApicFeature(pVCpu, true /*fVisible*/) == false)
+            LogRel(("APIC%u: Resetting mode to xAPIC\n", pVCpu->idCpu));
+    }
+
+    /* Commit. */
+    ASMAtomicWriteU64(&pApicCpu->uApicBaseMsr, uApicBaseMsr);
+}
+
+
+/**
+ * Initializes per-VCPU APIC to the state following an INIT reset
+ * ("Wait-for-SIPI" state).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+void apicInitIpi(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+
+    /*
+     * See Intel spec. 10.4.7.3 "Local APIC State After an INIT Reset (Wait-for-SIPI State)"
+     * and AMD spec 16.3.2 "APIC Registers".
+     *
+     * The reason we don't simply zero out the entire APIC page and only set the non-zero members
+     * is because there are some registers that are not touched by the INIT IPI (e.g. version)
+     * operation and this function is only a subset of the reset operation.
+     */
+    RT_ZERO(pXApicPage->irr);
+    RT_ZERO(pXApicPage->irr);
+    RT_ZERO(pXApicPage->isr);
+    RT_ZERO(pXApicPage->tmr);
+    RT_ZERO(pXApicPage->icr_hi);
+    RT_ZERO(pXApicPage->icr_lo);
+    RT_ZERO(pXApicPage->ldr);
+    RT_ZERO(pXApicPage->tpr);
+    RT_ZERO(pXApicPage->ppr);
+    RT_ZERO(pXApicPage->timer_icr);
+    RT_ZERO(pXApicPage->timer_ccr);
+    RT_ZERO(pXApicPage->timer_dcr);
+
+    pXApicPage->dfr.u.u4Model        = XAPICDESTFORMAT_FLAT;
+    pXApicPage->dfr.u.u28ReservedMb1 = UINT32_C(0xfffffff);
+
+    /** @todo CMCI. */
+
+    RT_ZERO(pXApicPage->lvt_timer);
+    pXApicPage->lvt_timer.u.u1Mask = 1;
+
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    RT_ZERO(pXApicPage->lvt_thermal);
+    pXApicPage->lvt_thermal.u.u1Mask = 1;
+#endif
+
+    RT_ZERO(pXApicPage->lvt_perf);
+    pXApicPage->lvt_perf.u.u1Mask = 1;
+
+    RT_ZERO(pXApicPage->lvt_lint0);
+    pXApicPage->lvt_lint0.u.u1Mask = 1;
+
+    RT_ZERO(pXApicPage->lvt_lint1);
+    pXApicPage->lvt_lint1.u.u1Mask = 1;
+
+    RT_ZERO(pXApicPage->lvt_error);
+    pXApicPage->lvt_error.u.u1Mask = 1;
+
+    RT_ZERO(pXApicPage->svr);
+    pXApicPage->svr.u.u8SpuriousVector = 0xff;
+
+    /* The self-IPI register is reset to 0. See Intel spec. 10.12.5.1 "x2APIC States" */
+    PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu);
+    RT_ZERO(pX2ApicPage->self_ipi);
+
+    /* Clear the pending-interrupt bitmaps. */
+    PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    RT_BZERO(&pApicCpu->ApicPibLevel, sizeof(APICPIB));
+    RT_BZERO(pApicCpu->CTX_SUFF(pvApicPib), sizeof(APICPIB));
+
+    /* Clear the interrupt line states for LINT0 and LINT1 pins. */
+    pApicCpu->fActiveLint0 = false;
+    pApicCpu->fActiveLint1 = false;
+}
+
+
+/**
+ * Initializes per-VCPU APIC to the state following a power-up or hardware
+ * reset.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   fResetApicBaseMsr   Whether to reset the APIC base MSR.
+ */
+void apicResetCpu(PVMCPUCC pVCpu, bool fResetApicBaseMsr)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+
+    LogFlow(("APIC%u: apicR3ResetCpu: fResetApicBaseMsr=%RTbool\n", pVCpu->idCpu, fResetApicBaseMsr));
+
+#ifdef VBOX_STRICT
+    /* Verify that the initial APIC ID reported via CPUID matches our VMCPU ID assumption. */
+    uint32_t uEax, uEbx, uEcx, uEdx;
+    uEax = uEbx = uEcx = uEdx = UINT32_MAX;
+    CPUMGetGuestCpuId(pVCpu, 1, 0, -1 /*f64BitMode*/, &uEax, &uEbx, &uEcx, &uEdx);
+    Assert(((uEbx >> 24) & 0xff) == pVCpu->idCpu);
+#endif
+
+    /*
+     * The state following a power-up or reset is a superset of the INIT state.
+     * See Intel spec. 10.4.7.3 "Local APIC State After an INIT Reset ('Wait-for-SIPI' State)"
+     */
+    apicInitIpi(pVCpu);
+
+    /*
+     * The APIC version register is read-only, so just initialize it here.
+     * It is not clear from the specs, where exactly it is initialized.
+     * The version determines the number of LVT entries and size of the APIC ID (8 bits for P4).
+     */
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+#if XAPIC_HARDWARE_VERSION == XAPIC_HARDWARE_VERSION_P4
+    pXApicPage->version.u.u8MaxLvtEntry = XAPIC_MAX_LVT_ENTRIES_P4 - 1;
+    pXApicPage->version.u.u8Version     = XAPIC_HARDWARE_VERSION_P4;
+    AssertCompile(sizeof(pXApicPage->id.u8ApicId) >= XAPIC_APIC_ID_BIT_COUNT_P4 / 8);
+#else
+# error "Implement Pentium and P6 family APIC architectures"
+#endif
+
+    /** @todo It isn't clear in the spec. where exactly the default base address
+     *        is (re)initialized, atm we do it here in Reset. */
+    if (fResetApicBaseMsr)
+        apicResetBaseMsr(pVCpu);
+
+    /*
+     * Initialize the APIC ID register to xAPIC format.
+     */
+    ASMMemZero32(&pXApicPage->id, sizeof(pXApicPage->id));
+    pXApicPage->id.u8ApicId = pVCpu->idCpu;
+}
+
+
+/**
+ * Sets the APIC base MSR.
+ *
+ * @returns VBox status code - no informational ones, esp. not
+ *          VINF_CPUM_R3_MSR_WRITE.  Only the following two:
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u64BaseMsr  The value to set.
+ */
+VMM_INT_DECL(int) APICSetBaseMsr(PVMCPUCC pVCpu, uint64_t u64BaseMsr)
+{
+    Assert(pVCpu);
+
+    PAPICCPU pApicCpu   = VMCPU_TO_APICCPU(pVCpu);
+    PAPIC    pApic      = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    APICMODE enmOldMode = apicGetMode(pApicCpu->uApicBaseMsr);
+    APICMODE enmNewMode = apicGetMode(u64BaseMsr);
+    uint64_t uBaseMsr   = pApicCpu->uApicBaseMsr;
+
+    Log2(("APIC%u: ApicSetBaseMsr: u64BaseMsr=%#RX64 enmNewMode=%s enmOldMode=%s\n", pVCpu->idCpu, u64BaseMsr,
+          apicGetModeName(enmNewMode), apicGetModeName(enmOldMode)));
+
+    /*
+     * We do not support re-mapping the APIC base address because:
+     *    - We'll have to manage all the mappings ourselves in the APIC (reference counting based unmapping etc.)
+     *      i.e. we can only unmap the MMIO region if no other APIC is mapped on that location.
+     *    - It's unclear how/if IOM can fallback to handling regions as regular memory (if the MMIO
+     *      region remains mapped but doesn't belong to the called VCPU's APIC).
+     */
+    /** @todo Handle per-VCPU APIC base relocation. */
+    if (MSR_IA32_APICBASE_GET_ADDR(uBaseMsr) != MSR_IA32_APICBASE_ADDR)
+    {
+        if (pVCpu->apic.s.cLogMaxSetApicBaseAddr++ < 5)
+            LogRel(("APIC%u: Attempt to relocate base to %#RGp, unsupported -> #GP(0)\n", pVCpu->idCpu,
+                    MSR_IA32_APICBASE_GET_ADDR(uBaseMsr)));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    /* Don't allow enabling xAPIC/x2APIC if the VM is configured with the APIC disabled. */
+    if (pApic->enmMaxMode == PDMAPICMODE_NONE)
+    {
+        LogRel(("APIC%u: Disallowing APIC base MSR write as the VM is configured with APIC disabled!\n", pVCpu->idCpu));
+        return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_DISALLOWED_CONFIG);
+    }
+
+    /*
+     * Act on state transition.
+     */
+    if (enmNewMode != enmOldMode)
+    {
+        switch (enmNewMode)
+        {
+            case APICMODE_DISABLED:
+            {
+                /*
+                 * The APIC state needs to be reset (especially the APIC ID as x2APIC APIC ID bit layout
+                 * is different). We can start with a clean slate identical to the state after a power-up/reset.
+                 *
+                 * See Intel spec. 10.4.3 "Enabling or Disabling the Local APIC".
+                 *
+                 * We'll also manually manage the APIC base MSR here. We want a single-point of commit
+                 * at the end of this function rather than updating it in apicR3ResetCpu. This means we also
+                 * need to update the CPUID leaf ourselves.
+                 */
+                apicResetCpu(pVCpu, false /* fResetApicBaseMsr */);
+                uBaseMsr &= ~(MSR_IA32_APICBASE_EN | MSR_IA32_APICBASE_EXTD);
+                CPUMSetGuestCpuIdPerCpuApicFeature(pVCpu, false /*fVisible*/);
+                LogRel(("APIC%u: Switched mode to disabled\n", pVCpu->idCpu));
+                break;
+            }
+
+            case APICMODE_XAPIC:
+            {
+                if (enmOldMode != APICMODE_DISABLED)
+                {
+                    LogRel(("APIC%u: Can only transition to xAPIC state from disabled state\n", pVCpu->idCpu));
+                    return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID);
+                }
+
+                uBaseMsr |= MSR_IA32_APICBASE_EN;
+                CPUMSetGuestCpuIdPerCpuApicFeature(pVCpu, true /*fVisible*/);
+                LogRel(("APIC%u: Switched mode to xAPIC\n", pVCpu->idCpu));
+                break;
+            }
+
+            case APICMODE_X2APIC:
+            {
+                if (pApic->enmMaxMode != PDMAPICMODE_X2APIC)
+                {
+                    LogRel(("APIC%u: Disallowing transition to x2APIC mode as the VM is configured with the x2APIC disabled!\n",
+                            pVCpu->idCpu));
+                    return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID);
+                }
+
+                if (enmOldMode != APICMODE_XAPIC)
+                {
+                    LogRel(("APIC%u: Can only transition to x2APIC state from xAPIC state\n", pVCpu->idCpu));
+                    return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID);
+                }
+
+                uBaseMsr |= MSR_IA32_APICBASE_EN | MSR_IA32_APICBASE_EXTD;
+
+                /*
+                 * The APIC ID needs updating when entering x2APIC mode.
+                 * Software written APIC ID in xAPIC mode isn't preserved.
+                 * The APIC ID becomes read-only to software in x2APIC mode.
+                 *
+                 * See Intel spec. 10.12.5.1 "x2APIC States".
+                 */
+                PX2APICPAGE pX2ApicPage = VMCPU_TO_X2APICPAGE(pVCpu);
+                ASMMemZero32(&pX2ApicPage->id, sizeof(pX2ApicPage->id));
+                pX2ApicPage->id.u32ApicId = pVCpu->idCpu;
+
+                /*
+                 * LDR initialization occurs when entering x2APIC mode.
+                 * See Intel spec. 10.12.10.2 "Deriving Logical x2APIC ID from the Local x2APIC ID".
+                 */
+                pX2ApicPage->ldr.u32LogicalApicId = ((pX2ApicPage->id.u32ApicId & UINT32_C(0xffff0)) << 16)
+                                                  | (UINT32_C(1) << pX2ApicPage->id.u32ApicId & UINT32_C(0xf));
+
+                LogRel(("APIC%u: Switched mode to x2APIC\n", pVCpu->idCpu));
+                break;
+            }
+
+            case APICMODE_INVALID:
+            default:
+            {
+                Log(("APIC%u: Invalid state transition attempted\n", pVCpu->idCpu));
+                return apicMsrAccessError(pVCpu, MSR_IA32_APICBASE, APICMSRACCESS_WRITE_INVALID);
+            }
+        }
+    }
+
+    ASMAtomicWriteU64(&pApicCpu->uApicBaseMsr, uBaseMsr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the APIC base MSR (no checks are performed wrt APIC hardware or its
+ * state).
+ *
+ * @returns The base MSR value.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(uint64_t) APICGetBaseMsrNoCheck(PCVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+    PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    return pApicCpu->uApicBaseMsr;
+}
+
+
+/**
+ * Gets the APIC base MSR.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pu64Value   Where to store the MSR value.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICGetBaseMsr(PVMCPUCC pVCpu, uint64_t *pu64Value)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+
+    PCAPIC pApic = VM_TO_APIC(pVCpu->CTX_SUFF(pVM));
+    if (pApic->enmMaxMode != PDMAPICMODE_NONE)
+    {
+        *pu64Value = APICGetBaseMsrNoCheck(pVCpu);
+        return VINF_SUCCESS;
+    }
+
+    if (pVCpu->apic.s.cLogMaxGetApicBaseAddr++ < 5)
+        LogRel(("APIC%u: Reading APIC base MSR (%#x) when there is no APIC -> #GP(0)\n", pVCpu->idCpu, MSR_IA32_APICBASE));
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * Sets the TPR (Task Priority Register).
+ *
+ * @retval  VINF_SUCCESS
+ * @retval  VERR_CPUM_RAISE_GP_0
+ * @retval  VERR_PDM_NO_APIC_INSTANCE
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u8Tpr       The TPR value to set.
+ */
+VMMDECL(int) APICSetTpr(PVMCPUCC pVCpu, uint8_t u8Tpr)
+{
+    if (APICIsEnabled(pVCpu))
+        return apicSetTprEx(pVCpu, u8Tpr, false /* fForceX2ApicBehaviour */);
+    return VERR_PDM_NO_APIC_INSTANCE;
+}
+
+
+/**
+ * Gets the highest priority pending interrupt.
+ *
+ * @returns true if any interrupt is pending, false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pu8PendingIntr      Where to store the interrupt vector if the
+ *                              interrupt is pending (optional, can be NULL).
+ */
+static bool apicGetHighestPendingInterrupt(PCVMCPUCC pVCpu, uint8_t *pu8PendingIntr)
+{
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    int const irrv = apicGetHighestSetBitInReg(&pXApicPage->irr, -1);
+    if (irrv >= 0)
+    {
+        Assert(irrv <= (int)UINT8_MAX);
+        if (pu8PendingIntr)
+            *pu8PendingIntr = (uint8_t)irrv;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the APIC TPR (Task Priority Register).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pu8Tpr          Where to store the TPR.
+ * @param   pfPending       Where to store whether there is a pending interrupt
+ *                          (optional, can be NULL).
+ * @param   pu8PendingIntr  Where to store the highest-priority pending
+ *                          interrupt (optional, can be NULL).
+ */
+VMMDECL(int) APICGetTpr(PCVMCPUCC pVCpu, uint8_t *pu8Tpr, bool *pfPending, uint8_t *pu8PendingIntr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    if (APICIsEnabled(pVCpu))
+    {
+        PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+        if (pfPending)
+        {
+            /*
+             * Just return whatever the highest pending interrupt is in the IRR.
+             * The caller is responsible for figuring out if it's masked by the TPR etc.
+             */
+            *pfPending = apicGetHighestPendingInterrupt(pVCpu, pu8PendingIntr);
+        }
+
+        *pu8Tpr = pXApicPage->tpr.u8Tpr;
+        return VINF_SUCCESS;
+    }
+
+    *pu8Tpr = 0;
+    return VERR_PDM_NO_APIC_INSTANCE;
+}
+
+
+/**
+ * Gets the APIC timer frequency.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pu64Value       Where to store the timer frequency.
+ */
+VMM_INT_DECL(int) APICGetTimerFreq(PVMCC pVM, uint64_t *pu64Value)
+{
+    /*
+     * Validate.
+     */
+    Assert(pVM);
+    AssertPtrReturn(pu64Value, VERR_INVALID_PARAMETER);
+
+    PVMCPUCC pVCpu = pVM->CTX_SUFF(apCpus)[0];
+    if (APICIsEnabled(pVCpu))
+    {
+        PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+        *pu64Value = PDMDevHlpTimerGetFreq(VMCPU_TO_DEVINS(pVCpu), pApicCpu->hTimer);
+        return VINF_SUCCESS;
+    }
+    return VERR_PDM_NO_APIC_INSTANCE;
+}
+
+
+/**
+ * Delivers an interrupt message via the system bus.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   uDest           The destination mask.
+ * @param   uDestMode       The destination mode.
+ * @param   uDeliveryMode   The delivery mode.
+ * @param   uVector         The interrupt vector.
+ * @param   uPolarity       The interrupt line polarity.
+ * @param   uTriggerMode    The trigger mode.
+ * @param   uSrcTag         The interrupt source tag (debugging).
+ */
+VMM_INT_DECL(int) APICBusDeliver(PVMCC pVM, uint8_t uDest, uint8_t uDestMode, uint8_t uDeliveryMode, uint8_t uVector,
+                                 uint8_t uPolarity, uint8_t uTriggerMode, uint32_t uSrcTag)
+{
+    NOREF(uPolarity);
+
+    /*
+     * If the APIC isn't enabled, do nothing and pretend success.
+     */
+    if (APICIsEnabled(pVM->CTX_SUFF(apCpus)[0]))
+    { /* likely */ }
+    else
+        return VINF_SUCCESS;
+
+    /*
+     * The destination field (mask) in the IO APIC redirectable table entry is 8-bits.
+     * Hence, the broadcast mask is 0xff.
+     * See IO APIC spec. 3.2.4. "IOREDTBL[23:0] - I/O Redirectable Table Registers".
+     */
+    XAPICTRIGGERMODE  enmTriggerMode  = (XAPICTRIGGERMODE)uTriggerMode;
+    XAPICDELIVERYMODE enmDeliveryMode = (XAPICDELIVERYMODE)uDeliveryMode;
+    XAPICDESTMODE     enmDestMode     = (XAPICDESTMODE)uDestMode;
+    uint32_t          fDestMask       = uDest;
+    uint32_t          fBroadcastMask  = UINT32_C(0xff);
+
+    Log2(("APIC: apicBusDeliver: fDestMask=%#x enmDestMode=%s enmTriggerMode=%s enmDeliveryMode=%s uVector=%#x uSrcTag=%#x\n",
+          fDestMask, apicGetDestModeName(enmDestMode), apicGetTriggerModeName(enmTriggerMode),
+          apicGetDeliveryModeName(enmDeliveryMode), uVector, uSrcTag));
+
+    bool     fIntrAccepted;
+    VMCPUSET DestCpuSet;
+    apicGetDestCpuSet(pVM, fDestMask, fBroadcastMask, enmDestMode, enmDeliveryMode, &DestCpuSet);
+    VBOXSTRICTRC rcStrict = apicSendIntr(pVM, NULL /* pVCpu */, uVector, enmTriggerMode, enmDeliveryMode, &DestCpuSet,
+                                         &fIntrAccepted, uSrcTag, VINF_SUCCESS /* rcRZ */);
+    if (fIntrAccepted)
+        return VBOXSTRICTRC_VAL(rcStrict);
+    return VERR_APIC_INTR_DISCARDED;
+}
+
+
+/**
+ * Assert/de-assert the local APIC's LINT0/LINT1 interrupt pins.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u8Pin       The interrupt pin (0 for LINT0 or 1 for LINT1).
+ * @param   u8Level     The level (0 for low or 1 for high).
+ * @param   rcRZ        The return code if the operation cannot be performed in
+ *                      the current context.
+ *
+ * @note    All callers totally ignores the status code!
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICLocalInterrupt(PVMCPUCC pVCpu, uint8_t u8Pin, uint8_t u8Level, int rcRZ)
+{
+    AssertReturn(u8Pin <= 1, VERR_INVALID_PARAMETER);
+    AssertReturn(u8Level <= 1, VERR_INVALID_PARAMETER);
+
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+
+    /* If the APIC is enabled, the interrupt is subject to LVT programming. */
+    if (APICIsEnabled(pVCpu))
+    {
+        PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+
+        /* Pick the LVT entry corresponding to the interrupt pin. */
+        static const uint16_t s_au16LvtOffsets[] =
+        {
+            XAPIC_OFF_LVT_LINT0,
+            XAPIC_OFF_LVT_LINT1
+        };
+        Assert(u8Pin < RT_ELEMENTS(s_au16LvtOffsets));
+        uint16_t const offLvt = s_au16LvtOffsets[u8Pin];
+        uint32_t const uLvt   = apicReadRaw32(pXApicPage, offLvt);
+
+        /* If software hasn't masked the interrupt in the LVT entry, proceed interrupt processing. */
+        if (!XAPIC_LVT_IS_MASKED(uLvt))
+        {
+            XAPICDELIVERYMODE const enmDeliveryMode = XAPIC_LVT_GET_DELIVERY_MODE(uLvt);
+            XAPICTRIGGERMODE        enmTriggerMode  = XAPIC_LVT_GET_TRIGGER_MODE(uLvt);
+
+            switch (enmDeliveryMode)
+            {
+                case XAPICDELIVERYMODE_INIT:
+                {
+                    /** @todo won't work in R0/RC because callers don't care about rcRZ. */
+                    AssertMsgFailed(("INIT through LINT0/LINT1 is not yet supported\n"));
+                }
+                RT_FALL_THRU();
+                case XAPICDELIVERYMODE_FIXED:
+                {
+                    PAPICCPU       pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+                    uint8_t const  uVector  = XAPIC_LVT_GET_VECTOR(uLvt);
+                    bool           fActive  = RT_BOOL(u8Level & 1);
+                    bool volatile *pfActiveLine = u8Pin == 0 ? &pApicCpu->fActiveLint0 : &pApicCpu->fActiveLint1;
+                    /** @todo Polarity is busted elsewhere, we need to fix that
+                     *        first. See @bugref{8386#c7}. */
+#if 0
+                    uint8_t const u8Polarity = XAPIC_LVT_GET_POLARITY(uLvt);
+                    fActive ^= u8Polarity; */
+#endif
+                    if (!fActive)
+                    {
+                        ASMAtomicCmpXchgBool(pfActiveLine, false, true);
+                        break;
+                    }
+
+                    /* Level-sensitive interrupts are not supported for LINT1. See Intel spec. 10.5.1 "Local Vector Table". */
+                    if (offLvt == XAPIC_OFF_LVT_LINT1)
+                        enmTriggerMode = XAPICTRIGGERMODE_EDGE;
+                    /** @todo figure out what "If the local APIC is not used in conjunction with an I/O APIC and fixed
+                              delivery mode is selected; the Pentium 4, Intel Xeon, and P6 family processors will always
+                              use level-sensitive triggering, regardless if edge-sensitive triggering is selected."
+                              means. */
+
+                    bool fSendIntr;
+                    if (enmTriggerMode == XAPICTRIGGERMODE_EDGE)
+                    {
+                        /* Recognize and send the interrupt only on an edge transition. */
+                        fSendIntr = ASMAtomicCmpXchgBool(pfActiveLine, true, false);
+                    }
+                    else
+                    {
+                        /* For level-triggered interrupts, redundant interrupts are not a problem. */
+                        Assert(enmTriggerMode == XAPICTRIGGERMODE_LEVEL);
+                        ASMAtomicCmpXchgBool(pfActiveLine, true, false);
+
+                        /* Only when the remote IRR isn't set, set it and send the interrupt. */
+                        if (!(pXApicPage->lvt_lint0.all.u32LvtLint0 & XAPIC_LVT_REMOTE_IRR))
+                        {
+                            Assert(offLvt == XAPIC_OFF_LVT_LINT0);
+                            ASMAtomicOrU32((volatile uint32_t *)&pXApicPage->lvt_lint0.all.u32LvtLint0, XAPIC_LVT_REMOTE_IRR);
+                            fSendIntr = true;
+                        }
+                        else
+                            fSendIntr = false;
+                    }
+
+                    if (fSendIntr)
+                    {
+                        VMCPUSET DestCpuSet;
+                        VMCPUSET_EMPTY(&DestCpuSet);
+                        VMCPUSET_ADD(&DestCpuSet, pVCpu->idCpu);
+                        rcStrict = apicSendIntr(pVCpu->CTX_SUFF(pVM), pVCpu, uVector, enmTriggerMode, enmDeliveryMode,
+                                                &DestCpuSet, NULL /* pfIntrAccepted */, 0 /* uSrcTag */, rcRZ);
+                    }
+                    break;
+                }
+
+                case XAPICDELIVERYMODE_SMI:
+                case XAPICDELIVERYMODE_NMI:
+                {
+                    VMCPUSET DestCpuSet;
+                    VMCPUSET_EMPTY(&DestCpuSet);
+                    VMCPUSET_ADD(&DestCpuSet, pVCpu->idCpu);
+                    uint8_t const uVector = XAPIC_LVT_GET_VECTOR(uLvt);
+                    rcStrict = apicSendIntr(pVCpu->CTX_SUFF(pVM), pVCpu, uVector, enmTriggerMode, enmDeliveryMode, &DestCpuSet,
+                                            NULL /* pfIntrAccepted */, 0 /* uSrcTag */, rcRZ);
+                    break;
+                }
+
+                case XAPICDELIVERYMODE_EXTINT:
+                {
+                    Log2(("APIC%u: apicLocalInterrupt: %s ExtINT through LINT%u\n", pVCpu->idCpu,
+                          u8Level ? "Raising" : "Lowering", u8Pin));
+                    if (u8Level)
+                        apicSetInterruptFF(pVCpu, PDMAPICIRQ_EXTINT);
+                    else
+                        apicClearInterruptFF(pVCpu, PDMAPICIRQ_EXTINT);
+                    break;
+                }
+
+                /* Reserved/unknown delivery modes: */
+                case XAPICDELIVERYMODE_LOWEST_PRIO:
+                case XAPICDELIVERYMODE_STARTUP:
+                default:
+                {
+                    AssertMsgFailed(("APIC%u: LocalInterrupt: Invalid delivery mode %#x (%s) on LINT%d\n", pVCpu->idCpu,
+                                     enmDeliveryMode, apicGetDeliveryModeName(enmDeliveryMode), u8Pin));
+                    rcStrict = VERR_INTERNAL_ERROR_3;
+                    break;
+                }
+            }
+        }
+    }
+    else
+    {
+        /* The APIC is hardware disabled. The CPU behaves as though there is no on-chip APIC. */
+        if (u8Pin == 0)
+        {
+            /* LINT0 behaves as an external interrupt pin. */
+            Log2(("APIC%u: apicLocalInterrupt: APIC hardware-disabled, %s INTR\n", pVCpu->idCpu,
+                  u8Level ? "raising" : "lowering"));
+            if (u8Level)
+                apicSetInterruptFF(pVCpu, PDMAPICIRQ_EXTINT);
+            else
+                apicClearInterruptFF(pVCpu, PDMAPICIRQ_EXTINT);
+        }
+        else
+        {
+            /* LINT1 behaves as NMI. */
+            Log2(("APIC%u: apicLocalInterrupt: APIC hardware-disabled, raising NMI\n", pVCpu->idCpu));
+            apicSetInterruptFF(pVCpu, PDMAPICIRQ_NMI);
+        }
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Gets the next highest-priority interrupt from the APIC, marking it as an
+ * "in-service" interrupt.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pu8Vector   Where to store the vector.
+ * @param   puSrcTag    Where to store the interrupt source tag (debugging).
+ */
+VMM_INT_DECL(int) APICGetInterrupt(PVMCPUCC pVCpu, uint8_t *pu8Vector, uint32_t *puSrcTag)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(pu8Vector);
+
+    LogFlow(("APIC%u: apicGetInterrupt:\n", pVCpu->idCpu));
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    bool const fApicHwEnabled = APICIsEnabled(pVCpu);
+    if (   fApicHwEnabled
+        && pXApicPage->svr.u.fApicSoftwareEnable)
+    {
+        int const irrv = apicGetHighestSetBitInReg(&pXApicPage->irr, -1);
+        if (RT_LIKELY(irrv >= 0))
+        {
+            Assert(irrv <= (int)UINT8_MAX);
+            uint8_t const uVector = irrv;
+
+            /*
+             * This can happen if the APIC receives an interrupt when the CPU has interrupts
+             * disabled but the TPR is raised by the guest before re-enabling interrupts.
+             */
+            uint8_t const uTpr = pXApicPage->tpr.u8Tpr;
+            if (   uTpr > 0
+                && XAPIC_TPR_GET_TP(uVector) <= XAPIC_TPR_GET_TP(uTpr))
+            {
+                Log2(("APIC%u: apicGetInterrupt: Interrupt masked. uVector=%#x uTpr=%#x SpuriousVector=%#x\n", pVCpu->idCpu,
+                      uVector, uTpr, pXApicPage->svr.u.u8SpuriousVector));
+                *pu8Vector = uVector;
+                *puSrcTag  = 0;
+                STAM_COUNTER_INC(&pVCpu->apic.s.StatMaskedByTpr);
+                return VERR_APIC_INTR_MASKED_BY_TPR;
+            }
+
+            /*
+             * The PPR should be up-to-date at this point through apicSetEoi().
+             * We're on EMT so no parallel updates possible.
+             * Subject the pending vector to PPR prioritization.
+             */
+            uint8_t const uPpr = pXApicPage->ppr.u8Ppr;
+            if (   !uPpr
+                || XAPIC_PPR_GET_PP(uVector) > XAPIC_PPR_GET_PP(uPpr))
+            {
+                apicClearVectorInReg(&pXApicPage->irr, uVector);
+                apicSetVectorInReg(&pXApicPage->isr, uVector);
+                apicUpdatePpr(pVCpu);
+                apicSignalNextPendingIntr(pVCpu);
+
+                /* Retrieve the interrupt source tag associated with this interrupt. */
+                PAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+                AssertCompile(RT_ELEMENTS(pApicCpu->auSrcTags) > UINT8_MAX);
+                *puSrcTag = pApicCpu->auSrcTags[uVector];
+                pApicCpu->auSrcTags[uVector] = 0;
+
+                Log2(("APIC%u: apicGetInterrupt: Valid Interrupt. uVector=%#x uSrcTag=%#x\n", pVCpu->idCpu, uVector, *puSrcTag));
+                *pu8Vector = uVector;
+                return VINF_SUCCESS;
+            }
+
+            STAM_COUNTER_INC(&pVCpu->apic.s.StatMaskedByPpr);
+            Log2(("APIC%u: apicGetInterrupt: Interrupt's priority is not higher than the PPR. uVector=%#x PPR=%#x\n",
+                  pVCpu->idCpu, uVector, uPpr));
+        }
+        else
+            Log2(("APIC%u: apicGetInterrupt: No pending bits in IRR\n", pVCpu->idCpu));
+    }
+    else
+        Log2(("APIC%u: apicGetInterrupt: APIC %s disabled\n", pVCpu->idCpu, !fApicHwEnabled ? "hardware" : "software"));
+
+    *pu8Vector = 0;
+    *puSrcTag  = 0;
+    return VERR_APIC_INTR_NOT_PENDING;
+}
+
+
+/**
+ * @callback_method_impl{FNIOMMMIONEWREAD}
+ */
+DECLCALLBACK(VBOXSTRICTRC) apicReadMmio(PPDMDEVINS pDevIns, void *pvUser, RTGCPHYS off, void *pv, unsigned cb)
+{
+    NOREF(pvUser);
+    Assert(!(off & 0xf));
+    Assert(cb == 4); RT_NOREF_PV(cb);
+
+    PVMCPUCC pVCpu    = PDMDevHlpGetVMCPU(pDevIns);
+    uint16_t offReg   = off & 0xff0;
+    uint32_t uValue   = 0;
+
+    STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMmioRead));
+
+    VBOXSTRICTRC rc = VBOXSTRICTRC_VAL(apicReadRegister(pDevIns, pVCpu, offReg, &uValue));
+    *(uint32_t *)pv = uValue;
+
+    Log2(("APIC%u: apicReadMmio: offReg=%#RX16 uValue=%#RX32\n", pVCpu->idCpu, offReg, uValue));
+    return rc;
+}
+
+
+/**
+ * @callback_method_impl{FNIOMMMIONEWWRITE}
+ */
+DECLCALLBACK(VBOXSTRICTRC) apicWriteMmio(PPDMDEVINS pDevIns, void *pvUser, RTGCPHYS off, void const *pv, unsigned cb)
+{
+    NOREF(pvUser);
+    Assert(!(off & 0xf));
+    Assert(cb == 4); RT_NOREF_PV(cb);
+
+    PVMCPUCC pVCpu    = PDMDevHlpGetVMCPU(pDevIns);
+    uint16_t offReg   = off & 0xff0;
+    uint32_t uValue   = *(uint32_t *)pv;
+
+    STAM_COUNTER_INC(&pVCpu->apic.s.CTX_SUFF_Z(StatMmioWrite));
+
+    Log2(("APIC%u: apicWriteMmio: offReg=%#RX16 uValue=%#RX32\n", pVCpu->idCpu, offReg, uValue));
+
+    return apicWriteRegister(pDevIns, pVCpu, offReg, uValue);
+}
+
+
+/**
+ * Sets the interrupt pending force-flag and pokes the EMT if required.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmType         The IRQ type.
+ */
+static void apicSetInterruptFF(PVMCPUCC pVCpu, PDMAPICIRQ enmType)
+{
+#ifdef IN_RING3
+    /* IRQ state should be loaded as-is by "LoadExec". Changes can be made from LoadDone. */
+    Assert(pVCpu->pVMR3->enmVMState != VMSTATE_LOADING || PDMR3HasLoadedState(pVCpu->pVMR3));
+#endif
+
+    switch (enmType)
+    {
+        case PDMAPICIRQ_HARDWARE:
+            VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+            break;
+        case PDMAPICIRQ_UPDATE_PENDING: VMCPU_FF_SET(pVCpu, VMCPU_FF_UPDATE_APIC);    break;
+        case PDMAPICIRQ_NMI:            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);  break;
+        case PDMAPICIRQ_SMI:            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI);  break;
+        case PDMAPICIRQ_EXTINT:         VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC);  break;
+        default:
+            AssertMsgFailed(("enmType=%d\n", enmType));
+            break;
+    }
+
+    /*
+     * We need to wake up the target CPU if we're not on EMT.
+     */
+    /** @todo r=bird: Why do we skip this waking up for PDMAPICIRQ_HARDWARE? */
+    /** @todo r=bird: We could just use RTThreadNativeSelf() here, couldn't we? */
+#if defined(IN_RING0)
+    PVMCC   pVM   = pVCpu->CTX_SUFF(pVM);
+    VMCPUID idCpu = pVCpu->idCpu;
+    if (   enmType != PDMAPICIRQ_HARDWARE
+        && VMMGetCpuId(pVM) != idCpu)
+    {
+        switch (VMCPU_GET_STATE(pVCpu))
+        {
+            case VMCPUSTATE_STARTED_EXEC:
+                Log7Func(("idCpu=%u VMCPUSTATE_STARTED_EXEC\n", idCpu));
+                GVMMR0SchedPokeNoGVMNoLock(pVM, idCpu);
+                break;
+
+            case VMCPUSTATE_STARTED_HALTED:
+                Log7Func(("idCpu=%u VMCPUSTATE_STARTED_HALTED\n", idCpu));
+                GVMMR0SchedWakeUpNoGVMNoLock(pVM, idCpu);
+                break;
+
+            default:
+                Log7Func(("idCpu=%u enmState=%d\n", idCpu, pVCpu->enmState));
+                break; /* nothing to do in other states. */
+        }
+    }
+#elif defined(IN_RING3)
+    PVMCC   pVM   = pVCpu->CTX_SUFF(pVM);
+    VMCPUID idCpu = pVCpu->idCpu;
+    if (   enmType != PDMAPICIRQ_HARDWARE
+        && VMMGetCpuId(pVM) != idCpu)
+    {
+        Log7Func(("idCpu=%u enmState=%d\n", idCpu, pVCpu->enmState));
+        VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM | VMNOTIFYFF_FLAGS_POKE);
+    }
+#endif
+}
+
+
+/**
+ * Clears the interrupt pending force-flag.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmType         The IRQ type.
+ */
+void apicClearInterruptFF(PVMCPUCC pVCpu, PDMAPICIRQ enmType)
+{
+#ifdef IN_RING3
+    /* IRQ state should be loaded as-is by "LoadExec". Changes can be made from LoadDone. */
+    Assert(pVCpu->pVMR3->enmVMState != VMSTATE_LOADING || PDMR3HasLoadedState(pVCpu->pVMR3));
+#endif
+
+    /* NMI/SMI can't be cleared. */
+    switch (enmType)
+    {
+        case PDMAPICIRQ_HARDWARE:   VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC); break;
+        case PDMAPICIRQ_EXTINT:     VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);  break;
+        default:
+            AssertMsgFailed(("enmType=%d\n", enmType));
+            break;
+    }
+}
+
+
+/**
+ * Posts an interrupt to a target APIC.
+ *
+ * This function handles interrupts received from the system bus or
+ * interrupts generated locally from the LVT or via a self IPI.
+ *
+ * Don't use this function to try and deliver ExtINT style interrupts.
+ *
+ * @returns true if the interrupt was accepted, false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uVector             The vector of the interrupt to be posted.
+ * @param   enmTriggerMode      The trigger mode of the interrupt.
+ * @param   uSrcTag             The interrupt source tag (debugging).
+ *
+ * @thread  Any.
+ */
+bool apicPostInterrupt(PVMCPUCC pVCpu, uint8_t uVector, XAPICTRIGGERMODE enmTriggerMode, uint32_t uSrcTag)
+{
+    Assert(pVCpu);
+    Assert(uVector > XAPIC_ILLEGAL_VECTOR_END);
+
+    PVMCC    pVM       = pVCpu->CTX_SUFF(pVM);
+    PCAPIC   pApic     = VM_TO_APIC(pVM);
+    PAPICCPU pApicCpu  = VMCPU_TO_APICCPU(pVCpu);
+    bool     fAccepted = true;
+
+    STAM_PROFILE_START(&pApicCpu->StatPostIntr, a);
+    STAM_REL_COUNTER_INC(&pApicCpu->StatPostIntrCnt);
+    STAM_REL_COUNTER_INC(&pApicCpu->aStatVectors[uVector]);
+
+    /*
+     * Only post valid interrupt vectors.
+     * See Intel spec. 10.5.2 "Valid Interrupt Vectors".
+     */
+    if (RT_LIKELY(uVector > XAPIC_ILLEGAL_VECTOR_END))
+    {
+        /*
+         * If the interrupt is already pending in the IRR we can skip the
+         * potential expensive operation of poking the guest EMT out of execution.
+         */
+        PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+        if (!apicTestVectorInReg(&pXApicPage->irr, uVector))     /* PAV */
+        {
+            /* Update the interrupt source tag (debugging). */
+            if (!pApicCpu->auSrcTags[uVector])
+                pApicCpu->auSrcTags[uVector]  = uSrcTag;
+            else
+                pApicCpu->auSrcTags[uVector] |= RT_BIT_32(31);
+
+            Log2(("APIC: apicPostInterrupt: SrcCpu=%u TargetCpu=%u uVector=%#x %s\n",
+                  VMMGetCpuId(pVM), pVCpu->idCpu, uVector, enmTriggerMode == XAPICTRIGGERMODE_EDGE ? "edge" : "lvl"));
+            if (enmTriggerMode == XAPICTRIGGERMODE_EDGE)
+            {
+                if (pApic->fPostedIntrsEnabled)
+                { /** @todo posted-interrupt call to hardware */ }
+                else
+                {
+                    apicSetVectorInPib(pApicCpu->CTX_SUFF(pvApicPib), uVector);
+                    uint32_t const fAlreadySet = apicSetNotificationBitInPib((PAPICPIB)pApicCpu->CTX_SUFF(pvApicPib));
+                    if (!fAlreadySet)
+                    {
+                        Log2(("APIC: apicPostInterrupt: Setting UPDATE_APIC FF for edge-triggered intr. uVector=%#x\n", uVector));
+                        apicSetInterruptFF(pVCpu, PDMAPICIRQ_UPDATE_PENDING);
+                    }
+                }
+            }
+            else
+            {
+                /*
+                 * Level-triggered interrupts requires updating of the TMR and thus cannot be
+                 * delivered asynchronously.
+                 */
+                apicSetVectorInPib(&pApicCpu->ApicPibLevel, uVector);
+                uint32_t const fAlreadySet = apicSetNotificationBitInPib(&pApicCpu->ApicPibLevel);
+                if (!fAlreadySet)
+                {
+                    Log2(("APIC: apicPostInterrupt: Setting UPDATE_APIC FF for level-triggered intr. uVector=%#x\n", uVector));
+                    apicSetInterruptFF(pVCpu, PDMAPICIRQ_UPDATE_PENDING);
+                }
+            }
+        }
+        else
+        {
+            Log2(("APIC: apicPostInterrupt: SrcCpu=%u TargetCpu=%u. Vector %#x Already in IRR, skipping\n", VMMGetCpuId(pVM),
+                  pVCpu->idCpu, uVector));
+            STAM_COUNTER_INC(&pApicCpu->StatPostIntrAlreadyPending);
+        }
+    }
+    else
+    {
+        fAccepted = false;
+        apicSetError(pVCpu, XAPIC_ESR_RECV_ILLEGAL_VECTOR);
+    }
+
+    STAM_PROFILE_STOP(&pApicCpu->StatPostIntr, a);
+    return fAccepted;
+}
+
+
+/**
+ * Starts the APIC timer.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uInitialCount   The timer's Initial-Count Register (ICR), must be >
+ *                          0.
+ * @thread  Any.
+ */
+void apicStartTimer(PVMCPUCC pVCpu, uint32_t uInitialCount)
+{
+    Assert(pVCpu);
+    PAPICCPU   pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    PPDMDEVINS pDevIns  = VMCPU_TO_DEVINS(pVCpu);
+    Assert(PDMDevHlpTimerIsLockOwner(pDevIns, pApicCpu->hTimer));
+    Assert(uInitialCount > 0);
+
+    PCXAPICPAGE    pXApicPage   = APICCPU_TO_CXAPICPAGE(pApicCpu);
+    uint8_t  const uTimerShift  = apicGetTimerShift(pXApicPage);
+    uint64_t const cTicksToNext = (uint64_t)uInitialCount << uTimerShift;
+
+    Log2(("APIC%u: apicStartTimer: uInitialCount=%#RX32 uTimerShift=%u cTicksToNext=%RU64\n", pVCpu->idCpu, uInitialCount,
+          uTimerShift, cTicksToNext));
+
+    /*
+     * The assumption here is that the timer doesn't tick during this call
+     * and thus setting a relative time to fire next is accurate. The advantage
+     * however is updating u64TimerInitial 'atomically' while setting the next
+     * tick.
+     */
+    PDMDevHlpTimerSetRelative(pDevIns, pApicCpu->hTimer, cTicksToNext, &pApicCpu->u64TimerInitial);
+    apicHintTimerFreq(pDevIns, pApicCpu, uInitialCount, uTimerShift);
+}
+
+
+/**
+ * Stops the APIC timer.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @thread  Any.
+ */
+static void apicStopTimer(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu);
+    PAPICCPU   pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    PPDMDEVINS pDevIns  = VMCPU_TO_DEVINS(pVCpu);
+    Assert(PDMDevHlpTimerIsLockOwner(pDevIns, pApicCpu->hTimer));
+
+    Log2(("APIC%u: apicStopTimer\n", pVCpu->idCpu));
+
+    PDMDevHlpTimerStop(pDevIns, pApicCpu->hTimer); /* This will reset the hint, no need to explicitly call TMTimerSetFrequencyHint(). */
+    pApicCpu->uHintedTimerInitialCount = 0;
+    pApicCpu->uHintedTimerShift = 0;
+}
+
+
+/**
+ * Queues a pending interrupt as in-service.
+ *
+ * This function should only be needed without virtualized APIC
+ * registers. With virtualized APIC registers, it's sufficient to keep
+ * the interrupts pending in the IRR as the hardware takes care of
+ * virtual interrupt delivery.
+ *
+ * @returns true if the interrupt was queued to in-service interrupts,
+ *          false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   u8PendingIntr       The pending interrupt to queue as
+ *                              in-service.
+ *
+ * @remarks This assumes the caller has done the necessary checks and
+ *          is ready to take actually service the interrupt (TPR,
+ *          interrupt shadow etc.)
+ */
+VMM_INT_DECL(bool) APICQueueInterruptToService(PVMCPUCC pVCpu, uint8_t u8PendingIntr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PVMCC pVM   = pVCpu->CTX_SUFF(pVM);
+    PAPIC pApic = VM_TO_APIC(pVM);
+    Assert(!pApic->fVirtApicRegsEnabled);
+    NOREF(pApic);
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    bool const fIsPending = apicTestVectorInReg(&pXApicPage->irr, u8PendingIntr);
+    if (fIsPending)
+    {
+        apicClearVectorInReg(&pXApicPage->irr, u8PendingIntr);
+        apicSetVectorInReg(&pXApicPage->isr, u8PendingIntr);
+        apicUpdatePpr(pVCpu);
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * De-queues a pending interrupt from in-service.
+ *
+ * This undoes APICQueueInterruptToService() for premature VM-exits before event
+ * injection.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   u8PendingIntr       The pending interrupt to de-queue from
+ *                              in-service.
+ */
+VMM_INT_DECL(void) APICDequeueInterruptFromService(PVMCPUCC pVCpu, uint8_t u8PendingIntr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PVMCC pVM   = pVCpu->CTX_SUFF(pVM);
+    PAPIC pApic = VM_TO_APIC(pVM);
+    Assert(!pApic->fVirtApicRegsEnabled);
+    NOREF(pApic);
+
+    PXAPICPAGE pXApicPage = VMCPU_TO_XAPICPAGE(pVCpu);
+    bool const fInService = apicTestVectorInReg(&pXApicPage->isr, u8PendingIntr);
+    if (fInService)
+    {
+        apicClearVectorInReg(&pXApicPage->isr, u8PendingIntr);
+        apicSetVectorInReg(&pXApicPage->irr, u8PendingIntr);
+        apicUpdatePpr(pVCpu);
+    }
+}
+
+
+/**
+ * Updates pending interrupts from the pending-interrupt bitmaps to the IRR.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ *
+ * @note    NEM/win is ASSUMING the an up to date TPR is not required here.
+ */
+VMMDECL(void) APICUpdatePendingInterrupts(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+
+    PAPICCPU   pApicCpu         = VMCPU_TO_APICCPU(pVCpu);
+    PXAPICPAGE pXApicPage       = VMCPU_TO_XAPICPAGE(pVCpu);
+    bool       fHasPendingIntrs = false;
+
+    Log3(("APIC%u: APICUpdatePendingInterrupts:\n", pVCpu->idCpu));
+    STAM_PROFILE_START(&pApicCpu->StatUpdatePendingIntrs, a);
+
+    /* Update edge-triggered pending interrupts. */
+    PAPICPIB pPib = (PAPICPIB)pApicCpu->CTX_SUFF(pvApicPib);
+    for (;;)
+    {
+        uint32_t const fAlreadySet = apicClearNotificationBitInPib((PAPICPIB)pApicCpu->CTX_SUFF(pvApicPib));
+        if (!fAlreadySet)
+            break;
+
+        AssertCompile(RT_ELEMENTS(pXApicPage->irr.u) == 2 * RT_ELEMENTS(pPib->au64VectorBitmap));
+        for (size_t idxPib = 0, idxReg = 0; idxPib < RT_ELEMENTS(pPib->au64VectorBitmap); idxPib++, idxReg += 2)
+        {
+            uint64_t const u64Fragment = ASMAtomicXchgU64(&pPib->au64VectorBitmap[idxPib], 0);
+            if (u64Fragment)
+            {
+                uint32_t const u32FragmentLo = RT_LO_U32(u64Fragment);
+                uint32_t const u32FragmentHi = RT_HI_U32(u64Fragment);
+                Log6Func(("edge[%u/%u]: %'016RX64: irr=%08RX32'%08RX32 |; tmr=%08RX32'%08RX32 &~\n", idxPib, idxReg, u64Fragment,
+                          pXApicPage->irr.u[idxReg].u32Reg, pXApicPage->irr.u[idxReg + 1].u32Reg,
+                          pXApicPage->tmr.u[idxReg].u32Reg, pXApicPage->tmr.u[idxReg + 1].u32Reg));
+
+                pXApicPage->irr.u[idxReg].u32Reg     |=  u32FragmentLo;
+                pXApicPage->irr.u[idxReg + 1].u32Reg |=  u32FragmentHi;
+
+                pXApicPage->tmr.u[idxReg].u32Reg     &= ~u32FragmentLo;
+                pXApicPage->tmr.u[idxReg + 1].u32Reg &= ~u32FragmentHi;
+                fHasPendingIntrs = true;
+            }
+        }
+    }
+
+    /* Update level-triggered pending interrupts. */
+    pPib = (PAPICPIB)&pApicCpu->ApicPibLevel;
+    for (;;)
+    {
+        uint32_t const fAlreadySet = apicClearNotificationBitInPib((PAPICPIB)&pApicCpu->ApicPibLevel);
+        if (!fAlreadySet)
+            break;
+
+        AssertCompile(RT_ELEMENTS(pXApicPage->irr.u) == 2 * RT_ELEMENTS(pPib->au64VectorBitmap));
+        for (size_t idxPib = 0, idxReg = 0; idxPib < RT_ELEMENTS(pPib->au64VectorBitmap); idxPib++, idxReg += 2)
+        {
+            uint64_t const u64Fragment = ASMAtomicXchgU64(&pPib->au64VectorBitmap[idxPib], 0);
+            if (u64Fragment)
+            {
+                Log6Func(("level[%u/%u]: %'016RX64: irr=%08RX32'%08RX32 |; tmr=%08RX32'%08RX32 |\n", idxPib, idxReg, u64Fragment,
+                          pXApicPage->irr.u[idxReg].u32Reg, pXApicPage->irr.u[idxReg + 1].u32Reg,
+                          pXApicPage->tmr.u[idxReg].u32Reg, pXApicPage->tmr.u[idxReg + 1].u32Reg));
+                uint32_t const u32FragmentLo = RT_LO_U32(u64Fragment);
+                uint32_t const u32FragmentHi = RT_HI_U32(u64Fragment);
+
+                pXApicPage->irr.u[idxReg].u32Reg     |= u32FragmentLo;
+                pXApicPage->irr.u[idxReg + 1].u32Reg |= u32FragmentHi;
+
+                pXApicPage->tmr.u[idxReg].u32Reg     |= u32FragmentLo;
+                pXApicPage->tmr.u[idxReg + 1].u32Reg |= u32FragmentHi;
+                fHasPendingIntrs = true;
+            }
+        }
+    }
+
+    STAM_PROFILE_STOP(&pApicCpu->StatUpdatePendingIntrs, a);
+    Log3(("APIC%u: APICUpdatePendingInterrupts: fHasPendingIntrs=%RTbool\n", pVCpu->idCpu, fHasPendingIntrs));
+
+    if (   fHasPendingIntrs
+        && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC))
+        apicSignalNextPendingIntr(pVCpu);
+}
+
+
+/**
+ * Gets the highest priority pending interrupt.
+ *
+ * @returns true if any interrupt is pending, false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pu8PendingIntr      Where to store the interrupt vector if the
+ *                              interrupt is pending.
+ */
+VMM_INT_DECL(bool) APICGetHighestPendingInterrupt(PVMCPUCC pVCpu, uint8_t *pu8PendingIntr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    return apicGetHighestPendingInterrupt(pVCpu, pu8PendingIntr);
+}
+
+
+/**
+ * Posts an interrupt to a target APIC, Hyper-V interface.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uVector             The vector of the interrupt to be posted.
+ * @param   fAutoEoi            Whether this interrupt has automatic EOI
+ *                              treatment.
+ * @param   enmTriggerMode      The trigger mode of the interrupt.
+ *
+ * @thread  Any.
+ */
+VMM_INT_DECL(void) APICHvSendInterrupt(PVMCPUCC pVCpu, uint8_t uVector, bool fAutoEoi, XAPICTRIGGERMODE enmTriggerMode)
+{
+    Assert(pVCpu);
+    Assert(!fAutoEoi);    /** @todo AutoEOI.  */
+    RT_NOREF(fAutoEoi);
+    apicPostInterrupt(pVCpu, uVector, enmTriggerMode, 0 /* uSrcTag */);
+}
+
+
+/**
+ * Sets the Task Priority Register (TPR), Hyper-V interface.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uTpr        The TPR value to set.
+ *
+ * @remarks Validates like in x2APIC mode.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICHvSetTpr(PVMCPUCC pVCpu, uint8_t uTpr)
+{
+    Assert(pVCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+    return apicSetTprEx(pVCpu, uTpr, true /* fForceX2ApicBehaviour */);
+}
+
+
+/**
+ * Gets the Task Priority Register (TPR), Hyper-V interface.
+ *
+ * @returns The TPR value.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(uint8_t) APICHvGetTpr(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * The APIC could be operating in xAPIC mode and thus we should not use the apicReadMsr()
+     * interface which validates the APIC mode and will throw a #GP(0) if not in x2APIC mode.
+     * We could use the apicReadRegister() MMIO interface, but why bother getting the PDMDEVINS
+     * pointer, so just directly read the APIC page.
+     */
+    PCXAPICPAGE pXApicPage = VMCPU_TO_CXAPICPAGE(pVCpu);
+    return apicReadRaw32(pXApicPage, XAPIC_OFF_TPR);
+}
+
+
+/**
+ * Sets the Interrupt Command Register (ICR), Hyper-V interface.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uIcr        The ICR value to set.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICHvSetIcr(PVMCPUCC pVCpu, uint64_t uIcr)
+{
+    Assert(pVCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+    return apicSetIcr(pVCpu, uIcr, VINF_CPUM_R3_MSR_WRITE);
+}
+
+
+/**
+ * Gets the Interrupt Command Register (ICR), Hyper-V interface.
+ *
+ * @returns The ICR value.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(uint64_t) APICHvGetIcr(PVMCPUCC pVCpu)
+{
+    Assert(pVCpu);
+    VMCPU_ASSERT_EMT(pVCpu);
+    return apicGetIcrNoCheck(pVCpu);
+}
+
+
+/**
+ * Sets the End-Of-Interrupt (EOI) register, Hyper-V interface.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uEoi            The EOI value.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) APICHvSetEoi(PVMCPUCC pVCpu, uint32_t uEoi)
+{
+    Assert(pVCpu);
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+    return apicSetEoi(pVCpu, uEoi, true /* fForceX2ApicBehaviour */);
+}
+
+
+/**
+ * Gets the APIC page pointers for the specified VCPU.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pHCPhys         Where to store the host-context physical address.
+ * @param   pR0Ptr          Where to store the ring-0 address.
+ * @param   pR3Ptr          Where to store the ring-3 address (optional).
+ */
+VMM_INT_DECL(int) APICGetApicPageForCpu(PCVMCPUCC pVCpu, PRTHCPHYS pHCPhys, PRTR0PTR pR0Ptr, PRTR3PTR pR3Ptr)
+{
+    AssertReturn(pVCpu,   VERR_INVALID_PARAMETER);
+    AssertReturn(pHCPhys, VERR_INVALID_PARAMETER);
+    AssertReturn(pR0Ptr,  VERR_INVALID_PARAMETER);
+
+    Assert(PDMHasApic(pVCpu->CTX_SUFF(pVM)));
+
+    PCAPICCPU pApicCpu = VMCPU_TO_APICCPU(pVCpu);
+    *pHCPhys = pApicCpu->HCPhysApicPage;
+    *pR0Ptr  = pApicCpu->pvApicPageR0;
+    if (pR3Ptr)
+        *pR3Ptr  = pApicCpu->pvApicPageR3;
+    return VINF_SUCCESS;
+}
+
+#ifndef IN_RING3
+
+/**
+ * @callback_method_impl{PDMDEVREGR0,pfnConstruct}
+ */
+static DECLCALLBACK(int) apicRZConstruct(PPDMDEVINS pDevIns)
+{
+    PDMDEV_CHECK_VERSIONS_RETURN(pDevIns);
+    PAPICDEV pThis = PDMDEVINS_2_DATA(pDevIns, PAPICDEV);
+    PVMCC    pVM   = PDMDevHlpGetVM(pDevIns);
+
+    pVM->apicr0.s.pDevInsR0 = pDevIns;
+
+    int rc = PDMDevHlpSetDeviceCritSect(pDevIns, PDMDevHlpCritSectGetNop(pDevIns));
+    AssertRCReturn(rc, rc);
+
+    rc = PDMDevHlpApicSetUpContext(pDevIns);
+    AssertRCReturn(rc, rc);
+
+    rc = PDMDevHlpMmioSetUpContext(pDevIns, pThis->hMmio, apicWriteMmio, apicReadMmio, NULL /*pvUser*/);
+    AssertRCReturn(rc, rc);
+
+    return VINF_SUCCESS;
+}
+#endif /* !IN_RING3 */
+
+/**
+ * APIC device registration structure.
+ */
+const PDMDEVREG g_DeviceAPIC =
+{
+    /* .u32Version = */             PDM_DEVREG_VERSION,
+    /* .uReserved0 = */             0,
+    /* .szName = */                 "apic",
+    /* .fFlags = */                 PDM_DEVREG_FLAGS_DEFAULT_BITS | PDM_DEVREG_FLAGS_RZ | PDM_DEVREG_FLAGS_NEW_STYLE
+                                    | PDM_DEVREG_FLAGS_REQUIRE_R0 | PDM_DEVREG_FLAGS_REQUIRE_RC,
+    /* .fClass = */                 PDM_DEVREG_CLASS_PIC,
+    /* .cMaxInstances = */          1,
+    /* .uSharedVersion = */         42,
+    /* .cbInstanceShared = */       sizeof(APICDEV),
+    /* .cbInstanceCC = */           0,
+    /* .cbInstanceRC = */           0,
+    /* .cMaxPciDevices = */         0,
+    /* .cMaxMsixVectors = */        0,
+    /* .pszDescription = */         "Advanced Programmable Interrupt Controller",
+#if defined(IN_RING3)
+    /* .szRCMod = */                "VMMRC.rc",
+    /* .szR0Mod = */                "VMMR0.r0",
+    /* .pfnConstruct = */           apicR3Construct,
+    /* .pfnDestruct = */            apicR3Destruct,
+    /* .pfnRelocate = */            apicR3Relocate,
+    /* .pfnMemSetup = */            NULL,
+    /* .pfnPowerOn = */             NULL,
+    /* .pfnReset = */               apicR3Reset,
+    /* .pfnSuspend = */             NULL,
+    /* .pfnResume = */              NULL,
+    /* .pfnAttach = */              NULL,
+    /* .pfnDetach = */              NULL,
+    /* .pfnQueryInterface = */      NULL,
+    /* .pfnInitComplete = */        apicR3InitComplete,
+    /* .pfnPowerOff = */            NULL,
+    /* .pfnSoftReset = */           NULL,
+    /* .pfnReserved0 = */           NULL,
+    /* .pfnReserved1 = */           NULL,
+    /* .pfnReserved2 = */           NULL,
+    /* .pfnReserved3 = */           NULL,
+    /* .pfnReserved4 = */           NULL,
+    /* .pfnReserved5 = */           NULL,
+    /* .pfnReserved6 = */           NULL,
+    /* .pfnReserved7 = */           NULL,
+#elif defined(IN_RING0)
+    /* .pfnEarlyConstruct = */      NULL,
+    /* .pfnConstruct = */           apicRZConstruct,
+    /* .pfnDestruct = */            NULL,
+    /* .pfnFinalDestruct = */       NULL,
+    /* .pfnRequest = */             NULL,
+    /* .pfnReserved0 = */           NULL,
+    /* .pfnReserved1 = */           NULL,
+    /* .pfnReserved2 = */           NULL,
+    /* .pfnReserved3 = */           NULL,
+    /* .pfnReserved4 = */           NULL,
+    /* .pfnReserved5 = */           NULL,
+    /* .pfnReserved6 = */           NULL,
+    /* .pfnReserved7 = */           NULL,
+#elif defined(IN_RC)
+    /* .pfnConstruct = */           apicRZConstruct,
+    /* .pfnReserved0 = */           NULL,
+    /* .pfnReserved1 = */           NULL,
+    /* .pfnReserved2 = */           NULL,
+    /* .pfnReserved3 = */           NULL,
+    /* .pfnReserved4 = */           NULL,
+    /* .pfnReserved5 = */           NULL,
+    /* .pfnReserved6 = */           NULL,
+    /* .pfnReserved7 = */           NULL,
+#else
+# error "Not in IN_RING3, IN_RING0 or IN_RC!"
+#endif
+    /* .u32VersionEnd = */          PDM_DEVREG_VERSION
+};
+
diff --git a/src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp b/src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp
new file mode 100644
index 00000000..827236a5
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/AllPdbTypeHack.cpp
@@ -0,0 +1,111 @@
+/* $Id: AllPdbTypeHack.cpp $ */
+/** @file
+ * Debug info hack for the VM and VMCPU structures.
+ */
+
+/*
+ * Copyright (C) 2016-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include <VBox/vmm/cfgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/stam.h>
+#include "../include/PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include "../include/CFGMInternal.h"
+#include "../include/CPUMInternal.h"
+#include "../include/MMInternal.h"
+#include "../include/PGMInternal.h"
+#include "../include/SELMInternal.h"
+#include "../include/TRPMInternal.h"
+#include "../include/TMInternal.h"
+#include "../include/IOMInternal.h"
+#ifdef IN_RING3
+# include "../include/SSMInternal.h"
+#endif
+#include "../include/HMInternal.h"
+#include "../include/VMMInternal.h"
+#include "../include/DBGFInternal.h"
+#include "../include/GIMInternal.h"
+#include "../include/APICInternal.h"
+#include "../include/STAMInternal.h"
+#include "../include/VMInternal.h"
+#include "../include/EMInternal.h"
+#include "../include/IEMInternal.h"
+#include "../include/NEMInternal.h"
+#include "../VMMR0/GMMR0Internal.h"
+#include "../VMMR0/GVMMR0Internal.h"
+#include <VBox/vmm/vmcc.h>
+#ifdef IN_RING3
+# include <VBox/vmm/uvm.h>
+#endif
+#include <VBox/vmm/gvm.h>
+
+
+extern "C" {
+
+/* Global pointer variables as an alternative to the parameter list.  Just to ensure the precense of the types. */
+PVM                 g_PdbTypeHack1 = NULL;
+PVMCPU              g_PdbTypeHack2 = NULL;
+PPDMCRITSECT        g_PdbTypeHack3 = NULL;
+PPDMCRITSECTRW      g_PdbTypeHack4 = NULL;
+PPDMDEVINS          g_PdbTypeHack5 = NULL;
+PPDMDRVINS          g_PdbTypeHack6 = NULL;
+PPDMUSBINS          g_PdbTypeHack7 = NULL;
+PCVMCPU             g_PdbTypeHack8 = NULL;
+CTX_SUFF(PVM)       g_PdbTypeHack9 = NULL;
+CTX_SUFF(PVMCPU)    g_PdbTypeHack10 = NULL;
+
+DECLEXPORT(uint32_t) PdbTypeHack(PVM pVM, PVMCPU pVCpu, PPDMCRITSECT pCs1, PPDMCRITSECTRW pCs2);
+}
+
+DECLEXPORT(uint32_t) PdbTypeHack(PVM pVM, PVMCPU pVCpu, PPDMCRITSECT pCs1, PPDMCRITSECTRW pCs2)
+{
+    /* Just some dummy operations accessing each type. Probably not necessary, but
+       helps making sure we've included all we need to get at the internal stuff.. */
+    return pVM->fGlobalForcedActions
+         | (pVM == g_PdbTypeHack1)
+         | (pVM == g_PdbTypeHack9)
+         | pVCpu->fLocalForcedActions
+         | (pVCpu == g_PdbTypeHack2)
+         | (pVCpu == g_PdbTypeHack8)
+         | (pVCpu == g_PdbTypeHack10)
+         | pCs1->s.Core.fFlags
+         | (pCs1 == g_PdbTypeHack3)
+         | pCs2->s.Core.fFlags
+         | (pCs2 == g_PdbTypeHack4)
+         | g_PdbTypeHack5->Internal.s.idxR0Device
+         | (g_PdbTypeHack5 != NULL)
+         | (uint32_t)g_PdbTypeHack6->Internal.s.fDetaching
+         | (g_PdbTypeHack6 != NULL)
+         | (uint32_t)g_PdbTypeHack7->Internal.s.fVMSuspended
+         | (g_PdbTypeHack7 != NULL);
+}
+
diff --git a/src/VBox/VMM/VMMAll/CPUMAllCpuId.cpp b/src/VBox/VMM/VMMAll/CPUMAllCpuId.cpp
new file mode 100644
index 00000000..48a309b4
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/CPUMAllCpuId.cpp
@@ -0,0 +1,1599 @@
+/* $Id: CPUMAllCpuId.cpp $ */
+/** @file
+ * CPUM - CPU ID part, common bits.
+ */
+
+/*
+ * Copyright (C) 2013-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/ssm.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/sup.h>
+
+#include <VBox/err.h>
+#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
+# include <iprt/asm-amd64-x86.h>
+#endif
+#include <iprt/ctype.h>
+#include <iprt/mem.h>
+#include <iprt/string.h>
+#include <iprt/x86-helpers.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/**
+ * The intel pentium family.
+ */
+static const CPUMMICROARCH g_aenmIntelFamily06[] =
+{
+    /* [ 0(0x00)] = */ kCpumMicroarch_Intel_P6,           /* Pentium Pro A-step (says sandpile.org). */
+    /* [ 1(0x01)] = */ kCpumMicroarch_Intel_P6,           /* Pentium Pro */
+    /* [ 2(0x02)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [ 3(0x03)] = */ kCpumMicroarch_Intel_P6_II,        /* PII Klamath */
+    /* [ 4(0x04)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [ 5(0x05)] = */ kCpumMicroarch_Intel_P6_II,        /* PII Deschutes */
+    /* [ 6(0x06)] = */ kCpumMicroarch_Intel_P6_II,        /* Celeron Mendocino. */
+    /* [ 7(0x07)] = */ kCpumMicroarch_Intel_P6_III,       /* PIII Katmai. */
+    /* [ 8(0x08)] = */ kCpumMicroarch_Intel_P6_III,       /* PIII Coppermine (includes Celeron). */
+    /* [ 9(0x09)] = */ kCpumMicroarch_Intel_P6_M_Banias,  /* Pentium/Celeron M Banias. */
+    /* [10(0x0a)] = */ kCpumMicroarch_Intel_P6_III,       /* PIII Xeon */
+    /* [11(0x0b)] = */ kCpumMicroarch_Intel_P6_III,       /* PIII Tualatin (includes Celeron). */
+    /* [12(0x0c)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [13(0x0d)] = */ kCpumMicroarch_Intel_P6_M_Dothan,  /* Pentium/Celeron M Dothan. */
+    /* [14(0x0e)] = */ kCpumMicroarch_Intel_Core_Yonah,   /* Core Yonah (Enhanced Pentium M). */
+    /* [15(0x0f)] = */ kCpumMicroarch_Intel_Core2_Merom,  /* Merom */
+    /* [16(0x10)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [17(0x11)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [18(0x12)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [19(0x13)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [20(0x14)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [21(0x15)] = */ kCpumMicroarch_Intel_P6_M_Dothan,  /* Tolapai - System-on-a-chip. */
+    /* [22(0x16)] = */ kCpumMicroarch_Intel_Core2_Merom,
+    /* [23(0x17)] = */ kCpumMicroarch_Intel_Core2_Penryn,
+    /* [24(0x18)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [25(0x19)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [26(0x1a)] = */ kCpumMicroarch_Intel_Core7_Nehalem, /* Nehalem-EP */
+    /* [27(0x1b)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [28(0x1c)] = */ kCpumMicroarch_Intel_Atom_Bonnell, /* Diamonville, Pineview, */
+    /* [29(0x1d)] = */ kCpumMicroarch_Intel_Core2_Penryn,
+    /* [30(0x1e)] = */ kCpumMicroarch_Intel_Core7_Nehalem, /* Clarksfield, Lynnfield, Jasper Forest. */
+    /* [31(0x1f)] = */ kCpumMicroarch_Intel_Core7_Nehalem, /* Only listed by sandpile.org.  2 cores ABD/HVD, whatever that means. */
+    /* [32(0x20)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [33(0x21)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [34(0x22)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [35(0x23)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [36(0x24)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [37(0x25)] = */ kCpumMicroarch_Intel_Core7_Westmere, /* Arrandale, Clarksdale. */
+    /* [38(0x26)] = */ kCpumMicroarch_Intel_Atom_Lincroft,
+    /* [39(0x27)] = */ kCpumMicroarch_Intel_Atom_Saltwell,
+    /* [40(0x28)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [41(0x29)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [42(0x2a)] = */ kCpumMicroarch_Intel_Core7_SandyBridge,
+    /* [43(0x2b)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [44(0x2c)] = */ kCpumMicroarch_Intel_Core7_Westmere, /* Gulftown, Westmere-EP. */
+    /* [45(0x2d)] = */ kCpumMicroarch_Intel_Core7_SandyBridge, /* SandyBridge-E, SandyBridge-EN, SandyBridge-EP. */
+    /* [46(0x2e)] = */ kCpumMicroarch_Intel_Core7_Nehalem,  /* Beckton (Xeon). */
+    /* [47(0x2f)] = */ kCpumMicroarch_Intel_Core7_Westmere, /* Westmere-EX. */
+    /* [48(0x30)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [49(0x31)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [50(0x32)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [51(0x33)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [52(0x34)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [53(0x35)] = */ kCpumMicroarch_Intel_Atom_Saltwell, /* ?? */
+    /* [54(0x36)] = */ kCpumMicroarch_Intel_Atom_Saltwell, /* Cedarview, ++ */
+    /* [55(0x37)] = */ kCpumMicroarch_Intel_Atom_Silvermont,
+    /* [56(0x38)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [57(0x39)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [58(0x3a)] = */ kCpumMicroarch_Intel_Core7_IvyBridge,
+    /* [59(0x3b)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [60(0x3c)] = */ kCpumMicroarch_Intel_Core7_Haswell,
+    /* [61(0x3d)] = */ kCpumMicroarch_Intel_Core7_Broadwell,
+    /* [62(0x3e)] = */ kCpumMicroarch_Intel_Core7_IvyBridge,
+    /* [63(0x3f)] = */ kCpumMicroarch_Intel_Core7_Haswell,
+    /* [64(0x40)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [65(0x41)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [66(0x42)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [67(0x43)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [68(0x44)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [69(0x45)] = */ kCpumMicroarch_Intel_Core7_Haswell,
+    /* [70(0x46)] = */ kCpumMicroarch_Intel_Core7_Haswell,
+    /* [71(0x47)] = */ kCpumMicroarch_Intel_Core7_Broadwell,    /* i7-5775C */
+    /* [72(0x48)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [73(0x49)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [74(0x4a)] = */ kCpumMicroarch_Intel_Atom_Silvermont,
+    /* [75(0x4b)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [76(0x4c)] = */ kCpumMicroarch_Intel_Atom_Airmount,
+    /* [77(0x4d)] = */ kCpumMicroarch_Intel_Atom_Silvermont,
+    /* [78(0x4e)] = */ kCpumMicroarch_Intel_Core7_Skylake,
+    /* [79(0x4f)] = */ kCpumMicroarch_Intel_Core7_Broadwell,    /* Broadwell-E */
+    /* [80(0x50)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [81(0x51)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [82(0x52)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [83(0x53)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [84(0x54)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [85(0x55)] = */ kCpumMicroarch_Intel_Core7_Skylake,      /* server cpu; skylake <= 4, cascade lake > 5 */
+    /* [86(0x56)] = */ kCpumMicroarch_Intel_Core7_Broadwell,    /* Xeon D-1540, Broadwell-DE */
+    /* [87(0x57)] = */ kCpumMicroarch_Intel_Phi_KnightsLanding,
+    /* [88(0x58)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [89(0x59)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [90(0x5a)] = */ kCpumMicroarch_Intel_Atom_Silvermont,    /* Moorefield */
+    /* [91(0x5b)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [92(0x5c)] = */ kCpumMicroarch_Intel_Atom_Goldmont,      /* Apollo Lake */
+    /* [93(0x5d)] = */ kCpumMicroarch_Intel_Atom_Silvermont,    /* x3-C3230 */
+    /* [94(0x5e)] = */ kCpumMicroarch_Intel_Core7_Skylake,      /* i7-6700K */
+    /* [95(0x5f)] = */ kCpumMicroarch_Intel_Atom_Goldmont,      /* Denverton */
+    /* [96(0x60)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [97(0x61)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [98(0x62)] = */ kCpumMicroarch_Intel_Unknown,
+    /* [99(0x63)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[100(0x64)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[101(0x65)] = */ kCpumMicroarch_Intel_Atom_Silvermont,    /* SoFIA */
+    /*[102(0x66)] = */ kCpumMicroarch_Intel_Core7_CannonLake, /* unconfirmed */
+    /*[103(0x67)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[104(0x68)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[105(0x69)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[106(0x6a)] = */ kCpumMicroarch_Intel_Core7_IceLake,      /* unconfirmed server */
+    /*[107(0x6b)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[108(0x6c)] = */ kCpumMicroarch_Intel_Core7_IceLake,      /* unconfirmed server */
+    /*[109(0x6d)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[110(0x6e)] = */ kCpumMicroarch_Intel_Atom_Airmount,      /* or silvermount? */
+    /*[111(0x6f)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[112(0x70)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[113(0x71)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[114(0x72)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[115(0x73)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[116(0x74)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[117(0x75)] = */ kCpumMicroarch_Intel_Atom_Airmount,      /* or silvermount? */
+    /*[118(0x76)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[119(0x77)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[120(0x78)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[121(0x79)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[122(0x7a)] = */ kCpumMicroarch_Intel_Atom_GoldmontPlus,
+    /*[123(0x7b)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[124(0x7c)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[125(0x7d)] = */ kCpumMicroarch_Intel_Core7_IceLake,      /* unconfirmed */
+    /*[126(0x7e)] = */ kCpumMicroarch_Intel_Core7_IceLake,      /* unconfirmed */
+    /*[127(0x7f)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[128(0x80)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[129(0x81)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[130(0x82)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[131(0x83)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[132(0x84)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[133(0x85)] = */ kCpumMicroarch_Intel_Phi_KnightsMill,
+    /*[134(0x86)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[135(0x87)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[136(0x88)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[137(0x89)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[138(0x8a)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[139(0x8b)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[140(0x8c)] = */ kCpumMicroarch_Intel_Core7_TigerLake,    /* 11th Gen Intel(R) Core(TM) i7-1185G7 @ 3.00GHz (bird) */
+    /*[141(0x8d)] = */ kCpumMicroarch_Intel_Core7_TigerLake,    /* unconfirmed */
+    /*[142(0x8e)] = */ kCpumMicroarch_Intel_Core7_KabyLake,     /* Stepping >= 0xB is Whiskey Lake, 0xA is CoffeeLake. */
+    /*[143(0x8f)] = */ kCpumMicroarch_Intel_Core7_SapphireRapids,
+    /*[144(0x90)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[145(0x91)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[146(0x92)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[147(0x93)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[148(0x94)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[149(0x95)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[150(0x96)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[151(0x97)] = */ kCpumMicroarch_Intel_Core7_AlderLake,    /* unconfirmed, unreleased */
+    /*[152(0x98)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[153(0x99)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[154(0x9a)] = */ kCpumMicroarch_Intel_Core7_AlderLake,    /* unconfirmed, unreleased */
+    /*[155(0x9b)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[156(0x9c)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[157(0x9d)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[158(0x9e)] = */ kCpumMicroarch_Intel_Core7_KabyLake, /* Stepping >= 0xB is Whiskey Lake, 0xA is CoffeeLake. */
+    /*[159(0x9f)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[160(0xa0)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[161(0xa1)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[162(0xa2)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[163(0xa3)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[164(0xa4)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[165(0xa5)] = */ kCpumMicroarch_Intel_Core7_CometLake,    /* unconfirmed */
+    /*[166(0xa6)] = */ kCpumMicroarch_Intel_Unknown,
+    /*[167(0xa7)] = */ kCpumMicroarch_Intel_Core7_CypressCove, /* 14nm backport, unconfirmed */
+};
+AssertCompile(RT_ELEMENTS(g_aenmIntelFamily06) == 0xa7+1);
+
+
+/**
+ * Figures out the (sub-)micro architecture given a bit of CPUID info.
+ *
+ * @returns Micro architecture.
+ * @param   enmVendor           The CPU vendor.
+ * @param   bFamily             The CPU family.
+ * @param   bModel              The CPU model.
+ * @param   bStepping           The CPU stepping.
+ */
+VMMDECL(CPUMMICROARCH) CPUMCpuIdDetermineX86MicroarchEx(CPUMCPUVENDOR enmVendor, uint8_t bFamily,
+                                                        uint8_t bModel, uint8_t bStepping)
+{
+    if (enmVendor == CPUMCPUVENDOR_AMD)
+    {
+        switch (bFamily)
+        {
+            case 0x02:  return kCpumMicroarch_AMD_Am286; /* Not really kosher... */
+            case 0x03:  return kCpumMicroarch_AMD_Am386;
+            case 0x23:  return kCpumMicroarch_AMD_Am386; /* SX*/
+            case 0x04:  return bModel < 14 ? kCpumMicroarch_AMD_Am486 : kCpumMicroarch_AMD_Am486Enh;
+            case 0x05:  return bModel <  6 ? kCpumMicroarch_AMD_K5    : kCpumMicroarch_AMD_K6; /* Genode LX is 0x0a, lump it with K6. */
+            case 0x06:
+                switch (bModel)
+                {
+                    case  0: return kCpumMicroarch_AMD_K7_Palomino;
+                    case  1: return kCpumMicroarch_AMD_K7_Palomino;
+                    case  2: return kCpumMicroarch_AMD_K7_Palomino;
+                    case  3: return kCpumMicroarch_AMD_K7_Spitfire;
+                    case  4: return kCpumMicroarch_AMD_K7_Thunderbird;
+                    case  6: return kCpumMicroarch_AMD_K7_Palomino;
+                    case  7: return kCpumMicroarch_AMD_K7_Morgan;
+                    case  8: return kCpumMicroarch_AMD_K7_Thoroughbred;
+                    case 10: return kCpumMicroarch_AMD_K7_Barton; /* Thorton too. */
+                }
+                return kCpumMicroarch_AMD_K7_Unknown;
+            case 0x0f:
+                /*
+                 * This family is a friggin mess. Trying my best to make some
+                 * sense out of it. Too much happened in the 0x0f family to
+                 * lump it all together as K8 (130nm->90nm->65nm, AMD-V, ++).
+                 *
+                 * Emperical CPUID.01h.EAX evidence from revision guides, wikipedia,
+                 * cpu-world.com, and other places:
+                 *  - 130nm:
+                 *     - ClawHammer:    F7A/SH-CG, F5A/-CG, F4A/-CG, F50/-B0, F48/-C0, F58/-C0,
+                 *     - SledgeHammer:  F50/SH-B0, F48/-C0, F58/-C0, F4A/-CG, F5A/-CG, F7A/-CG, F51/-B3
+                 *     - Newcastle:     FC0/DH-CG (erratum #180: FE0/DH-CG), FF0/DH-CG
+                 *     - Dublin:        FC0/-CG, FF0/-CG, F82/CH-CG, F4A/-CG, F48/SH-C0,
+                 *     - Odessa:        FC0/DH-CG (erratum #180: FE0/DH-CG)
+                 *     - Paris:         FF0/DH-CG, FC0/DH-CG (erratum #180: FE0/DH-CG),
+                 *  - 90nm:
+                 *     - Winchester:    10FF0/DH-D0, 20FF0/DH-E3.
+                 *     - Oakville:      10FC0/DH-D0.
+                 *     - Georgetown:    10FC0/DH-D0.
+                 *     - Sonora:        10FC0/DH-D0.
+                 *     - Venus:         20F71/SH-E4
+                 *     - Troy:          20F51/SH-E4
+                 *     - Athens:        20F51/SH-E4
+                 *     - San Diego:     20F71/SH-E4.
+                 *     - Lancaster:     20F42/SH-E5
+                 *     - Newark:        20F42/SH-E5.
+                 *     - Albany:        20FC2/DH-E6.
+                 *     - Roma:          20FC2/DH-E6.
+                 *     - Venice:        20FF0/DH-E3, 20FC2/DH-E6, 20FF2/DH-E6.
+                 *     - Palermo:       10FC0/DH-D0, 20FF0/DH-E3, 20FC0/DH-E3, 20FC2/DH-E6, 20FF2/DH-E6
+                 *  - 90nm introducing Dual core:
+                 *     - Denmark:       20F30/JH-E1, 20F32/JH-E6
+                 *     - Italy:         20F10/JH-E1, 20F12/JH-E6
+                 *     - Egypt:         20F10/JH-E1, 20F12/JH-E6
+                 *     - Toledo:        20F32/JH-E6, 30F72/DH-E6 (single code variant).
+                 *     - Manchester:    20FB1/BH-E4, 30FF2/BH-E4.
+                 *  - 90nm 2nd gen opteron ++, AMD-V introduced (might be missing in some cheaper models):
+                 *     - Santa Ana:     40F32/JH-F2, /-F3
+                 *     - Santa Rosa:    40F12/JH-F2, 40F13/JH-F3
+                 *     - Windsor:       40F32/JH-F2, 40F33/JH-F3, C0F13/JH-F3, 40FB2/BH-F2, ??20FB1/BH-E4??.
+                 *     - Manila:        50FF2/DH-F2, 40FF2/DH-F2
+                 *     - Orleans:       40FF2/DH-F2, 50FF2/DH-F2, 50FF3/DH-F3.
+                 *     - Keene:         40FC2/DH-F2.
+                 *     - Richmond:      40FC2/DH-F2
+                 *     - Taylor:        40F82/BH-F2
+                 *     - Trinidad:      40F82/BH-F2
+                 *
+                 *  - 65nm:
+                 *     - Brisbane:      60FB1/BH-G1, 60FB2/BH-G2.
+                 *     - Tyler:         60F81/BH-G1, 60F82/BH-G2.
+                 *     - Sparta:        70FF1/DH-G1, 70FF2/DH-G2.
+                 *     - Lima:          70FF1/DH-G1, 70FF2/DH-G2.
+                 *     - Sherman:       /-G1, 70FC2/DH-G2.
+                 *     - Huron:         70FF2/DH-G2.
+                 */
+                if (bModel < 0x10)
+                    return kCpumMicroarch_AMD_K8_130nm;
+                if (bModel >= 0x60 && bModel < 0x80)
+                    return kCpumMicroarch_AMD_K8_65nm;
+                if (bModel >= 0x40)
+                    return kCpumMicroarch_AMD_K8_90nm_AMDV;
+                switch (bModel)
+                {
+                    case 0x21:
+                    case 0x23:
+                    case 0x2b:
+                    case 0x2f:
+                    case 0x37:
+                    case 0x3f:
+                        return kCpumMicroarch_AMD_K8_90nm_DualCore;
+                }
+                return kCpumMicroarch_AMD_K8_90nm;
+            case 0x10:
+                return kCpumMicroarch_AMD_K10;
+            case 0x11:
+                return kCpumMicroarch_AMD_K10_Lion;
+            case 0x12:
+                return kCpumMicroarch_AMD_K10_Llano;
+            case 0x14:
+                return kCpumMicroarch_AMD_Bobcat;
+            case 0x15:
+                switch (bModel)
+                {
+                    case 0x00:  return kCpumMicroarch_AMD_15h_Bulldozer;    /* Any? prerelease? */
+                    case 0x01:  return kCpumMicroarch_AMD_15h_Bulldozer;    /* Opteron 4200, FX-81xx. */
+                    case 0x02:  return kCpumMicroarch_AMD_15h_Piledriver;   /* Opteron 4300, FX-83xx. */
+                    case 0x10:  return kCpumMicroarch_AMD_15h_Piledriver;   /* A10-5800K for e.g. */
+                    case 0x11:  /* ?? */
+                    case 0x12:  /* ?? */
+                    case 0x13:  return kCpumMicroarch_AMD_15h_Piledriver;   /* A10-6800K for e.g. */
+                }
+                return kCpumMicroarch_AMD_15h_Unknown;
+            case 0x16:
+                return kCpumMicroarch_AMD_Jaguar;
+            case 0x17:
+                return kCpumMicroarch_AMD_Zen_Ryzen;
+        }
+        return kCpumMicroarch_AMD_Unknown;
+    }
+
+    if (enmVendor == CPUMCPUVENDOR_INTEL)
+    {
+        switch (bFamily)
+        {
+            case 3:
+                return kCpumMicroarch_Intel_80386;
+            case 4:
+                return kCpumMicroarch_Intel_80486;
+            case 5:
+                return kCpumMicroarch_Intel_P5;
+            case 6:
+                if (bModel < RT_ELEMENTS(g_aenmIntelFamily06))
+                {
+                    CPUMMICROARCH enmMicroArch = g_aenmIntelFamily06[bModel];
+                    if (enmMicroArch == kCpumMicroarch_Intel_Core7_KabyLake)
+                    {
+                        if (bStepping >= 0xa && bStepping <= 0xc)
+                            enmMicroArch = kCpumMicroarch_Intel_Core7_CoffeeLake;
+                        else if (bStepping >= 0xc)
+                            enmMicroArch = kCpumMicroarch_Intel_Core7_WhiskeyLake;
+                    }
+                    else if (   enmMicroArch == kCpumMicroarch_Intel_Core7_Skylake
+                             && bModel == 0x55
+                             && bStepping >= 5)
+                        enmMicroArch = kCpumMicroarch_Intel_Core7_CascadeLake;
+                    return enmMicroArch;
+                }
+                return kCpumMicroarch_Intel_Atom_Unknown;
+            case 15:
+                switch (bModel)
+                {
+                    case 0:     return kCpumMicroarch_Intel_NB_Willamette;
+                    case 1:     return kCpumMicroarch_Intel_NB_Willamette;
+                    case 2:     return kCpumMicroarch_Intel_NB_Northwood;
+                    case 3:     return kCpumMicroarch_Intel_NB_Prescott;
+                    case 4:     return kCpumMicroarch_Intel_NB_Prescott2M; /* ?? */
+                    case 5:     return kCpumMicroarch_Intel_NB_Unknown; /*??*/
+                    case 6:     return kCpumMicroarch_Intel_NB_CedarMill;
+                    case 7:     return kCpumMicroarch_Intel_NB_Gallatin;
+                    default:    return kCpumMicroarch_Intel_NB_Unknown;
+                }
+                break;
+            /* The following are not kosher but kind of follow intuitively from 6, 5 & 4. */
+            case 0:
+                return kCpumMicroarch_Intel_8086;
+            case 1:
+                return kCpumMicroarch_Intel_80186;
+            case 2:
+                return kCpumMicroarch_Intel_80286;
+        }
+        return kCpumMicroarch_Intel_Unknown;
+    }
+
+    if (enmVendor == CPUMCPUVENDOR_VIA)
+    {
+        switch (bFamily)
+        {
+            case 5:
+                switch (bModel)
+                {
+                    case 1: return kCpumMicroarch_Centaur_C6;
+                    case 4: return kCpumMicroarch_Centaur_C6;
+                    case 8: return kCpumMicroarch_Centaur_C2;
+                    case 9: return kCpumMicroarch_Centaur_C3;
+                }
+                break;
+
+            case 6:
+                switch (bModel)
+                {
+                    case  5: return kCpumMicroarch_VIA_C3_M2;
+                    case  6: return kCpumMicroarch_VIA_C3_C5A;
+                    case  7: return bStepping < 8 ? kCpumMicroarch_VIA_C3_C5B : kCpumMicroarch_VIA_C3_C5C;
+                    case  8: return kCpumMicroarch_VIA_C3_C5N;
+                    case  9: return bStepping < 8 ? kCpumMicroarch_VIA_C3_C5XL : kCpumMicroarch_VIA_C3_C5P;
+                    case 10: return kCpumMicroarch_VIA_C7_C5J;
+                    case 15: return kCpumMicroarch_VIA_Isaiah;
+                }
+                break;
+        }
+        return kCpumMicroarch_VIA_Unknown;
+    }
+
+    if (enmVendor == CPUMCPUVENDOR_SHANGHAI)
+    {
+        switch (bFamily)
+        {
+            case 6:
+            case 7:
+                return kCpumMicroarch_Shanghai_Wudaokou;
+            default:
+                break;
+        }
+        return kCpumMicroarch_Shanghai_Unknown;
+    }
+
+    if (enmVendor == CPUMCPUVENDOR_CYRIX)
+    {
+        switch (bFamily)
+        {
+            case 4:
+                switch (bModel)
+                {
+                    case 9: return kCpumMicroarch_Cyrix_5x86;
+                }
+                break;
+
+            case 5:
+                switch (bModel)
+                {
+                    case 2: return kCpumMicroarch_Cyrix_M1;
+                    case 4: return kCpumMicroarch_Cyrix_MediaGX;
+                    case 5: return kCpumMicroarch_Cyrix_MediaGXm;
+                }
+                break;
+
+            case 6:
+                switch (bModel)
+                {
+                    case 0: return kCpumMicroarch_Cyrix_M2;
+                }
+                break;
+
+        }
+        return kCpumMicroarch_Cyrix_Unknown;
+    }
+
+    if (enmVendor == CPUMCPUVENDOR_HYGON)
+    {
+        switch (bFamily)
+        {
+            case 0x18:
+                return kCpumMicroarch_Hygon_Dhyana;
+            default:
+                break;
+        }
+        return kCpumMicroarch_Hygon_Unknown;
+    }
+
+    return kCpumMicroarch_Unknown;
+}
+
+
+/**
+ * Translates a microarchitecture enum value to the corresponding string
+ * constant.
+ *
+ * @returns Read-only string constant (omits "kCpumMicroarch_" prefix). Returns
+ *          NULL if the value is invalid.
+ *
+ * @param   enmMicroarch    The enum value to convert.
+ */
+VMMDECL(const char *) CPUMMicroarchName(CPUMMICROARCH enmMicroarch)
+{
+    switch (enmMicroarch)
+    {
+#define CASE_RET_STR(enmValue)  case enmValue: return #enmValue + (sizeof("kCpumMicroarch_") - 1)
+        CASE_RET_STR(kCpumMicroarch_Intel_8086);
+        CASE_RET_STR(kCpumMicroarch_Intel_80186);
+        CASE_RET_STR(kCpumMicroarch_Intel_80286);
+        CASE_RET_STR(kCpumMicroarch_Intel_80386);
+        CASE_RET_STR(kCpumMicroarch_Intel_80486);
+        CASE_RET_STR(kCpumMicroarch_Intel_P5);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_P6);
+        CASE_RET_STR(kCpumMicroarch_Intel_P6_II);
+        CASE_RET_STR(kCpumMicroarch_Intel_P6_III);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_P6_M_Banias);
+        CASE_RET_STR(kCpumMicroarch_Intel_P6_M_Dothan);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core_Yonah);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_Core2_Merom);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core2_Penryn);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_Nehalem);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_Westmere);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_SandyBridge);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_IvyBridge);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_Haswell);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_Broadwell);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_Skylake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_KabyLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_CoffeeLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_WhiskeyLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_CascadeLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_CannonLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_CometLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_IceLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_RocketLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_TigerLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_AlderLake);
+        CASE_RET_STR(kCpumMicroarch_Intel_Core7_SapphireRapids);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Bonnell);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Lincroft);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Saltwell);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Silvermont);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Airmount);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Goldmont);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_GoldmontPlus);
+        CASE_RET_STR(kCpumMicroarch_Intel_Atom_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsFerry);
+        CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsCorner);
+        CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsLanding);
+        CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsHill);
+        CASE_RET_STR(kCpumMicroarch_Intel_Phi_KnightsMill);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Willamette);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Northwood);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Prescott);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Prescott2M);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_CedarMill);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Gallatin);
+        CASE_RET_STR(kCpumMicroarch_Intel_NB_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Intel_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_Am286);
+        CASE_RET_STR(kCpumMicroarch_AMD_Am386);
+        CASE_RET_STR(kCpumMicroarch_AMD_Am486);
+        CASE_RET_STR(kCpumMicroarch_AMD_Am486Enh);
+        CASE_RET_STR(kCpumMicroarch_AMD_K5);
+        CASE_RET_STR(kCpumMicroarch_AMD_K6);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Palomino);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Spitfire);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Thunderbird);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Morgan);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Thoroughbred);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Barton);
+        CASE_RET_STR(kCpumMicroarch_AMD_K7_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_K8_130nm);
+        CASE_RET_STR(kCpumMicroarch_AMD_K8_90nm);
+        CASE_RET_STR(kCpumMicroarch_AMD_K8_90nm_DualCore);
+        CASE_RET_STR(kCpumMicroarch_AMD_K8_90nm_AMDV);
+        CASE_RET_STR(kCpumMicroarch_AMD_K8_65nm);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_K10);
+        CASE_RET_STR(kCpumMicroarch_AMD_K10_Lion);
+        CASE_RET_STR(kCpumMicroarch_AMD_K10_Llano);
+        CASE_RET_STR(kCpumMicroarch_AMD_Bobcat);
+        CASE_RET_STR(kCpumMicroarch_AMD_Jaguar);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_15h_Bulldozer);
+        CASE_RET_STR(kCpumMicroarch_AMD_15h_Piledriver);
+        CASE_RET_STR(kCpumMicroarch_AMD_15h_Steamroller);
+        CASE_RET_STR(kCpumMicroarch_AMD_15h_Excavator);
+        CASE_RET_STR(kCpumMicroarch_AMD_15h_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_16h_First);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_Zen_Ryzen);
+
+        CASE_RET_STR(kCpumMicroarch_AMD_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Hygon_Dhyana);
+        CASE_RET_STR(kCpumMicroarch_Hygon_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Centaur_C6);
+        CASE_RET_STR(kCpumMicroarch_Centaur_C2);
+        CASE_RET_STR(kCpumMicroarch_Centaur_C3);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_M2);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5A);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5B);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5C);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5N);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5XL);
+        CASE_RET_STR(kCpumMicroarch_VIA_C3_C5P);
+        CASE_RET_STR(kCpumMicroarch_VIA_C7_C5J);
+        CASE_RET_STR(kCpumMicroarch_VIA_Isaiah);
+        CASE_RET_STR(kCpumMicroarch_VIA_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Shanghai_Wudaokou);
+        CASE_RET_STR(kCpumMicroarch_Shanghai_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_Cyrix_5x86);
+        CASE_RET_STR(kCpumMicroarch_Cyrix_M1);
+        CASE_RET_STR(kCpumMicroarch_Cyrix_MediaGX);
+        CASE_RET_STR(kCpumMicroarch_Cyrix_MediaGXm);
+        CASE_RET_STR(kCpumMicroarch_Cyrix_M2);
+        CASE_RET_STR(kCpumMicroarch_Cyrix_Unknown);
+
+        CASE_RET_STR(kCpumMicroarch_NEC_V20);
+        CASE_RET_STR(kCpumMicroarch_NEC_V30);
+
+        CASE_RET_STR(kCpumMicroarch_Unknown);
+
+#undef CASE_RET_STR
+        case kCpumMicroarch_Invalid:
+        case kCpumMicroarch_Intel_End:
+        case kCpumMicroarch_Intel_Core2_End:
+        case kCpumMicroarch_Intel_Core7_End:
+        case kCpumMicroarch_Intel_Atom_End:
+        case kCpumMicroarch_Intel_P6_Core_Atom_End:
+        case kCpumMicroarch_Intel_Phi_End:
+        case kCpumMicroarch_Intel_NB_End:
+        case kCpumMicroarch_AMD_K7_End:
+        case kCpumMicroarch_AMD_K8_End:
+        case kCpumMicroarch_AMD_15h_End:
+        case kCpumMicroarch_AMD_16h_End:
+        case kCpumMicroarch_AMD_Zen_End:
+        case kCpumMicroarch_AMD_End:
+        case kCpumMicroarch_Hygon_End:
+        case kCpumMicroarch_VIA_End:
+        case kCpumMicroarch_Shanghai_End:
+        case kCpumMicroarch_Cyrix_End:
+        case kCpumMicroarch_NEC_End:
+        case kCpumMicroarch_32BitHack:
+            break;
+        /* no default! */
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Gets a matching leaf in the CPUID leaf array.
+ *
+ * @returns Pointer to the matching leaf, or NULL if not found.
+ * @param   paLeaves            The CPUID leaves to search.  This is sorted.
+ * @param   cLeaves             The number of leaves in the array.
+ * @param   uLeaf               The leaf to locate.
+ * @param   uSubLeaf            The subleaf to locate.  Pass 0 if no sub-leaves.
+ */
+PCPUMCPUIDLEAF cpumCpuIdGetLeafInt(PCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, uint32_t uLeaf, uint32_t uSubLeaf)
+{
+    /* Lazy bird does linear lookup here since this is only used for the
+       occational CPUID overrides. */
+    for (uint32_t i = 0; i < cLeaves; i++)
+        if (   paLeaves[i].uLeaf    == uLeaf
+            && paLeaves[i].uSubLeaf == (uSubLeaf & paLeaves[i].fSubLeafMask))
+            return &paLeaves[i];
+    return NULL;
+}
+
+
+/**
+ * Ensures that the CPUID leaf array can hold one more leaf.
+ *
+ * @returns Pointer to the CPUID leaf array (*ppaLeaves) on success.  NULL on
+ *          failure.
+ * @param   pVM         The cross context VM structure.  If NULL, use
+ *                      the process heap, otherwise the VM's hyper heap.
+ * @param   ppaLeaves   Pointer to the variable holding the array pointer
+ *                      (input/output).
+ * @param   cLeaves     The current array size.
+ *
+ * @remarks This function will automatically update the R0 and RC pointers when
+ *          using the hyper heap, which means @a ppaLeaves and @a cLeaves must
+ *          be the corresponding VM's CPUID arrays (which is asserted).
+ */
+PCPUMCPUIDLEAF cpumCpuIdEnsureSpace(PVM pVM, PCPUMCPUIDLEAF *ppaLeaves, uint32_t cLeaves)
+{
+    /*
+     * If pVM is not specified, we're on the regular heap and can waste a
+     * little space to speed things up.
+     */
+    uint32_t cAllocated;
+    if (!pVM)
+    {
+        cAllocated = RT_ALIGN(cLeaves, 16);
+        if (cLeaves + 1 > cAllocated)
+        {
+            void *pvNew = RTMemRealloc(*ppaLeaves, (cAllocated + 16) * sizeof(**ppaLeaves));
+            if (pvNew)
+                *ppaLeaves = (PCPUMCPUIDLEAF)pvNew;
+            else
+            {
+                RTMemFree(*ppaLeaves);
+                *ppaLeaves = NULL;
+            }
+        }
+    }
+    /*
+     * Otherwise, we're on the hyper heap and are probably just inserting
+     * one or two leaves and should conserve space.
+     */
+    else
+    {
+#ifdef IN_VBOX_CPU_REPORT
+        AssertReleaseFailed();
+#else
+# ifdef IN_RING3
+        Assert(ppaLeaves == &pVM->cpum.s.GuestInfo.paCpuIdLeavesR3);
+        Assert(*ppaLeaves == pVM->cpum.s.GuestInfo.aCpuIdLeaves);
+        Assert(cLeaves == pVM->cpum.s.GuestInfo.cCpuIdLeaves);
+
+        if (cLeaves + 1 <= RT_ELEMENTS(pVM->cpum.s.GuestInfo.aCpuIdLeaves))
+        { }
+        else
+# endif
+        {
+            *ppaLeaves = NULL;
+            LogRel(("CPUM: cpumR3CpuIdEnsureSpace: Out of CPUID space!\n"));
+        }
+#endif
+    }
+    return *ppaLeaves;
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * Checks that we've updated the CPUID leaves array correctly.
+ *
+ * This is a no-op in non-strict builds.
+ *
+ * @param   paLeaves            The leaves array.
+ * @param   cLeaves             The number of leaves.
+ */
+void cpumCpuIdAssertOrder(PCPUMCPUIDLEAF paLeaves, uint32_t cLeaves)
+{
+    for (uint32_t i = 1; i < cLeaves; i++)
+        if (paLeaves[i].uLeaf != paLeaves[i - 1].uLeaf)
+            AssertMsg(paLeaves[i].uLeaf > paLeaves[i - 1].uLeaf, ("%#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i - 1].uLeaf));
+        else
+        {
+            AssertMsg(paLeaves[i].uSubLeaf > paLeaves[i - 1].uSubLeaf,
+                      ("%#x: %#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, paLeaves[i - 1].uSubLeaf));
+            AssertMsg(paLeaves[i].fSubLeafMask == paLeaves[i - 1].fSubLeafMask,
+                      ("%#x/%#x: %#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, paLeaves[i].fSubLeafMask, paLeaves[i - 1].fSubLeafMask));
+            AssertMsg(paLeaves[i].fFlags == paLeaves[i - 1].fFlags,
+                      ("%#x/%#x: %#x vs %#x\n", paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, paLeaves[i].fFlags, paLeaves[i - 1].fFlags));
+        }
+}
+#endif
+
+#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
+
+/**
+ * Append a CPUID leaf or sub-leaf.
+ *
+ * ASSUMES linear insertion order, so we'll won't need to do any searching or
+ * replace anything.  Use cpumR3CpuIdInsert() for those cases.
+ *
+ * @returns VINF_SUCCESS or VERR_NO_MEMORY.  On error, *ppaLeaves is freed, so
+ *          the caller need do no more work.
+ * @param   ppaLeaves       Pointer to the pointer to the array of sorted
+ *                          CPUID leaves and sub-leaves.
+ * @param   pcLeaves        Where we keep the leaf count for *ppaLeaves.
+ * @param   uLeaf           The leaf we're adding.
+ * @param   uSubLeaf        The sub-leaf number.
+ * @param   fSubLeafMask    The sub-leaf mask.
+ * @param   uEax            The EAX value.
+ * @param   uEbx            The EBX value.
+ * @param   uEcx            The ECX value.
+ * @param   uEdx            The EDX value.
+ * @param   fFlags          The flags.
+ */
+static int cpumCollectCpuIdInfoAddOne(PCPUMCPUIDLEAF *ppaLeaves, uint32_t *pcLeaves,
+                                      uint32_t uLeaf, uint32_t uSubLeaf, uint32_t fSubLeafMask,
+                                      uint32_t uEax, uint32_t uEbx, uint32_t uEcx, uint32_t uEdx, uint32_t fFlags)
+{
+    if (!cpumCpuIdEnsureSpace(NULL /* pVM */, ppaLeaves, *pcLeaves))
+        return VERR_NO_MEMORY;
+
+    PCPUMCPUIDLEAF pNew = &(*ppaLeaves)[*pcLeaves];
+    Assert(   *pcLeaves == 0
+           || pNew[-1].uLeaf < uLeaf
+           || (pNew[-1].uLeaf == uLeaf && pNew[-1].uSubLeaf < uSubLeaf) );
+
+    pNew->uLeaf        = uLeaf;
+    pNew->uSubLeaf     = uSubLeaf;
+    pNew->fSubLeafMask = fSubLeafMask;
+    pNew->uEax         = uEax;
+    pNew->uEbx         = uEbx;
+    pNew->uEcx         = uEcx;
+    pNew->uEdx         = uEdx;
+    pNew->fFlags       = fFlags;
+
+    *pcLeaves += 1;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if ECX make a difference when reading a given CPUID leaf.
+ *
+ * @returns @c true if it does, @c false if it doesn't.
+ * @param   uLeaf               The leaf we're reading.
+ * @param   pcSubLeaves         Number of sub-leaves accessible via ECX.
+ * @param   pfFinalEcxUnchanged Whether ECX is passed thru when going beyond the
+ *                              final sub-leaf (for leaf 0xb only).
+ */
+static bool cpumIsEcxRelevantForCpuIdLeaf(uint32_t uLeaf, uint32_t *pcSubLeaves, bool *pfFinalEcxUnchanged)
+{
+    *pfFinalEcxUnchanged = false;
+
+    uint32_t auCur[4];
+    uint32_t auPrev[4];
+    ASMCpuIdExSlow(uLeaf, 0, 0, 0, &auPrev[0], &auPrev[1], &auPrev[2], &auPrev[3]);
+
+    /* Look for sub-leaves. */
+    uint32_t uSubLeaf = 1;
+    for (;;)
+    {
+        ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &auCur[0], &auCur[1], &auCur[2], &auCur[3]);
+        if (memcmp(auCur, auPrev, sizeof(auCur)))
+            break;
+
+        /* Advance / give up. */
+        uSubLeaf++;
+        if (uSubLeaf >= 64)
+        {
+            *pcSubLeaves = 1;
+            return false;
+        }
+    }
+
+    /* Count sub-leaves. */
+    uint32_t cMinLeaves = uLeaf == 0xd ? 64 : 0;
+    uint32_t cRepeats = 0;
+    uSubLeaf = 0;
+    for (;;)
+    {
+        ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &auCur[0], &auCur[1], &auCur[2], &auCur[3]);
+
+        /* Figuring out when to stop isn't entirely straight forward as we need
+           to cover undocumented behavior up to a point and implementation shortcuts. */
+
+        /* 1. Look for more than 4 repeating value sets. */
+        if (   auCur[0] == auPrev[0]
+            && auCur[1] == auPrev[1]
+            && (    auCur[2] == auPrev[2]
+                || (   auCur[2]  == uSubLeaf
+                    && auPrev[2] == uSubLeaf - 1) )
+            && auCur[3] == auPrev[3])
+        {
+            if (   uLeaf != 0xd
+                || uSubLeaf >= 64
+                || (   auCur[0] == 0
+                    && auCur[1] == 0
+                    && auCur[2] == 0
+                    && auCur[3] == 0
+                    && auPrev[2] == 0) )
+                cRepeats++;
+            if (cRepeats > 4 && uSubLeaf >= cMinLeaves)
+                break;
+        }
+        else
+            cRepeats = 0;
+
+        /* 2. Look for zero values. */
+        if (   auCur[0] == 0
+            && auCur[1] == 0
+            && (auCur[2] == 0 || auCur[2] == uSubLeaf)
+            && (auCur[3] == 0 || uLeaf == 0xb /* edx is fixed */)
+            && uSubLeaf >= cMinLeaves)
+        {
+            cRepeats = 0;
+            break;
+        }
+
+        /* 3. Leaf 0xb level type 0 check. */
+        if (   uLeaf == 0xb
+            && (auCur[2]  & 0xff00) == 0
+            && (auPrev[2] & 0xff00) == 0)
+        {
+            cRepeats = 0;
+            break;
+        }
+
+        /* 99. Give up. */
+        if (uSubLeaf >= 128)
+        {
+# ifndef IN_VBOX_CPU_REPORT
+            /* Ok, limit it according to the documentation if possible just to
+               avoid annoying users with these detection issues. */
+            uint32_t cDocLimit = UINT32_MAX;
+            if (uLeaf == 0x4)
+                cDocLimit = 4;
+            else if (uLeaf == 0x7)
+                cDocLimit = 1;
+            else if (uLeaf == 0xd)
+                cDocLimit = 63;
+            else if (uLeaf == 0xf)
+                cDocLimit = 2;
+            if (cDocLimit != UINT32_MAX)
+            {
+                *pfFinalEcxUnchanged = auCur[2] == uSubLeaf && uLeaf == 0xb;
+                *pcSubLeaves = cDocLimit + 3;
+                return true;
+            }
+# endif
+            *pcSubLeaves = UINT32_MAX;
+            return true;
+        }
+
+        /* Advance. */
+        uSubLeaf++;
+        memcpy(auPrev, auCur, sizeof(auCur));
+    }
+
+    /* Standard exit. */
+    *pfFinalEcxUnchanged = auCur[2] == uSubLeaf && uLeaf == 0xb;
+    *pcSubLeaves = uSubLeaf + 1 - cRepeats;
+    if (*pcSubLeaves == 0)
+        *pcSubLeaves = 1;
+    return true;
+}
+
+
+/**
+ * Collects CPUID leaves and sub-leaves, returning a sorted array of them.
+ *
+ * @returns VBox status code.
+ * @param   ppaLeaves           Where to return the array pointer on success.
+ *                              Use RTMemFree to release.
+ * @param   pcLeaves            Where to return the size of the array on
+ *                              success.
+ */
+VMMDECL(int) CPUMCpuIdCollectLeavesX86(PCPUMCPUIDLEAF *ppaLeaves, uint32_t *pcLeaves)
+{
+    *ppaLeaves = NULL;
+    *pcLeaves = 0;
+
+    /*
+     * Try out various candidates. This must be sorted!
+     */
+    static struct { uint32_t uMsr; bool fSpecial; } const s_aCandidates[] =
+    {
+        { UINT32_C(0x00000000), false },
+        { UINT32_C(0x10000000), false },
+        { UINT32_C(0x20000000), false },
+        { UINT32_C(0x30000000), false },
+        { UINT32_C(0x40000000), false },
+        { UINT32_C(0x50000000), false },
+        { UINT32_C(0x60000000), false },
+        { UINT32_C(0x70000000), false },
+        { UINT32_C(0x80000000), false },
+        { UINT32_C(0x80860000), false },
+        { UINT32_C(0x8ffffffe), true  },
+        { UINT32_C(0x8fffffff), true  },
+        { UINT32_C(0x90000000), false },
+        { UINT32_C(0xa0000000), false },
+        { UINT32_C(0xb0000000), false },
+        { UINT32_C(0xc0000000), false },
+        { UINT32_C(0xd0000000), false },
+        { UINT32_C(0xe0000000), false },
+        { UINT32_C(0xf0000000), false },
+    };
+
+    for (uint32_t iOuter = 0; iOuter < RT_ELEMENTS(s_aCandidates); iOuter++)
+    {
+        uint32_t uLeaf = s_aCandidates[iOuter].uMsr;
+        uint32_t uEax, uEbx, uEcx, uEdx;
+        ASMCpuIdExSlow(uLeaf, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+
+        /*
+         * Does EAX look like a typical leaf count value?
+         */
+        if (   uEax         > uLeaf
+            && uEax - uLeaf < UINT32_C(0xff)) /* Adjust 0xff limit when exceeded by real HW. */
+        {
+            /* Yes, dump them. */
+            uint32_t cLeaves = uEax - uLeaf + 1;
+            while (cLeaves-- > 0)
+            {
+                ASMCpuIdExSlow(uLeaf, 0, 0, 0, &uEax, &uEbx, &uEcx, &uEdx);
+
+                uint32_t fFlags = 0;
+
+                /* There are currently three known leaves containing an APIC ID
+                   that needs EMT specific attention */
+                if (uLeaf == 1)
+                    fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC_ID;
+                else if (uLeaf == 0xb && uEcx != 0)
+                    fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC_ID;
+                else if (   uLeaf == UINT32_C(0x8000001e)
+                         && (   uEax
+                             || uEbx
+                             || uEdx
+                             || RTX86IsAmdCpu((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx)
+                             || RTX86IsHygonCpu((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx)) )
+                    fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC_ID;
+
+                /* The APIC bit is per-VCpu and needs flagging. */
+                if (uLeaf == 1)
+                    fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC;
+                else if (   uLeaf == UINT32_C(0x80000001)
+                         && (   (uEdx & X86_CPUID_AMD_FEATURE_EDX_APIC)
+                             || RTX86IsAmdCpu((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx)
+                             || RTX86IsHygonCpu((*ppaLeaves)[0].uEbx, (*ppaLeaves)[0].uEcx, (*ppaLeaves)[0].uEdx)) )
+                    fFlags |= CPUMCPUIDLEAF_F_CONTAINS_APIC;
+
+                /* Check three times here to reduce the chance of CPU migration
+                   resulting in false positives with things like the APIC ID. */
+                uint32_t cSubLeaves;
+                bool fFinalEcxUnchanged;
+                if (   cpumIsEcxRelevantForCpuIdLeaf(uLeaf, &cSubLeaves, &fFinalEcxUnchanged)
+                    && cpumIsEcxRelevantForCpuIdLeaf(uLeaf, &cSubLeaves, &fFinalEcxUnchanged)
+                    && cpumIsEcxRelevantForCpuIdLeaf(uLeaf, &cSubLeaves, &fFinalEcxUnchanged))
+                {
+                    if (cSubLeaves > (uLeaf == 0xd ? 68U : 16U))
+                    {
+                        /* This shouldn't happen.  But in case it does, file all
+                           relevant details in the release log. */
+                        LogRel(("CPUM: VERR_CPUM_TOO_MANY_CPUID_SUBLEAVES! uLeaf=%#x cSubLeaves=%#x\n", uLeaf, cSubLeaves));
+                        LogRel(("------------------ dump of problematic sub-leaves -----------------\n"));
+                        for (uint32_t uSubLeaf = 0; uSubLeaf < 128; uSubLeaf++)
+                        {
+                            uint32_t auTmp[4];
+                            ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &auTmp[0], &auTmp[1], &auTmp[2], &auTmp[3]);
+                            LogRel(("CPUM: %#010x, %#010x => %#010x %#010x %#010x %#010x\n",
+                                    uLeaf, uSubLeaf, auTmp[0], auTmp[1], auTmp[2], auTmp[3]));
+                        }
+                        LogRel(("----------------- dump of what we've found so far -----------------\n"));
+                        for (uint32_t i = 0 ; i < *pcLeaves; i++)
+                            LogRel(("CPUM: %#010x, %#010x/%#010x => %#010x %#010x %#010x %#010x\n",
+                                    (*ppaLeaves)[i].uLeaf, (*ppaLeaves)[i].uSubLeaf,  (*ppaLeaves)[i].fSubLeafMask,
+                                    (*ppaLeaves)[i].uEax, (*ppaLeaves)[i].uEbx, (*ppaLeaves)[i].uEcx, (*ppaLeaves)[i].uEdx));
+                        LogRel(("\nPlease create a defect on virtualbox.org and attach this log file!\n\n"));
+                        return VERR_CPUM_TOO_MANY_CPUID_SUBLEAVES;
+                    }
+
+                    if (fFinalEcxUnchanged)
+                        fFlags |= CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES;
+
+                    for (uint32_t uSubLeaf = 0; uSubLeaf < cSubLeaves; uSubLeaf++)
+                    {
+                        ASMCpuIdExSlow(uLeaf, 0, uSubLeaf, 0, &uEax, &uEbx, &uEcx, &uEdx);
+                        int rc = cpumCollectCpuIdInfoAddOne(ppaLeaves, pcLeaves,
+                                                            uLeaf, uSubLeaf, UINT32_MAX, uEax, uEbx, uEcx, uEdx, fFlags);
+                        if (RT_FAILURE(rc))
+                            return rc;
+                    }
+                }
+                else
+                {
+                    int rc = cpumCollectCpuIdInfoAddOne(ppaLeaves, pcLeaves, uLeaf, 0, 0, uEax, uEbx, uEcx, uEdx, fFlags);
+                    if (RT_FAILURE(rc))
+                        return rc;
+                }
+
+                /* next */
+                uLeaf++;
+            }
+        }
+        /*
+         * Special CPUIDs needs special handling as they don't follow the
+         * leaf count principle used above.
+         */
+        else if (s_aCandidates[iOuter].fSpecial)
+        {
+            bool fKeep = false;
+            if (uLeaf == 0x8ffffffe && uEax == UINT32_C(0x00494544))
+                fKeep = true;
+            else if (   uLeaf == 0x8fffffff
+                     && RT_C_IS_PRINT(RT_BYTE1(uEax))
+                     && RT_C_IS_PRINT(RT_BYTE2(uEax))
+                     && RT_C_IS_PRINT(RT_BYTE3(uEax))
+                     && RT_C_IS_PRINT(RT_BYTE4(uEax))
+                     && RT_C_IS_PRINT(RT_BYTE1(uEbx))
+                     && RT_C_IS_PRINT(RT_BYTE2(uEbx))
+                     && RT_C_IS_PRINT(RT_BYTE3(uEbx))
+                     && RT_C_IS_PRINT(RT_BYTE4(uEbx))
+                     && RT_C_IS_PRINT(RT_BYTE1(uEcx))
+                     && RT_C_IS_PRINT(RT_BYTE2(uEcx))
+                     && RT_C_IS_PRINT(RT_BYTE3(uEcx))
+                     && RT_C_IS_PRINT(RT_BYTE4(uEcx))
+                     && RT_C_IS_PRINT(RT_BYTE1(uEdx))
+                     && RT_C_IS_PRINT(RT_BYTE2(uEdx))
+                     && RT_C_IS_PRINT(RT_BYTE3(uEdx))
+                     && RT_C_IS_PRINT(RT_BYTE4(uEdx)) )
+                fKeep = true;
+            if (fKeep)
+            {
+                int rc = cpumCollectCpuIdInfoAddOne(ppaLeaves, pcLeaves, uLeaf, 0, 0, uEax, uEbx, uEcx, uEdx, 0);
+                if (RT_FAILURE(rc))
+                    return rc;
+            }
+        }
+    }
+
+# ifdef VBOX_STRICT
+    cpumCpuIdAssertOrder(*ppaLeaves, *pcLeaves);
+# endif
+    return VINF_SUCCESS;
+}
+#endif /* RT_ARCH_X86 || RT_ARCH_AMD64 */
+
+
+/**
+ * Detect the CPU vendor give n the
+ *
+ * @returns The vendor.
+ * @param   uEAX                EAX from CPUID(0).
+ * @param   uEBX                EBX from CPUID(0).
+ * @param   uECX                ECX from CPUID(0).
+ * @param   uEDX                EDX from CPUID(0).
+ */
+VMMDECL(CPUMCPUVENDOR) CPUMCpuIdDetectX86VendorEx(uint32_t uEAX, uint32_t uEBX, uint32_t uECX, uint32_t uEDX)
+{
+    if (RTX86IsValidStdRange(uEAX))
+    {
+        if (RTX86IsAmdCpu(uEBX, uECX, uEDX))
+            return CPUMCPUVENDOR_AMD;
+
+        if (RTX86IsIntelCpu(uEBX, uECX, uEDX))
+            return CPUMCPUVENDOR_INTEL;
+
+        if (RTX86IsViaCentaurCpu(uEBX, uECX, uEDX))
+            return CPUMCPUVENDOR_VIA;
+
+        if (RTX86IsShanghaiCpu(uEBX, uECX, uEDX))
+            return CPUMCPUVENDOR_SHANGHAI;
+
+        if (   uEBX == UINT32_C(0x69727943) /* CyrixInstead */
+            && uECX == UINT32_C(0x64616574)
+            && uEDX == UINT32_C(0x736E4978))
+            return CPUMCPUVENDOR_CYRIX;
+
+        if (RTX86IsHygonCpu(uEBX, uECX, uEDX))
+            return CPUMCPUVENDOR_HYGON;
+
+        /* "Geode by NSC", example: family 5, model 9.  */
+
+        /** @todo detect the other buggers... */
+    }
+
+    return CPUMCPUVENDOR_UNKNOWN;
+}
+
+
+/**
+ * Translates a CPU vendor enum value into the corresponding string constant.
+ *
+ * The named can be prefixed with 'CPUMCPUVENDOR_' to construct a valid enum
+ * value name.  This can be useful when generating code.
+ *
+ * @returns Read only name string.
+ * @param   enmVendor           The CPU vendor value.
+ */
+VMMDECL(const char *) CPUMCpuVendorName(CPUMCPUVENDOR enmVendor)
+{
+    switch (enmVendor)
+    {
+        case CPUMCPUVENDOR_INTEL:       return "INTEL";
+        case CPUMCPUVENDOR_AMD:         return "AMD";
+        case CPUMCPUVENDOR_VIA:         return "VIA";
+        case CPUMCPUVENDOR_CYRIX:       return "CYRIX";
+        case CPUMCPUVENDOR_SHANGHAI:    return "SHANGHAI";
+        case CPUMCPUVENDOR_HYGON:       return "HYGON";
+        case CPUMCPUVENDOR_UNKNOWN:     return "UNKNOWN";
+
+        case CPUMCPUVENDOR_INVALID:
+        case CPUMCPUVENDOR_32BIT_HACK:
+            break;
+    }
+    return "Invalid-cpu-vendor";
+}
+
+
+static PCCPUMCPUIDLEAF cpumCpuIdFindLeaf(PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, uint32_t uLeaf)
+{
+    /* Could do binary search, doing linear now because I'm lazy. */
+    PCCPUMCPUIDLEAF pLeaf = paLeaves;
+    while (cLeaves-- > 0)
+    {
+        if (pLeaf->uLeaf == uLeaf)
+            return pLeaf;
+        pLeaf++;
+    }
+    return NULL;
+}
+
+
+static PCCPUMCPUIDLEAF cpumCpuIdFindLeafEx(PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, uint32_t uLeaf, uint32_t uSubLeaf)
+{
+    PCCPUMCPUIDLEAF pLeaf = cpumCpuIdFindLeaf(paLeaves, cLeaves, uLeaf);
+    if (   !pLeaf
+        || pLeaf->uSubLeaf != (uSubLeaf & pLeaf->fSubLeafMask))
+        return pLeaf;
+
+    /* Linear sub-leaf search. Lazy as usual. */
+    cLeaves -= pLeaf - paLeaves;
+    while (   cLeaves-- > 0
+           && pLeaf->uLeaf == uLeaf)
+    {
+        if (pLeaf->uSubLeaf == (uSubLeaf & pLeaf->fSubLeafMask))
+            return pLeaf;
+        pLeaf++;
+    }
+
+    return NULL;
+}
+
+
+static void cpumExplodeVmxFeatures(PCVMXMSRS pVmxMsrs, PCPUMFEATURES pFeatures)
+{
+    Assert(pVmxMsrs);
+    Assert(pFeatures);
+    Assert(pFeatures->fVmx);
+
+    /* Basic information. */
+    bool const fVmxTrueMsrs = RT_BOOL(pVmxMsrs->u64Basic & VMX_BF_BASIC_TRUE_CTLS_MASK);
+    {
+        uint64_t const u64Basic = pVmxMsrs->u64Basic;
+        pFeatures->fVmxInsOutInfo            = RT_BF_GET(u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
+    }
+
+    /* Pin-based VM-execution controls. */
+    {
+        uint32_t const fPinCtls = fVmxTrueMsrs ? pVmxMsrs->TruePinCtls.n.allowed1 : pVmxMsrs->PinCtls.n.allowed1;
+        pFeatures->fVmxExtIntExit            = RT_BOOL(fPinCtls & VMX_PIN_CTLS_EXT_INT_EXIT);
+        pFeatures->fVmxNmiExit               = RT_BOOL(fPinCtls & VMX_PIN_CTLS_NMI_EXIT);
+        pFeatures->fVmxVirtNmi               = RT_BOOL(fPinCtls & VMX_PIN_CTLS_VIRT_NMI);
+        pFeatures->fVmxPreemptTimer          = RT_BOOL(fPinCtls & VMX_PIN_CTLS_PREEMPT_TIMER);
+        pFeatures->fVmxPostedInt             = RT_BOOL(fPinCtls & VMX_PIN_CTLS_POSTED_INT);
+    }
+
+    /* Processor-based VM-execution controls. */
+    {
+        uint32_t const fProcCtls = fVmxTrueMsrs ? pVmxMsrs->TrueProcCtls.n.allowed1 : pVmxMsrs->ProcCtls.n.allowed1;
+        pFeatures->fVmxIntWindowExit         = RT_BOOL(fProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
+        pFeatures->fVmxTscOffsetting         = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING);
+        pFeatures->fVmxHltExit               = RT_BOOL(fProcCtls & VMX_PROC_CTLS_HLT_EXIT);
+        pFeatures->fVmxInvlpgExit            = RT_BOOL(fProcCtls & VMX_PROC_CTLS_INVLPG_EXIT);
+        pFeatures->fVmxMwaitExit             = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MWAIT_EXIT);
+        pFeatures->fVmxRdpmcExit             = RT_BOOL(fProcCtls & VMX_PROC_CTLS_RDPMC_EXIT);
+        pFeatures->fVmxRdtscExit             = RT_BOOL(fProcCtls & VMX_PROC_CTLS_RDTSC_EXIT);
+        pFeatures->fVmxCr3LoadExit           = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR3_LOAD_EXIT);
+        pFeatures->fVmxCr3StoreExit          = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR3_STORE_EXIT);
+        pFeatures->fVmxTertiaryExecCtls      = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS);
+        pFeatures->fVmxCr8LoadExit           = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR8_LOAD_EXIT);
+        pFeatures->fVmxCr8StoreExit          = RT_BOOL(fProcCtls & VMX_PROC_CTLS_CR8_STORE_EXIT);
+        pFeatures->fVmxUseTprShadow          = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+        pFeatures->fVmxNmiWindowExit         = RT_BOOL(fProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
+        pFeatures->fVmxMovDRxExit            = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
+        pFeatures->fVmxUncondIoExit          = RT_BOOL(fProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
+        pFeatures->fVmxUseIoBitmaps          = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS);
+        pFeatures->fVmxMonitorTrapFlag       = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
+        pFeatures->fVmxUseMsrBitmaps         = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
+        pFeatures->fVmxMonitorExit           = RT_BOOL(fProcCtls & VMX_PROC_CTLS_MONITOR_EXIT);
+        pFeatures->fVmxPauseExit             = RT_BOOL(fProcCtls & VMX_PROC_CTLS_PAUSE_EXIT);
+        pFeatures->fVmxSecondaryExecCtls     = RT_BOOL(fProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
+    }
+
+    /* Secondary processor-based VM-execution controls. */
+    {
+        uint32_t const fProcCtls2 = pFeatures->fVmxSecondaryExecCtls ? pVmxMsrs->ProcCtls2.n.allowed1 : 0;
+        pFeatures->fVmxVirtApicAccess        = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
+        pFeatures->fVmxEpt                   = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_EPT);
+        pFeatures->fVmxDescTableExit         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT);
+        pFeatures->fVmxRdtscp                = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_RDTSCP);
+        pFeatures->fVmxVirtX2ApicMode        = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE);
+        pFeatures->fVmxVpid                  = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VPID);
+        pFeatures->fVmxWbinvdExit            = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_WBINVD_EXIT);
+        pFeatures->fVmxUnrestrictedGuest     = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
+        pFeatures->fVmxApicRegVirt           = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT);
+        pFeatures->fVmxVirtIntDelivery       = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+        pFeatures->fVmxPauseLoopExit         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
+        pFeatures->fVmxRdrandExit            = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT);
+        pFeatures->fVmxInvpcid               = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_INVPCID);
+        pFeatures->fVmxVmFunc                = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VMFUNC);
+        pFeatures->fVmxVmcsShadowing         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING);
+        pFeatures->fVmxRdseedExit            = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_RDSEED_EXIT);
+        pFeatures->fVmxPml                   = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_PML);
+        pFeatures->fVmxEptXcptVe             = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_EPT_XCPT_VE);
+        pFeatures->fVmxConcealVmxFromPt      = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_CONCEAL_VMX_FROM_PT);
+        pFeatures->fVmxXsavesXrstors         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_XSAVES_XRSTORS);
+        pFeatures->fVmxModeBasedExecuteEpt   = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_MODE_BASED_EPT_PERM);
+        pFeatures->fVmxSppEpt                = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_SPP_EPT);
+        pFeatures->fVmxPtEpt                 = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_PT_EPT);
+        pFeatures->fVmxUseTscScaling         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_TSC_SCALING);
+        pFeatures->fVmxUserWaitPause         = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_USER_WAIT_PAUSE);
+        pFeatures->fVmxEnclvExit             = RT_BOOL(fProcCtls2 & VMX_PROC_CTLS2_ENCLV_EXIT);
+    }
+
+    /* Tertiary processor-based VM-execution controls. */
+    {
+        uint64_t const fProcCtls3 = pFeatures->fVmxTertiaryExecCtls ? pVmxMsrs->u64ProcCtls3 : 0;
+        pFeatures->fVmxLoadIwKeyExit         = RT_BOOL(fProcCtls3 & VMX_PROC_CTLS3_LOADIWKEY_EXIT);
+    }
+
+    /* VM-exit controls. */
+    {
+        uint32_t const fExitCtls = fVmxTrueMsrs ? pVmxMsrs->TrueExitCtls.n.allowed1 : pVmxMsrs->ExitCtls.n.allowed1;
+        pFeatures->fVmxExitSaveDebugCtls     = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_DEBUG);
+        pFeatures->fVmxHostAddrSpaceSize     = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+        pFeatures->fVmxExitAckExtInt         = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT);
+        pFeatures->fVmxExitSavePatMsr        = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_PAT_MSR);
+        pFeatures->fVmxExitLoadPatMsr        = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_LOAD_PAT_MSR);
+        pFeatures->fVmxExitSaveEferMsr       = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_EFER_MSR);
+        pFeatures->fVmxExitLoadEferMsr       = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR);
+        pFeatures->fVmxSavePreemptTimer      = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
+        pFeatures->fVmxSecondaryExitCtls     = RT_BOOL(fExitCtls & VMX_EXIT_CTLS_USE_SECONDARY_CTLS);
+    }
+
+    /* VM-entry controls. */
+    {
+        uint32_t const fEntryCtls = fVmxTrueMsrs ? pVmxMsrs->TrueEntryCtls.n.allowed1 : pVmxMsrs->EntryCtls.n.allowed1;
+        pFeatures->fVmxEntryLoadDebugCtls    = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG);
+        pFeatures->fVmxIa32eModeGuest        = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+        pFeatures->fVmxEntryLoadEferMsr      = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR);
+        pFeatures->fVmxEntryLoadPatMsr       = RT_BOOL(fEntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR);
+    }
+
+    /* Miscellaneous data. */
+    {
+        uint32_t const fMiscData = pVmxMsrs->u64Misc;
+        pFeatures->fVmxExitSaveEferLma       = RT_BOOL(fMiscData & VMX_MISC_EXIT_SAVE_EFER_LMA);
+        pFeatures->fVmxPt                    = RT_BOOL(fMiscData & VMX_MISC_INTEL_PT);
+        pFeatures->fVmxVmwriteAll            = RT_BOOL(fMiscData & VMX_MISC_VMWRITE_ALL);
+        pFeatures->fVmxEntryInjectSoftInt    = RT_BOOL(fMiscData & VMX_MISC_ENTRY_INJECT_SOFT_INT);
+    }
+}
+
+
+int cpumCpuIdExplodeFeaturesX86(PCCPUMCPUIDLEAF paLeaves, uint32_t cLeaves, PCCPUMMSRS pMsrs, PCPUMFEATURES pFeatures)
+{
+    Assert(pMsrs);
+    RT_ZERO(*pFeatures);
+    if (cLeaves >= 2)
+    {
+        AssertLogRelReturn(paLeaves[0].uLeaf == 0, VERR_CPUM_IPE_1);
+        AssertLogRelReturn(paLeaves[1].uLeaf == 1, VERR_CPUM_IPE_1);
+        PCCPUMCPUIDLEAF const pStd0Leaf = cpumCpuIdFindLeafEx(paLeaves, cLeaves, 0, 0);
+        AssertLogRelReturn(pStd0Leaf, VERR_CPUM_IPE_1);
+        PCCPUMCPUIDLEAF const pStd1Leaf = cpumCpuIdFindLeafEx(paLeaves, cLeaves, 1, 0);
+        AssertLogRelReturn(pStd1Leaf, VERR_CPUM_IPE_1);
+
+        pFeatures->enmCpuVendor = CPUMCpuIdDetectX86VendorEx(pStd0Leaf->uEax,
+                                                             pStd0Leaf->uEbx,
+                                                             pStd0Leaf->uEcx,
+                                                             pStd0Leaf->uEdx);
+        pFeatures->uFamily      = RTX86GetCpuFamily(pStd1Leaf->uEax);
+        pFeatures->uModel       = RTX86GetCpuModel(pStd1Leaf->uEax, pFeatures->enmCpuVendor == CPUMCPUVENDOR_INTEL);
+        pFeatures->uStepping    = RTX86GetCpuStepping(pStd1Leaf->uEax);
+        pFeatures->enmMicroarch = CPUMCpuIdDetermineX86MicroarchEx((CPUMCPUVENDOR)pFeatures->enmCpuVendor,
+                                                                   pFeatures->uFamily,
+                                                                   pFeatures->uModel,
+                                                                   pFeatures->uStepping);
+
+        PCCPUMCPUIDLEAF const pExtLeaf8 = cpumCpuIdFindLeaf(paLeaves, cLeaves, 0x80000008);
+        if (pExtLeaf8)
+        {
+            pFeatures->cMaxPhysAddrWidth   = pExtLeaf8->uEax & 0xff;
+            pFeatures->cMaxLinearAddrWidth = (pExtLeaf8->uEax >> 8) & 0xff;
+        }
+        else if (pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PSE36)
+        {
+            pFeatures->cMaxPhysAddrWidth   = 36;
+            pFeatures->cMaxLinearAddrWidth = 36;
+        }
+        else
+        {
+            pFeatures->cMaxPhysAddrWidth   = 32;
+            pFeatures->cMaxLinearAddrWidth = 32;
+        }
+
+        /* Standard features. */
+        pFeatures->fMsr                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_MSR);
+        pFeatures->fApic                = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_APIC);
+        pFeatures->fX2Apic              = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_X2APIC);
+        pFeatures->fPse                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PSE);
+        pFeatures->fPse36               = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PSE36);
+        pFeatures->fPae                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PAE);
+        pFeatures->fPge                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PGE);
+        pFeatures->fPat                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_PAT);
+        pFeatures->fFxSaveRstor         = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_FXSR);
+        pFeatures->fXSaveRstor          = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_XSAVE);
+        pFeatures->fOpSysXSaveRstor     = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_OSXSAVE);
+        pFeatures->fMmx                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_MMX);
+        pFeatures->fSse                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_SSE);
+        pFeatures->fSse2                = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_SSE2);
+        pFeatures->fSse3                = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSE3);
+        pFeatures->fSsse3               = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSSE3);
+        pFeatures->fSse41               = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSE4_1);
+        pFeatures->fSse42               = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_SSE4_2);
+        pFeatures->fAesNi               = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_AES);
+        pFeatures->fAvx                 = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_AVX);
+        pFeatures->fTsc                 = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_TSC);
+        pFeatures->fSysEnter            = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_SEP);
+        pFeatures->fHypervisorPresent   = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_HVP);
+        pFeatures->fMonitorMWait        = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_MONITOR);
+        pFeatures->fMovCmpXchg16b       = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_CX16);
+        pFeatures->fClFlush             = RT_BOOL(pStd1Leaf->uEdx & X86_CPUID_FEATURE_EDX_CLFSH);
+        pFeatures->fPcid                = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_PCID);
+        pFeatures->fPopCnt              = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_POPCNT);
+        pFeatures->fRdRand              = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_RDRAND);
+        pFeatures->fVmx                 = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_VMX);
+        pFeatures->fPclMul              = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_PCLMUL);
+        pFeatures->fMovBe               = RT_BOOL(pStd1Leaf->uEcx & X86_CPUID_FEATURE_ECX_MOVBE);
+        if (pFeatures->fVmx)
+            cpumExplodeVmxFeatures(&pMsrs->hwvirt.vmx, pFeatures);
+
+        /* Structured extended features. */
+        PCCPUMCPUIDLEAF const pSxfLeaf0 = cpumCpuIdFindLeafEx(paLeaves, cLeaves, 7, 0);
+        if (pSxfLeaf0)
+        {
+            pFeatures->fFsGsBase            = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_FSGSBASE);
+            pFeatures->fAvx2                = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_AVX2);
+            pFeatures->fAvx512Foundation    = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_AVX512F);
+            pFeatures->fClFlushOpt          = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_CLFLUSHOPT);
+            pFeatures->fInvpcid             = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_INVPCID);
+            pFeatures->fBmi1                = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_BMI1);
+            pFeatures->fBmi2                = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_BMI2);
+            pFeatures->fRdSeed              = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_RDSEED);
+            pFeatures->fHle                 = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_HLE);
+            pFeatures->fRtm                 = RT_BOOL(pSxfLeaf0->uEbx & X86_CPUID_STEXT_FEATURE_EBX_RTM);
+
+            pFeatures->fIbpb                = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_IBRS_IBPB);
+            pFeatures->fIbrs                = pFeatures->fIbpb;
+            pFeatures->fStibp               = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_STIBP);
+            pFeatures->fFlushCmd            = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_FLUSH_CMD);
+            pFeatures->fArchCap             = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_ARCHCAP);
+            pFeatures->fMdsClear            = RT_BOOL(pSxfLeaf0->uEdx & X86_CPUID_STEXT_FEATURE_EDX_MD_CLEAR);
+        }
+
+        /* MWAIT/MONITOR leaf. */
+        PCCPUMCPUIDLEAF const pMWaitLeaf = cpumCpuIdFindLeaf(paLeaves, cLeaves, 5);
+        if (pMWaitLeaf)
+            pFeatures->fMWaitExtensions = (pMWaitLeaf->uEcx & (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0))
+                                        ==                    (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0);
+
+        /* Extended features. */
+        PCCPUMCPUIDLEAF const pExtLeaf  = cpumCpuIdFindLeaf(paLeaves, cLeaves, 0x80000001);
+        if (pExtLeaf)
+        {
+            pFeatures->fLongMode        = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE);
+            pFeatures->fSysCall         = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_SYSCALL);
+            pFeatures->fNoExecute       = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_NX);
+            pFeatures->fLahfSahf        = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_EXT_FEATURE_ECX_LAHF_SAHF);
+            pFeatures->fRdTscP          = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_EXT_FEATURE_EDX_RDTSCP);
+            pFeatures->fMovCr8In32Bit   = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_CMPL);
+            pFeatures->f3DNow           = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_3DNOW);
+            pFeatures->f3DNowPrefetch   = (pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF)
+                                       || (pExtLeaf->uEdx & (  X86_CPUID_EXT_FEATURE_EDX_LONG_MODE
+                                                             | X86_CPUID_AMD_FEATURE_EDX_3DNOW));
+            pFeatures->fAbm             = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_ABM);
+        }
+
+        /* VMX (VMXON, VMCS region and related data structures) physical address width (depends on long-mode). */
+        pFeatures->cVmxMaxPhysAddrWidth = pFeatures->fLongMode ? pFeatures->cMaxPhysAddrWidth : 32;
+
+        if (   pExtLeaf
+            && (   pFeatures->enmCpuVendor == CPUMCPUVENDOR_AMD
+                || pFeatures->enmCpuVendor == CPUMCPUVENDOR_HYGON))
+        {
+            /* AMD features. */
+            pFeatures->fMsr            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_MSR);
+            pFeatures->fApic           |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_APIC);
+            pFeatures->fPse            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PSE);
+            pFeatures->fPse36          |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PSE36);
+            pFeatures->fPae            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PAE);
+            pFeatures->fPge            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PGE);
+            pFeatures->fPat            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_PAT);
+            pFeatures->fFxSaveRstor    |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_FXSR);
+            pFeatures->fMmx            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_MMX);
+            pFeatures->fTsc            |= RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_TSC);
+            pFeatures->fIbpb           |= pExtLeaf8 && (pExtLeaf8->uEbx & X86_CPUID_AMD_EFEID_EBX_IBPB);
+            pFeatures->fAmdMmxExts      = RT_BOOL(pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_AXMMX);
+            pFeatures->fXop             = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_XOP);
+            pFeatures->fTbm             = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_TBM);
+            pFeatures->fSvm             = RT_BOOL(pExtLeaf->uEcx & X86_CPUID_AMD_FEATURE_ECX_SVM);
+            if (pFeatures->fSvm)
+            {
+                PCCPUMCPUIDLEAF pSvmLeaf = cpumCpuIdFindLeaf(paLeaves, cLeaves, 0x8000000a);
+                AssertLogRelReturn(pSvmLeaf, VERR_CPUM_IPE_1);
+                pFeatures->fSvmNestedPaging         = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING);
+                pFeatures->fSvmLbrVirt              = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
+                pFeatures->fSvmSvmLock              = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_SVM_LOCK);
+                pFeatures->fSvmNextRipSave          = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
+                pFeatures->fSvmTscRateMsr           = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_TSC_RATE_MSR);
+                pFeatures->fSvmVmcbClean            = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
+                pFeatures->fSvmFlusbByAsid          = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID);
+                pFeatures->fSvmDecodeAssists        = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
+                pFeatures->fSvmPauseFilter          = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
+                pFeatures->fSvmPauseFilterThreshold = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
+                pFeatures->fSvmAvic                 = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_AVIC);
+                pFeatures->fSvmVirtVmsaveVmload     = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
+                pFeatures->fSvmVGif                 = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_VGIF);
+                pFeatures->fSvmGmet                 = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_GMET);
+                pFeatures->fSvmSSSCheck             = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_SSSCHECK);
+                pFeatures->fSvmSpecCtrl             = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_SPEC_CTRL);
+                pFeatures->fSvmHostMceOverride      = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_HOST_MCE_OVERRIDE);
+                pFeatures->fSvmTlbiCtl              = RT_BOOL(pSvmLeaf->uEdx & X86_CPUID_SVM_FEATURE_EDX_TLBICTL);
+                pFeatures->uSvmMaxAsid              = pSvmLeaf->uEbx;
+            }
+        }
+
+        /*
+         * Quirks.
+         */
+        pFeatures->fLeakyFxSR = pExtLeaf
+                             && (pExtLeaf->uEdx & X86_CPUID_AMD_FEATURE_EDX_FFXSR)
+                             && (   (   pFeatures->enmCpuVendor == CPUMCPUVENDOR_AMD
+                                     && pFeatures->uFamily >= 6 /* K7 and up */)
+                                 || pFeatures->enmCpuVendor == CPUMCPUVENDOR_HYGON);
+
+        /*
+         * Max extended (/FPU) state.
+         */
+        pFeatures->cbMaxExtendedState = pFeatures->fFxSaveRstor ? sizeof(X86FXSTATE) : sizeof(X86FPUSTATE);
+        if (pFeatures->fXSaveRstor)
+        {
+            PCCPUMCPUIDLEAF const pXStateLeaf0 = cpumCpuIdFindLeafEx(paLeaves, cLeaves, 13, 0);
+            if (pXStateLeaf0)
+            {
+                if (   pXStateLeaf0->uEcx >= sizeof(X86FXSTATE)
+                    && pXStateLeaf0->uEcx <= CPUM_MAX_XSAVE_AREA_SIZE
+                    && RT_ALIGN_32(pXStateLeaf0->uEcx, 8) == pXStateLeaf0->uEcx
+                    && pXStateLeaf0->uEbx >= sizeof(X86FXSTATE)
+                    && pXStateLeaf0->uEbx <= pXStateLeaf0->uEcx
+                    && RT_ALIGN_32(pXStateLeaf0->uEbx, 8) == pXStateLeaf0->uEbx)
+                {
+                    pFeatures->cbMaxExtendedState = pXStateLeaf0->uEcx;
+
+                    /* (paranoia:) */
+                    PCCPUMCPUIDLEAF const pXStateLeaf1 = cpumCpuIdFindLeafEx(paLeaves, cLeaves, 13, 1);
+                    if (   pXStateLeaf1
+                        && pXStateLeaf1->uEbx > pFeatures->cbMaxExtendedState
+                        && pXStateLeaf1->uEbx <= CPUM_MAX_XSAVE_AREA_SIZE
+                        && (pXStateLeaf1->uEcx || pXStateLeaf1->uEdx) )
+                        pFeatures->cbMaxExtendedState = pXStateLeaf1->uEbx;
+                }
+                else
+                    AssertLogRelMsgFailedStmt(("Unexpected max/cur XSAVE area sizes: %#x/%#x\n", pXStateLeaf0->uEcx, pXStateLeaf0->uEbx),
+                                              pFeatures->fXSaveRstor = 0);
+            }
+            else
+                AssertLogRelMsgFailedStmt(("Expected leaf eax=0xd/ecx=0 with the XSAVE/XRSTOR feature!\n"),
+                                          pFeatures->fXSaveRstor = 0);
+        }
+    }
+    else
+        AssertLogRelReturn(cLeaves == 0, VERR_CPUM_IPE_1);
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp b/src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp
new file mode 100644
index 00000000..29a4e52d
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/CPUMAllMsrs.cpp
@@ -0,0 +1,6505 @@
+/* $Id: CPUMAllMsrs.cpp $ */
+/** @file
+ * CPUM - CPU MSR Registers.
+ */
+
+/*
+ * Copyright (C) 2013-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/hm_vmx.h>
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+# include <VBox/vmm/iem.h>
+#endif
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/gim.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/**
+ * Validates the CPUMMSRRANGE::offCpumCpu value and declares a local variable
+ * pointing to it.
+ *
+ * ASSUMES sizeof(a_Type) is a power of two and that the member is aligned
+ * correctly.
+ */
+#define CPUM_MSR_ASSERT_CPUMCPU_OFFSET_RETURN(a_pVCpu, a_pRange, a_Type, a_VarName) \
+    AssertMsgReturn(   (a_pRange)->offCpumCpu >= 8 \
+                    && (a_pRange)->offCpumCpu < sizeof(CPUMCPU) \
+                    && !((a_pRange)->offCpumCpu & (RT_MIN(sizeof(a_Type), 8) - 1)) \
+                    , ("offCpumCpu=%#x %s\n", (a_pRange)->offCpumCpu, (a_pRange)->szName), \
+                    VERR_CPUM_MSR_BAD_CPUMCPU_OFFSET); \
+    a_Type *a_VarName = (a_Type *)((uintptr_t)&(a_pVCpu)->cpum.s + (a_pRange)->offCpumCpu)
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+
+/**
+ * Implements reading one or more MSRs.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_CPUM_R3_MSR_READ if the MSR read could not be serviced in the
+ *          current context (raw-mode or ring-0).
+ * @retval  VERR_CPUM_RAISE_GP_0 on failure (invalid MSR).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR we're reading.
+ * @param   pRange      The MSR range descriptor.
+ * @param   puValue     Where to return the value.
+ */
+typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNCPUMRDMSR,(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue));
+/** Pointer to a RDMSR worker for a specific MSR or range of MSRs.  */
+typedef FNCPUMRDMSR *PFNCPUMRDMSR;
+
+
+/**
+ * Implements writing one or more MSRs.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_CPUM_R3_MSR_WRITE if the MSR write could not be serviced in the
+ *          current context (raw-mode or ring-0).
+ * @retval  VERR_CPUM_RAISE_GP_0 on failure.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR we're writing.
+ * @param   pRange      The MSR range descriptor.
+ * @param   uValue      The value to set, ignored bits masked.
+ * @param   uRawValue   The raw value with the ignored bits not masked.
+ */
+typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNCPUMWRMSR,(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange,
+                                                    uint64_t uValue, uint64_t uRawValue));
+/** Pointer to a WRMSR worker for a specific MSR or range of MSRs.  */
+typedef FNCPUMWRMSR *PFNCPUMWRMSR;
+
+
+
+/*
+ * Generic functions.
+ * Generic functions.
+ * Generic functions.
+ */
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_FixedValue(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IgnoreWrite(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    Log(("CPUM: Ignoring WRMSR %#x (%s), %#llx\n", idMsr, pRange->szName, uValue));
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_WriteOnly(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(puValue);
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_ReadOnly(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    Assert(pRange->fWrGpMask == UINT64_MAX);
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+
+
+/*
+ * IA32
+ * IA32
+ * IA32
+ */
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32P5McAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0; /** @todo implement machine check injection. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32P5McAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement machine check injection. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32P5McType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0; /** @todo implement machine check injection. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32P5McType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement machine check injection. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32TimestampCounter(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = TMCpuTickGet(pVCpu);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    *puValue = CPUMApplyNestedGuestTscOffset(pVCpu, *puValue);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32TimestampCounter(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    TMCpuTickSet(pVCpu->CTX_SUFF(pVM), pVCpu, uValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PlatformId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    uint64_t uValue = pRange->uValue;
+    if (uValue & 0x1f00)
+    {
+        /* Max allowed bus ratio present. */
+        /** @todo Implement scaled BUS frequency. */
+    }
+
+    *puValue = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ApicBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    return APICGetBaseMsr(pVCpu, puValue);
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ApicBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return APICSetBaseMsr(pVCpu, uValue);
+}
+
+
+/**
+ * Gets IA32_FEATURE_CONTROL value for IEM, NEM and cpumMsrRd_Ia32FeatureControl.
+ *
+ * @returns IA32_FEATURE_CONTROL value.
+ * @param   pVCpu           The cross context per CPU structure.
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestIa32FeatCtrl(PCVMCPUCC pVCpu)
+{
+    uint64_t uFeatCtrlMsr = MSR_IA32_FEATURE_CONTROL_LOCK;
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        uFeatCtrlMsr |= MSR_IA32_FEATURE_CONTROL_VMXON;
+    return uFeatCtrlMsr;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32FeatureControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = CPUMGetGuestIa32FeatCtrl(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FeatureControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32BiosSignId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo fake microcode update. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32BiosSignId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Normally, zero is written to Ia32BiosSignId before reading it in order
+       to select the signature instead of the BBL_CR_D3 behaviour.  The GP mask
+       of the database entry should take care of most illegal writes for now, so
+       just ignore all writes atm. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32BiosUpdateTrigger(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+
+    /* Microcode updates cannot be loaded in VMX non-root mode. */
+    if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest))
+        return VINF_SUCCESS;
+
+    /** @todo Fake bios update trigger better.  The value is the address to an
+     *        update package, I think.  We should probably GP if it's invalid. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get MSR_IA32_SMM_MONITOR_CTL value for IEM and cpumMsrRd_Ia32SmmMonitorCtl.
+ *
+ * @returns The MSR_IA32_SMM_MONITOR_CTL value.
+ * @param   pVCpu           The cross context per CPU structure.
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestIa32SmmMonitorCtl(PCVMCPUCC pVCpu)
+{
+    /* We do not support dual-monitor treatment for SMI and SMM. */
+    /** @todo SMM. */
+    RT_NOREF(pVCpu);
+    return 0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SmmMonitorCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = CPUMGetGuestIa32SmmMonitorCtl(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SmmMonitorCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo SMM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PmcN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo check CPUID leaf 0ah. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PmcN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo check CPUID leaf 0ah. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MonitorFilterLineSize(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo return 0x1000 if we try emulate mwait 100% correctly. */
+    *puValue = 0x40; /** @todo Change to CPU cache line size. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MonitorFilterLineSize(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo should remember writes, though it's supposedly something only a BIOS
+     * would write so, it's not extremely important. */
+    return VINF_SUCCESS;
+}
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MPerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Read MPERF: Adjust against previously written MPERF value.  Is TSC
+     *        what we want? */
+    *puValue = TMCpuTickGet(pVCpu);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    *puValue = CPUMApplyNestedGuestTscOffset(pVCpu, *puValue);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MPerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Write MPERF: Calc adjustment. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32APerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Read APERF: Adjust against previously written MPERF value.  Is TSC
+     *        what we want? */
+    *puValue = TMCpuTickGet(pVCpu);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    *puValue = CPUMApplyNestedGuestTscOffset(pVCpu, *puValue);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32APerf(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Write APERF: Calc adjustment. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get fixed IA32_MTRR_CAP value for NEM and cpumMsrRd_Ia32MtrrCap.
+ *
+ * @returns Fixed IA32_MTRR_CAP value.
+ * @param   pVCpu           The cross context per CPU structure.
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestIa32MtrrCap(PCVMCPU pVCpu)
+{
+    RT_NOREF_PV(pVCpu);
+
+    /* This is currently a bit weird. :-) */
+    uint8_t const   cVariableRangeRegs              = 0;
+    bool const      fSystemManagementRangeRegisters = false;
+    bool const      fFixedRangeRegisters            = false;
+    bool const      fWriteCombiningType             = false;
+    return cVariableRangeRegs
+         | (fFixedRangeRegisters            ? RT_BIT_64(8)  : 0)
+         | (fWriteCombiningType             ? RT_BIT_64(10) : 0)
+         | (fSystemManagementRangeRegisters ? RT_BIT_64(11) : 0);
+}
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = CPUMGetGuestIa32MtrrCap(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrPhysBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement variable MTRR storage. */
+    Assert(pRange->uValue == (idMsr - 0x200) / 2);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrPhysBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    /*
+     * Validate the value.
+     */
+    Assert(pRange->uValue == (idMsr - 0x200) / 2);
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(uRawValue); RT_NOREF_PV(pRange);
+
+    uint8_t uType = uValue & 0xff;
+    if ((uType >= 7) || (uType == 2) || (uType == 3))
+    {
+        Log(("CPUM: Invalid type set writing MTRR PhysBase MSR %#x: %#llx (%#llx)\n", idMsr, uValue, uType));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    uint64_t fInvPhysMask = ~(RT_BIT_64(pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.cMaxPhysAddrWidth) - 1U);
+    if (fInvPhysMask & uValue)
+    {
+        Log(("CPUM: Invalid physical address bits set writing MTRR PhysBase MSR %#x: %#llx (%#llx)\n",
+             idMsr, uValue, uValue & fInvPhysMask));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    /*
+     * Store it.
+     */
+    /** @todo Implement variable MTRR storage. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrPhysMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement variable MTRR storage. */
+    Assert(pRange->uValue == (idMsr - 0x200) / 2);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrPhysMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    /*
+     * Validate the value.
+     */
+    Assert(pRange->uValue == (idMsr - 0x200) / 2);
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(uRawValue); RT_NOREF_PV(pRange);
+
+    uint64_t fInvPhysMask = ~(RT_BIT_64(pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.cMaxPhysAddrWidth) - 1U);
+    if (fInvPhysMask & uValue)
+    {
+        Log(("CPUM: Invalid physical address bits set writing MTRR PhysMask MSR %#x: %#llx (%#llx)\n",
+             idMsr, uValue, uValue & fInvPhysMask));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    /*
+     * Store it.
+     */
+    /** @todo Implement variable MTRR storage. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrFixed(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    CPUM_MSR_ASSERT_CPUMCPU_OFFSET_RETURN(pVCpu, pRange, uint64_t, puFixedMtrr);
+    *puValue = *puFixedMtrr;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrFixed(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    CPUM_MSR_ASSERT_CPUMCPU_OFFSET_RETURN(pVCpu, pRange, uint64_t, puFixedMtrr);
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(uRawValue);
+
+    for (uint32_t cShift = 0; cShift < 63; cShift += 8)
+    {
+        uint8_t uType = (uint8_t)(uValue >> cShift);
+        if ((uType >= 7) || (uType == 2) || (uType == 3))
+        {
+            Log(("CPUM: Invalid MTRR type at %u:%u in fixed range (%#x/%s): %#llx (%#llx)\n",
+                 cShift + 7, cShift, idMsr, pRange->szName, uValue, uType));
+            return VERR_CPUM_RAISE_GP_0;
+        }
+    }
+    *puFixedMtrr = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MtrrDefType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.MtrrDefType;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MtrrDefType(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+
+    uint8_t uType = uValue & 0xff;
+    if ((uType >= 7) || (uType == 2) || (uType == 3))
+    {
+        Log(("CPUM: Invalid MTRR default type value on %s: %#llx (%#llx)\n", pRange->szName, uValue, uType));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    pVCpu->cpum.s.GuestMsrs.msr.MtrrDefType = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32Pat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrPAT;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32Pat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    if (CPUMIsPatMsrValid(uValue))
+    {
+        pVCpu->cpum.s.Guest.msrPAT = uValue;
+        return VINF_SUCCESS;
+    }
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SysEnterCs(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.SysEnter.cs;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SysEnterCs(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+
+    /* Note! We used to mask this by 0xffff, but turns out real HW doesn't and
+             there are generally 32-bit working bits backing this register. */
+    pVCpu->cpum.s.Guest.SysEnter.cs = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SysEnterEsp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.SysEnter.esp;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SysEnterEsp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    if (X86_IS_CANONICAL(uValue))
+    {
+        pVCpu->cpum.s.Guest.SysEnter.esp = uValue;
+        return VINF_SUCCESS;
+    }
+    Log(("CPUM: IA32_SYSENTER_ESP not canonical! %#llx\n", uValue));
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SysEnterEip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.SysEnter.eip;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SysEnterEip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    if (X86_IS_CANONICAL(uValue))
+    {
+        pVCpu->cpum.s.Guest.SysEnter.eip = uValue;
+        return VINF_SUCCESS;
+    }
+    LogRel(("CPUM: IA32_SYSENTER_EIP not canonical! %#llx\n", uValue));
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McgCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+#if 0 /** @todo implement machine checks. */
+    *puValue = pRange->uValue & (RT_BIT_64(8) | 0);
+#else
+    *puValue = 0;
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McgStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement machine checks. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McgStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement machine checks. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McgCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement machine checks. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McgCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement machine checks. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32DebugCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_DEBUGCTL. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32DebugCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_DEBUGCTL. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SmrrPhysBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SmrrPhysBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement intel SMM. */
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SmrrPhysMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SmrrPhysMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement intel SMM. */
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PlatformDcaCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel direct cache access (DCA)?? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PlatformDcaCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement intel direct cache access (DCA)?? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32CpuDcaCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel direct cache access (DCA)?? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32Dca0Cap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement intel direct cache access (DCA)?? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32Dca0Cap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement intel direct cache access (DCA)?? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_PERFEVTSEL0+. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_PERFEVTSEL0+. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr);
+    uint64_t uValue = pRange->uValue;
+
+    /* Always provide the max bus ratio for now.  XNU expects it. */
+    uValue &= ~((UINT64_C(0x1f) << 40) | RT_BIT_64(46));
+
+    PVMCC    pVM            = pVCpu->CTX_SUFF(pVM);
+    uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM);
+    uint64_t uTscHz         = TMCpuTicksPerSecond(pVM);
+    uint8_t  uTscRatio      = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz);
+    if (uTscRatio > 0x1f)
+        uTscRatio = 0x1f;
+    uValue |= (uint64_t)uTscRatio << 40;
+
+    *puValue = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Pentium4 allows writing, but all bits are ignored. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_PERFCTL. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_PERFCTL. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32FixedCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_FIXED_CTRn (fixed performance counters). */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FixedCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_FIXED_CTRn (fixed performance counters). */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfCapabilities(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfCapabilities(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32FixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32PebsEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PebsEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ClockModulation(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_CLOCK_MODULATION. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ClockModulation(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_CLOCK_MODULATION. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ThermInterrupt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_THERM_INTERRUPT. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ThermInterrupt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_THERM_STATUS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ThermStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_THERM_STATUS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32ThermStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_THERM_INTERRUPT. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32Therm2Ctl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_THERM2_CTL. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32Therm2Ctl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement IA32_THERM2_CTL. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32MiscEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.MiscEnable;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32MiscEnable(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+#ifdef LOG_ENABLED
+    uint64_t const uOld = pVCpu->cpum.s.GuestMsrs.msr.MiscEnable;
+#endif
+
+    /* Unsupported bits are generally ignored and stripped by the MSR range
+       entry that got us here. So, we just need to preserve fixed bits. */
+    pVCpu->cpum.s.GuestMsrs.msr.MiscEnable = uValue
+                                           | MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL
+                                           | MSR_IA32_MISC_ENABLE_BTS_UNAVAIL;
+
+    Log(("CPUM: IA32_MISC_ENABLE; old=%#llx written=%#llx => %#llx\n",
+         uOld, uValue,  pVCpu->cpum.s.GuestMsrs.msr.MiscEnable));
+
+    /** @todo Wire IA32_MISC_ENABLE bit 22 to our NT 4 CPUID trick. */
+    /** @todo Wire up MSR_IA32_MISC_ENABLE_XD_DISABLE. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McCtlStatusAddrMiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(pRange);
+
+    /** @todo Implement machine check exception injection. */
+    switch (idMsr & 3)
+    {
+        case 0:
+        case 1:
+            *puValue = 0;
+            break;
+
+        /* The ADDR and MISC registers aren't accessible since the
+           corresponding STATUS bits are zero. */
+        case 2:
+            Log(("CPUM: Reading IA32_MCi_ADDR %#x -> #GP\n", idMsr));
+            return VERR_CPUM_RAISE_GP_0;
+        case 3:
+            Log(("CPUM: Reading IA32_MCi_MISC %#x -> #GP\n", idMsr));
+            return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McCtlStatusAddrMiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    switch (idMsr & 3)
+    {
+        case 0:
+            /* Ignore writes to the CTL register. */
+            break;
+
+        case 1:
+            /* According to specs, the STATUS register can only be written to
+               with the value 0.  VBoxCpuReport thinks different for a
+               Pentium M Dothan, but implementing according to specs now. */
+            if (uValue != 0)
+            {
+                Log(("CPUM: Writing non-zero value (%#llx) to IA32_MCi_STATUS %#x -> #GP\n", uValue, idMsr));
+                return VERR_CPUM_RAISE_GP_0;
+            }
+            break;
+
+        /* Specs states that ADDR and MISC can be cleared by writing zeros.
+           Writing 1s will GP.  Need to figure out how this relates to the
+           ADDRV and MISCV status flags.  If writing is independent of those
+           bits, we need to know whether the CPU really implements them since
+           that is exposed by writing 0 to them.
+           Implementing the solution with the fewer GPs for now. */
+        case 2:
+            if (uValue != 0)
+            {
+                Log(("CPUM: Writing non-zero value (%#llx) to IA32_MCi_ADDR %#x -> #GP\n", uValue, idMsr));
+                return VERR_CPUM_RAISE_GP_0;
+            }
+            break;
+        case 3:
+            if (uValue != 0)
+            {
+                Log(("CPUM: Writing non-zero value (%#llx) to IA32_MCi_MISC %#x -> #GP\n", uValue, idMsr));
+                return VERR_CPUM_RAISE_GP_0;
+            }
+            break;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32McNCtl2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement machine check exception injection. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32McNCtl2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Implement machine check exception injection. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32DsArea(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement IA32_DS_AREA. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32DsArea(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32TscDeadline(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement TSC deadline timer. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32TscDeadline(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement TSC deadline timer. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32X2ApicN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pRange);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (   CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest)
+        && CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.s.Guest, VMX_PROC_CTLS2_VIRT_X2APIC_MODE))
+    {
+        VBOXSTRICTRC rcStrict = IEMExecVmxVirtApicAccessMsr(pVCpu, idMsr, puValue, false /* fWrite */);
+        if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR)
+            return VINF_SUCCESS;
+        if (rcStrict == VERR_OUT_OF_RANGE)
+            return VERR_CPUM_RAISE_GP_0;
+        Assert(rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE);
+    }
+#endif
+    return APICReadMsr(pVCpu, idMsr, puValue);
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32X2ApicN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (   CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest)
+        && CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.s.Guest, VMX_PROC_CTLS2_VIRT_X2APIC_MODE))
+    {
+        VBOXSTRICTRC rcStrict = IEMExecVmxVirtApicAccessMsr(pVCpu, idMsr, &uValue, true /* fWrite */);
+        if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR)
+            return VINF_SUCCESS;
+        if (rcStrict == VERR_OUT_OF_RANGE)
+            return VERR_CPUM_RAISE_GP_0;
+        Assert(rcStrict == VINF_VMX_INTERCEPT_NOT_ACTIVE);
+    }
+#endif
+    return APICWriteMsr(pVCpu, idMsr, uValue);
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32DebugInterface(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo IA32_DEBUG_INTERFACE (no docs)  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32DebugInterface(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo IA32_DEBUG_INTERFACE (no docs)  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxBasic(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Basic;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxPinbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.PinCtls.u;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxProcbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.ProcCtls.u;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxExitCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.ExitCtls.u;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxEntryCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.EntryCtls.u;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxMisc(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Misc;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr0Fixed0(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr0Fixed0;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr0Fixed1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr4Fixed0(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxCr4Fixed1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxVmcsEnum(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64VmcsEnum;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxProcBasedCtls2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.ProcCtls2.u;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Get fixed IA32_VMX_EPT_VPID_CAP value for PGM and cpumMsrRd_Ia32VmxEptVpidCap.
+ *
+ * @returns Fixed IA32_VMX_EPT_VPID_CAP value.
+ * @param   pVCpu           The cross context per CPU structure.
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestIa32VmxEptVpidCap(PCVMCPUCC pVCpu)
+{
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        return pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64EptVpidCaps;
+    return 0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxEptVpidCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = CPUMGetGuestIa32VmxEptVpidCap(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTruePinbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.TruePinCtls.u;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTrueProcbasedCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.TrueProcCtls.u;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTrueExitCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.TrueExitCtls.u;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxTrueEntryCtls(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.TrueEntryCtls.u;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32VmxVmFunc(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    if (pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.fVmx)
+        *puValue = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64VmFunc;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32SpecCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32SpecCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+
+    /* NB: The STIBP bit can be set even when IBRS is present, regardless of whether STIBP is actually implemented. */
+    if (uValue & ~(MSR_IA32_SPEC_CTRL_F_IBRS | MSR_IA32_SPEC_CTRL_F_STIBP))
+    {
+        Log(("CPUM: Invalid IA32_SPEC_CTRL bits (trying to write %#llx)\n", uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32PredCmd(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Ia32ArchCapabilities(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.ArchCaps;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Ia32FlushCmd(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    if ((uValue & ~MSR_IA32_FLUSH_CMD_F_L1D) == 0)
+        return VINF_SUCCESS;
+    Log(("CPUM: Invalid MSR_IA32_FLUSH_CMD_ bits (trying to write %#llx)\n", uValue));
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+
+/*
+ * AMD64
+ * AMD64
+ * AMD64
+ */
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64Efer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrEFER;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64Efer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    uint64_t uValidatedEfer;
+    uint64_t const uOldEfer = pVCpu->cpum.s.Guest.msrEFER;
+    int rc = CPUMIsGuestEferMsrWriteValid(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.s.Guest.cr0, uOldEfer, uValue, &uValidatedEfer);
+    if (RT_FAILURE(rc))
+        return VERR_CPUM_RAISE_GP_0;
+
+    CPUMSetGuestEferMsrNoChecks(pVCpu, uOldEfer, uValidatedEfer);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64SyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrSTAR;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64SyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.Guest.msrSTAR = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64LongSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrLSTAR;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64LongSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    pVCpu->cpum.s.Guest.msrLSTAR = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64CompSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrCSTAR;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64CompSyscallTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    pVCpu->cpum.s.Guest.msrCSTAR = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64SyscallFlagMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrSFMASK;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64SyscallFlagMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.Guest.msrSFMASK = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64FsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.fs.u64Base;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64FsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.Guest.fs.u64Base = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64GsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.gs.u64Base;
+    return VINF_SUCCESS;
+}
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64GsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.Guest.gs.u64Base = uValue;
+    return VINF_SUCCESS;
+}
+
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64KernelGsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.msrKERNELGSBASE;
+    return VINF_SUCCESS;
+}
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64KernelGsBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.Guest.msrKERNELGSBASE = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Amd64TscAux(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.TscAux;
+    return VINF_SUCCESS;
+}
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Amd64TscAux(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    pVCpu->cpum.s.GuestMsrs.msr.TscAux = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/*
+ * Intel specific
+ * Intel specific
+ * Intel specific
+ */
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelEblCrPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo recalc clock frequency ratio? */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelEblCrPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Write EBL_CR_POWERON: Remember written bits. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7CoreThreadCount(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+
+    /* Note! According to cpuid_set_info in XNU (10.7.0), Westmere CPU only
+             have a 4-bit core count. */
+    uint16_t cCores   = pVCpu->CTX_SUFF(pVM)->cCpus;
+    uint16_t cThreads = cCores; /** @todo hyper-threading. */
+    *puValue = RT_MAKE_U32(cThreads, cCores);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP4EbcHardPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo P4 hard power on config */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP4EbcHardPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo P4 hard power on config */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP4EbcSoftPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo P4 soft power on config  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP4EbcSoftPowerOn(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo P4 soft power on config */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP4EbcFrequencyId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr);
+
+    uint64_t uValue;
+    PVMCC    pVM            = pVCpu->CTX_SUFF(pVM);
+    uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM);
+    if (pVM->cpum.s.GuestFeatures.uModel >= 2)
+    {
+        if (uScalableBusHz <= CPUM_SBUSFREQ_100MHZ && pVM->cpum.s.GuestFeatures.uModel <= 2)
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_100MHZ;
+            uValue = 0;
+        }
+        else if (uScalableBusHz <= CPUM_SBUSFREQ_133MHZ)
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_133MHZ;
+            uValue = 1;
+        }
+        else if (uScalableBusHz <= CPUM_SBUSFREQ_167MHZ)
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_167MHZ;
+            uValue = 3;
+        }
+        else if (uScalableBusHz <= CPUM_SBUSFREQ_200MHZ)
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_200MHZ;
+            uValue = 2;
+        }
+        else if (uScalableBusHz <= CPUM_SBUSFREQ_267MHZ && pVM->cpum.s.GuestFeatures.uModel > 2)
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_267MHZ;
+            uValue = 0;
+        }
+        else
+        {
+            uScalableBusHz = CPUM_SBUSFREQ_333MHZ;
+            uValue = 6;
+        }
+        uValue <<= 16;
+
+        uint64_t uTscHz    = TMCpuTicksPerSecond(pVM);
+        uint8_t  uTscRatio = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz);
+        uValue |= (uint32_t)uTscRatio << 24;
+
+        uValue |= pRange->uValue & ~UINT64_C(0xff0f0000);
+    }
+    else
+    {
+        /* Probably more stuff here, but intel doesn't want to tell us. */
+        uValue = pRange->uValue;
+        uValue &= ~(RT_BIT_64(21) | RT_BIT_64(22) | RT_BIT_64(23)); /* 100 MHz is only documented value */
+    }
+
+    *puValue = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP4EbcFrequencyId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo P4 bus frequency config  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP6FsbFrequency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr);
+
+    /* Convert the scalable bus frequency to the encoding in the intel manual (for core+). */
+    uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVCpu->CTX_SUFF(pVM));
+    if (uScalableBusHz <= CPUM_SBUSFREQ_100MHZ)
+        *puValue = 5;
+    else if (uScalableBusHz <= CPUM_SBUSFREQ_133MHZ)
+        *puValue = 1;
+    else if (uScalableBusHz <= CPUM_SBUSFREQ_167MHZ)
+        *puValue = 3;
+    else if (uScalableBusHz <= CPUM_SBUSFREQ_200MHZ)
+        *puValue = 2;
+    else if (uScalableBusHz <= CPUM_SBUSFREQ_267MHZ)
+        *puValue = 0;
+    else if (uScalableBusHz <= CPUM_SBUSFREQ_333MHZ)
+        *puValue = 4;
+    else /*if (uScalableBusHz <= CPUM_SBUSFREQ_400MHZ)*/
+        *puValue = 6;
+
+    *puValue |= pRange->uValue & ~UINT64_C(0x7);
+
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelPlatformInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+
+    /* Just indicate a fixed TSC, no turbo boost, no programmable anything. */
+    PVMCC    pVM            = pVCpu->CTX_SUFF(pVM);
+    uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM);
+    uint64_t uTscHz         = TMCpuTicksPerSecond(pVM);
+    uint8_t  uTscRatio      = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz);
+    uint64_t uValue         = ((uint32_t)uTscRatio << 8)   /* TSC invariant frequency. */
+                            | ((uint64_t)uTscRatio << 40); /* The max turbo frequency. */
+
+    /* Ivy bridge has a minimum operating ratio as well. */
+    if (true) /** @todo detect sandy bridge. */
+        uValue |= (uint64_t)uTscRatio << 48;
+
+    *puValue = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelFlexRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr);
+
+    uint64_t uValue = pRange->uValue & ~UINT64_C(0x1ff00);
+
+    PVMCC    pVM            = pVCpu->CTX_SUFF(pVM);
+    uint64_t uScalableBusHz = CPUMGetGuestScalableBusFrequency(pVM);
+    uint64_t uTscHz         = TMCpuTicksPerSecond(pVM);
+    uint8_t  uTscRatio      = (uint8_t)((uTscHz + uScalableBusHz / 2) / uScalableBusHz);
+    uValue |= (uint32_t)uTscRatio << 8;
+
+    *puValue = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelFlexRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement writing MSR_FLEX_RATIO. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelPkgCStConfigControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelPkgCStConfigControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+
+    if (pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl & RT_BIT_64(15))
+    {
+        Log(("CPUM: WRMSR %#x (%s), %#llx: Write protected -> #GP\n", idMsr, pRange->szName, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+#if 0 /** @todo check what real (old) hardware does. */
+    if ((uValue & 7) >= 5)
+    {
+        Log(("CPUM: WRMSR %#x (%s), %#llx: Invalid limit (%d) -> #GP\n", idMsr, pRange->szName, uValue, (uint32_t)(uValue & 7)));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+#endif
+    pVCpu->cpum.s.GuestMsrs.msr.PkgCStateCfgCtrl = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelPmgIoCaptureBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement I/O mwait wakeup. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelPmgIoCaptureBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement I/O mwait wakeup. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchFromToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchFromToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last branch records. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchFromN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchFromN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    /** @todo implement last branch records. */
+    /** @todo Probing indicates that bit 63 is settable on SandyBridge, at least
+     *        if the rest of the bits are zero.  Automatic sign extending?
+     *        Investigate! */
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchToN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last branch records. */
+    /** @todo Probing indicates that bit 63 is settable on SandyBridge, at least
+     *        if the rest of the bits are zero.  Automatic sign extending?
+     *        Investigate! */
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelLastBranchTos(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelLastBranchTos(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last branch records. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelBblCrCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelBblCrCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelBblCrCtl3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelBblCrCtl3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7TemperatureTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7TemperatureTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7MsrOffCoreResponseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo machine check. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7MsrOffCoreResponseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo machine check. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7MiscPwrMgmt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7MiscPwrMgmt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelP6CrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr);
+    int rc = CPUMGetGuestCRx(pVCpu, pRange->uValue, puValue);
+    AssertRC(rc);
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelP6CrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* This CRx interface differs from the MOV CRx, GReg interface in that
+       #GP(0) isn't raised if unsupported bits are written to.  Instead they
+       are simply ignored and masked off. (Pentium M Dothan)  */
+    /** @todo Implement MSR_P6_CRx writing.  Too much effort for very little, if
+     *        any, gain. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCpuId1FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement CPUID masking.  */
+    *puValue = UINT64_MAX;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCpuId1FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement CPUID masking.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCpuId1FeatureMaskEax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement CPUID masking.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCpuId1FeatureMaskEax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement CPUID masking.  */
+    return VINF_SUCCESS;
+}
+
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCpuId80000001FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement CPUID masking.  */
+    *puValue = UINT64_MAX;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCpuId80000001FeatureMaskEcdx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement CPUID masking.  */
+    return VINF_SUCCESS;
+}
+
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyAesNiCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement AES-NI.  */
+    *puValue = 3;  /* Bit 0 is lock bit, bit 1 disables AES-NI. That's what they say. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyAesNiCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement AES-NI.  */
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7TurboRatioLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo implement intel C states.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7TurboRatioLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement intel C states.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7LbrSelect(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last-branch-records.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7LbrSelect(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last-branch-records.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyErrorControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement memory error injection (MSR_ERROR_CONTROL).  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyErrorControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement memory error injection (MSR_ERROR_CONTROL).  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7VirtualLegacyWireCap(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo implement memory VLW?  */
+    *puValue = pRange->uValue;
+    /* Note: A20M is known to be bit 1 as this was disclosed in spec update
+       AAJ49/AAK51/????, which documents the inversion of this bit.  The
+       Sandy bridge CPU here has value 0x74, so it probably doesn't have a BIOS
+       that correct things.  Some guesses at the other bits:
+                 bit 2 = INTR
+                 bit 4 = SMI
+                 bit 5 = INIT
+                 bit 6 = NMI */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7PowerCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7PowerCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel power management  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyPebsNumAlt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel performance counters.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyPebsNumAlt(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel performance counters.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7PebsLdLat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel performance counters.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7PebsLdLat(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel performance counters.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7PkgCnResidencyN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7CoreCnResidencyN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyVrCurrentConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyVrCurrentConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyVrMiscConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyVrMiscConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Figure out what MSR_VR_CURRENT_CONFIG & MSR_VR_MISC_CONFIG are.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyRaplPowerUnit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo intel RAPL.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyRaplPowerUnit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Note! This is documented as read only and except for a Silvermont sample has
+             always been classified as read only.  This is just here to make it compile. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyPkgCnIrtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyPkgCnIrtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel power management.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SandyPkgC2Residency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7SandyPkgC2Residency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Note! This is documented as read only and except for a Silvermont sample has
+             always been classified as read only.  This is just here to make it compile. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPkgPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgEnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgPerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPkgPowerInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplDramPowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramEnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramPerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplDramPowerInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp0PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0EnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp0Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp0PerfStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp1PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp1PowerLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp1EnergyStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7RaplPp1Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel RAPL.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7RaplPp1Policy(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel RAPL.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpNominal(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo intel power management.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpLevel1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo intel power management.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpLevel2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo intel power management.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyConfigTdpControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7IvyConfigTdpControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel power management.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7IvyTurboActivationRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo intel power management.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7IvyTurboActivationRatio(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo intel power management.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfGlobalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfGlobalStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfGlobalOvfCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfFixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfFixedCtrCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncPerfFixedCtr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncPerfFixedCtr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncCBoxConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncArbPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncArbPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7UncArbPerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo uncore msrs.  */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelI7UncArbPerfEvtSelN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo uncore msrs.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelI7SmiCount(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+
+    /*
+     * 31:0 is SMI count (read only), 63:32 reserved.
+     * Since we don't do SMI, the count is always zero.
+     */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore2EmttmCrTablesN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo implement enhanced multi thread termal monitoring? */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore2EmttmCrTablesN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement enhanced multi thread termal monitoring? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore2SmmCStMiscInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo SMM & C-states? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore2SmmCStMiscInfo(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo SMM & C-states? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore1ExtConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Core1&2 EXT_CONFIG (whatever that is)? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore1ExtConfig(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Core1&2 EXT_CONFIG (whatever that is)? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore1DtsCalControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Core1&2(?) DTS_CAL_CTRL (whatever that is)? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore1DtsCalControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Core1&2(?) DTS_CAL_CTRL (whatever that is)? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelCore2PeciControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Core2+ platform environment control interface control register? */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_IntelCore2PeciControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Core2+ platform environment control interface control register? */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_IntelAtSilvCoreC1Recidency(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/*
+ * Multiple vendor P6 MSRs.
+ * Multiple vendor P6 MSRs.
+ * Multiple vendor P6 MSRs.
+ *
+ * These MSRs were introduced with the P6 but not elevated to architectural
+ * MSRs, despite other vendors implementing them.
+ */
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastBranchFromIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /* AMD seems to just record RIP, while intel claims to record RIP+CS.BASE
+       if I read the docs correctly, thus the need for separate functions. */
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastBranchToIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last branch records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastIntFromIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last exception records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_P6LastIntFromIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last exception records. */
+    /* Note! On many CPUs, the high bit of the 0x000001dd register is always writable, even when the result is
+             a non-cannonical address. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_P6LastIntToIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo implement last exception records. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_P6LastIntToIp(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo implement last exception records. */
+    return VINF_SUCCESS;
+}
+
+
+
+/*
+ * AMD specific
+ * AMD specific
+ * AMD specific
+ */
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hTscRate(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement TscRateMsr */
+    *puValue = RT_MAKE_U64(0, 1); /* 1.0 = reset value. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hTscRate(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Implement TscRateMsr */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hLwpCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */
+    /* Note: Only listes in BKDG for Family 15H. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hLwpCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hLwpCbAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */
+    /* Note: Only listes in BKDG for Family 15H. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hLwpCbAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Implement AMD LWP? (Instructions: LWPINS, LWPVAL, LLWPCB, SLWPCB) */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hMc4MiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo machine check. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hMc4MiscN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo machine check. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8PerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD performance events. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8PerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD performance events. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8PerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD performance events. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8PerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD performance events. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SysCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD SYS_CFG */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SysCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SYS_CFG */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8HwCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD HW_CFG */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8HwCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD HW_CFG */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IorrBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IorrMask/IorrBase */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IorrBaseN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IorrMask/IorrBase */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IorrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IorrMask/IorrBase */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IorrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IorrMask/IorrBase */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8TopOfMemN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = 0;
+    /** @todo return 4GB - RamHoleSize here for TOPMEM. Figure out what to return
+     *        for TOPMEM2. */
+    //if (pRange->uValue == 0)
+    //    *puValue = _4G - RamHoleSize;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8TopOfMemN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD TOPMEM and TOPMEM2/TOM2. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8NbCfg1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD NB_CFG1 */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8NbCfg1(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD NB_CFG1 */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8McXcptRedir(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo machine check. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8McXcptRedir(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo machine check. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuNameN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr);
+    PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), pRange->uValue / 2 + 0x80000001);
+    if (pLeaf)
+    {
+        if (!(pRange->uValue & 1))
+            *puValue = RT_MAKE_U64(pLeaf->uEax, pLeaf->uEbx);
+        else
+            *puValue = RT_MAKE_U64(pLeaf->uEcx, pLeaf->uEdx);
+    }
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuNameN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Remember guest programmed CPU name. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8HwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD HTC. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8HwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD HTC. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD STC. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SwThermalCtrl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD STC. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8FidVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD FIDVID_CTL. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8FidVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD FIDVID_CTL. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8FidVidStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD FIDVID_STATUS. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8McCtlMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD MC. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8McCtlMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD MC. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmiOnIoTrapN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM/SMI and I/O trap. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmiOnIoTrapN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM/SMI and I/O trap. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmiOnIoTrapCtlSts(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM/SMI and I/O trap. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmiOnIoTrapCtlSts(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM/SMI and I/O trap. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IntPendingMessage(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Interrupt pending message. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IntPendingMessage(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Interrupt pending message. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmiTriggerIoCycle(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM/SMI and trigger I/O cycle. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmiTriggerIoCycle(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM/SMI and trigger I/O cycle. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hMmioCfgBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD MMIO Configuration base address. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hMmioCfgBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD MMIO Configuration base address. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hTrapCtlMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD 0xc0010059. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hTrapCtlMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD 0xc0010059. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateCurLimit(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hPStateControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD P-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hPStateStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD P-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hPStateN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hPStateN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD P-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCofVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCofVidControl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD P-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCofVidStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD P-states. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCofVidStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Note! Writing 0 seems to not GP, not sure if it does anything to the value... */
+    /** @todo AMD P-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCStateIoBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD C-states. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCStateIoBaseAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD C-states. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hCpuWatchdogTimer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD machine checks. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hCpuWatchdogTimer(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD machine checks. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmBase(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM. */
+    return VINF_SUCCESS;
+}
+
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmMask(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8VmCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    if (pVM->cpum.s.GuestFeatures.fSvm)
+        *puValue = MSR_K8_VM_CR_LOCK;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8VmCr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    if (pVM->cpum.s.GuestFeatures.fSvm)
+    {
+        /* Silently ignore writes to LOCK and SVM_DISABLE bit when the LOCK bit is set (see cpumMsrRd_AmdK8VmCr). */
+        if (uValue & (MSR_K8_VM_CR_DPD | MSR_K8_VM_CR_R_INIT | MSR_K8_VM_CR_DIS_A20M))
+            return VERR_CPUM_RAISE_GP_0;
+        return VINF_SUCCESS;
+    }
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8IgnNe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IGNNE\# control. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8IgnNe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IGNNE\# control. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8SmmCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8SmmCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8VmHSavePa(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    *puValue = pVCpu->cpum.s.Guest.hwvirt.svm.uMsrHSavePa;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8VmHSavePa(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uRawValue);
+    if (uValue & UINT64_C(0xfff))
+    {
+        Log(("CPUM: Invalid setting of low 12 bits set writing host-state save area MSR %#x: %#llx\n", idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    uint64_t fInvPhysMask = ~(RT_BIT_64(pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.cMaxPhysAddrWidth) - 1U);
+    if (fInvPhysMask & uValue)
+    {
+        Log(("CPUM: Invalid physical address bits set writing host-state save area MSR %#x: %#llx (%#llx)\n",
+             idMsr, uValue, uValue & fInvPhysMask));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    pVCpu->cpum.s.Guest.hwvirt.svm.uMsrHSavePa = uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hVmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SVM. */
+    *puValue = 0; /* RAZ */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hVmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SVM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hSmmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM. */
+    *puValue = 0; /* RAZ */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hSmmLockKey(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hLocalSmiStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD SMM/SMI. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hLocalSmiStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD SMM/SMI. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hOsVisWrkIdLength(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo AMD OS visible workaround. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hOsVisWrkIdLength(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD OS visible workaround. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hOsVisWrkStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD OS visible workaround. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hOsVisWrkStatus(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD OS visible workaround. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam16hL2IPerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD L2I performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam16hL2IPerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD L2I performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam16hL2IPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD L2I performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam16hL2IPerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD L2I performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hNorthbridgePerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD Northbridge performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hNorthbridgePerfCtlN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD Northbridge performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hNorthbridgePerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD Northbridge performance counters. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hNorthbridgePerfCtrN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD Northbridge performance counters. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7MicrocodeCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus. Need to be explored and verify K7 presence. */
+    /** @todo Undocumented register only seen mentioned in fam15h erratum \#608. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7MicrocodeCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo Undocumented register only seen mentioned in fam15h erratum \#608. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7ClusterIdMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus. Need to be explored and verify K7 presence. */
+    /** @todo Undocumented register only seen mentioned in fam16h BKDG r3.00 when
+     *        describing EBL_CR_POWERON. */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7ClusterIdMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo Undocumented register only seen mentioned in fam16h BKDG r3.00 when
+     *        describing EBL_CR_POWERON. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlStd07hEbax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    bool           fIgnored;
+    PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeafEx(pVCpu->CTX_SUFF(pVM), 0x00000007, 0, &fIgnored);
+    if (pLeaf)
+        *puValue = RT_MAKE_U64(pLeaf->uEbx, pLeaf->uEax);
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlStd07hEbax(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Changing CPUID leaf 7/0. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlStd06hEcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), 0x00000006);
+    if (pLeaf)
+        *puValue = pLeaf->uEcx;
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlStd06hEcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Changing CPUID leaf 6. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlStd01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), 0x00000001);
+    if (pLeaf)
+        *puValue = RT_MAKE_U64(pLeaf->uEdx, pLeaf->uEcx);
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlStd01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Changing CPUID leaf 0x80000001. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8CpuIdCtlExt01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    PCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeaf(pVCpu->CTX_SUFF(pVM), 0x80000001);
+    if (pLeaf)
+        *puValue = RT_MAKE_U64(pLeaf->uEdx, pLeaf->uEcx);
+    else
+        *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8CpuIdCtlExt01hEdcx(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Changing CPUID leaf 0x80000001. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK8PatchLevel(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr);
+    /** @todo Fake AMD microcode patching.  */
+    *puValue = pRange->uValue;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK8PatchLoader(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Fake AMD microcode patching.  */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DebugStatusMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DebugStatusMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BHTraceBaseMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BHTraceBaseMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BHTracePtrMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BHTracePtrMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BHTraceLimitMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BHTraceLimitMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7HardwareDebugToolCfgMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7HardwareDebugToolCfgMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7FastFlushCountMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7FastFlushCountMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo undocumented */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7NodeId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD node ID and bios scratch. */
+    *puValue = 0; /* nodeid = 0; nodes-per-cpu = 1 */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7NodeId(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD node ID and bios scratch. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DrXAddrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD DRx address masking (range breakpoints). */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DrXAddrMaskN(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD DRx address masking (range breakpoints). */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7Dr0DataMatchMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD undocument debugging features. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7Dr0DataMatchMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD undocument debugging features. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7Dr0DataMaskMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD undocument debugging features. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7Dr0DataMaskMaybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD undocument debugging features. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7LoadStoreCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD load-store config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7LoadStoreCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD load-store config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7InstrCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD instruction cache config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7InstrCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD instruction cache config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DataCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD data cache config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DataCacheCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD data cache config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7BusUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD bus unit config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7BusUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo AMD bus unit config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdK7DebugCtl2Maybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo Undocument AMD debug control register \#2. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdK7DebugCtl2Maybe(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo Allegedly requiring edi=0x9c5a203a when execuing rdmsr/wrmsr on older
+     *  cpus.  Need to be explored and verify K7 presence.  */
+    /** @todo Undocument AMD debug control register \#2. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hFpuCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD FPU config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hFpuCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD FPU config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hDecoderCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD decoder config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hDecoderCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD decoder config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hBusUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /* Note! 10h and 16h */
+    /** @todo AMD bus unit  config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hBusUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /* Note! 10h and 16h */
+    /** @todo AMD bus unit config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hCombUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD unit config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hCombUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD unit config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hCombUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD unit config 2. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hCombUnitCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD unit config 2. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hCombUnitCfg3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD combined unit config 3. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hCombUnitCfg3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD combined unit config 3. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hExecUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD execution unit config. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hExecUnitCfg(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD execution unit config. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam15hLoadStoreCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD load-store config 2. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam15hLoadStoreCfg2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD load-store config 2. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsFetchCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsFetchCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsFetchLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsFetchLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsFetchPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsFetchPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpExecCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpExecCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpRip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpRip(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpData(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpData(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpData2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpData2(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsOpData3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsOpData3(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsDcLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsDcLinAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsDcPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsDcPhysAddr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam10hIbsCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam10hIbsCtl(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_AmdFam14hIbsBrTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange);
+    /** @todo AMD IBS. */
+    *puValue = 0;
+    return VINF_SUCCESS;
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_AmdFam14hIbsBrTarget(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(idMsr); RT_NOREF_PV(pRange); RT_NOREF_PV(uValue); RT_NOREF_PV(uRawValue);
+    /** @todo AMD IBS. */
+    if (!X86_IS_CANONICAL(uValue))
+    {
+        Log(("CPUM: wrmsr %s(%#x), %#llx -> #GP - not canonical\n", pRange->szName, idMsr, uValue));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+    return VINF_SUCCESS;
+}
+
+
+
+/*
+ * GIM MSRs.
+ * GIM MSRs.
+ * GIM MSRs.
+ */
+
+
+/** @callback_method_impl{FNCPUMRDMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrRd_Gim(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+    /* Raise #GP(0) like a physical CPU would since the nested-hypervisor hasn't intercept these MSRs. */
+    if (   CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.s.Guest)
+        || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest))
+        return VERR_CPUM_RAISE_GP_0;
+#endif
+    return GIMReadMsr(pVCpu, idMsr, pRange, puValue);
+}
+
+
+/** @callback_method_impl{FNCPUMWRMSR} */
+static DECLCALLBACK(VBOXSTRICTRC) cpumMsrWr_Gim(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+    /* Raise #GP(0) like a physical CPU would since the nested-hypervisor hasn't intercept these MSRs. */
+    if (   CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.s.Guest)
+        || CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest))
+        return VERR_CPUM_RAISE_GP_0;
+#endif
+    return GIMWriteMsr(pVCpu, idMsr, pRange, uValue, uRawValue);
+}
+
+
+/**
+ * MSR read function table.
+ */
+static const struct READMSRCLANG11WEIRDNOTHROW { PFNCPUMRDMSR pfnRdMsr; } g_aCpumRdMsrFns[kCpumMsrRdFn_End] =
+{
+    { NULL }, /* Invalid */
+    { cpumMsrRd_FixedValue },
+    { NULL }, /* Alias */
+    { cpumMsrRd_WriteOnly },
+    { cpumMsrRd_Ia32P5McAddr },
+    { cpumMsrRd_Ia32P5McType },
+    { cpumMsrRd_Ia32TimestampCounter },
+    { cpumMsrRd_Ia32PlatformId },
+    { cpumMsrRd_Ia32ApicBase },
+    { cpumMsrRd_Ia32FeatureControl },
+    { cpumMsrRd_Ia32BiosSignId },
+    { cpumMsrRd_Ia32SmmMonitorCtl },
+    { cpumMsrRd_Ia32PmcN },
+    { cpumMsrRd_Ia32MonitorFilterLineSize },
+    { cpumMsrRd_Ia32MPerf },
+    { cpumMsrRd_Ia32APerf },
+    { cpumMsrRd_Ia32MtrrCap },
+    { cpumMsrRd_Ia32MtrrPhysBaseN },
+    { cpumMsrRd_Ia32MtrrPhysMaskN },
+    { cpumMsrRd_Ia32MtrrFixed },
+    { cpumMsrRd_Ia32MtrrDefType },
+    { cpumMsrRd_Ia32Pat },
+    { cpumMsrRd_Ia32SysEnterCs },
+    { cpumMsrRd_Ia32SysEnterEsp },
+    { cpumMsrRd_Ia32SysEnterEip },
+    { cpumMsrRd_Ia32McgCap },
+    { cpumMsrRd_Ia32McgStatus },
+    { cpumMsrRd_Ia32McgCtl },
+    { cpumMsrRd_Ia32DebugCtl },
+    { cpumMsrRd_Ia32SmrrPhysBase },
+    { cpumMsrRd_Ia32SmrrPhysMask },
+    { cpumMsrRd_Ia32PlatformDcaCap },
+    { cpumMsrRd_Ia32CpuDcaCap },
+    { cpumMsrRd_Ia32Dca0Cap },
+    { cpumMsrRd_Ia32PerfEvtSelN },
+    { cpumMsrRd_Ia32PerfStatus },
+    { cpumMsrRd_Ia32PerfCtl },
+    { cpumMsrRd_Ia32FixedCtrN },
+    { cpumMsrRd_Ia32PerfCapabilities },
+    { cpumMsrRd_Ia32FixedCtrCtrl },
+    { cpumMsrRd_Ia32PerfGlobalStatus },
+    { cpumMsrRd_Ia32PerfGlobalCtrl },
+    { cpumMsrRd_Ia32PerfGlobalOvfCtrl },
+    { cpumMsrRd_Ia32PebsEnable },
+    { cpumMsrRd_Ia32ClockModulation },
+    { cpumMsrRd_Ia32ThermInterrupt },
+    { cpumMsrRd_Ia32ThermStatus },
+    { cpumMsrRd_Ia32Therm2Ctl },
+    { cpumMsrRd_Ia32MiscEnable },
+    { cpumMsrRd_Ia32McCtlStatusAddrMiscN },
+    { cpumMsrRd_Ia32McNCtl2 },
+    { cpumMsrRd_Ia32DsArea },
+    { cpumMsrRd_Ia32TscDeadline },
+    { cpumMsrRd_Ia32X2ApicN },
+    { cpumMsrRd_Ia32DebugInterface },
+    { cpumMsrRd_Ia32VmxBasic },
+    { cpumMsrRd_Ia32VmxPinbasedCtls },
+    { cpumMsrRd_Ia32VmxProcbasedCtls },
+    { cpumMsrRd_Ia32VmxExitCtls },
+    { cpumMsrRd_Ia32VmxEntryCtls },
+    { cpumMsrRd_Ia32VmxMisc },
+    { cpumMsrRd_Ia32VmxCr0Fixed0 },
+    { cpumMsrRd_Ia32VmxCr0Fixed1 },
+    { cpumMsrRd_Ia32VmxCr4Fixed0 },
+    { cpumMsrRd_Ia32VmxCr4Fixed1 },
+    { cpumMsrRd_Ia32VmxVmcsEnum },
+    { cpumMsrRd_Ia32VmxProcBasedCtls2 },
+    { cpumMsrRd_Ia32VmxEptVpidCap },
+    { cpumMsrRd_Ia32VmxTruePinbasedCtls },
+    { cpumMsrRd_Ia32VmxTrueProcbasedCtls },
+    { cpumMsrRd_Ia32VmxTrueExitCtls },
+    { cpumMsrRd_Ia32VmxTrueEntryCtls },
+    { cpumMsrRd_Ia32VmxVmFunc },
+    { cpumMsrRd_Ia32SpecCtrl },
+    { cpumMsrRd_Ia32ArchCapabilities },
+
+    { cpumMsrRd_Amd64Efer },
+    { cpumMsrRd_Amd64SyscallTarget },
+    { cpumMsrRd_Amd64LongSyscallTarget },
+    { cpumMsrRd_Amd64CompSyscallTarget },
+    { cpumMsrRd_Amd64SyscallFlagMask },
+    { cpumMsrRd_Amd64FsBase },
+    { cpumMsrRd_Amd64GsBase },
+    { cpumMsrRd_Amd64KernelGsBase },
+    { cpumMsrRd_Amd64TscAux },
+
+    { cpumMsrRd_IntelEblCrPowerOn },
+    { cpumMsrRd_IntelI7CoreThreadCount },
+    { cpumMsrRd_IntelP4EbcHardPowerOn },
+    { cpumMsrRd_IntelP4EbcSoftPowerOn },
+    { cpumMsrRd_IntelP4EbcFrequencyId },
+    { cpumMsrRd_IntelP6FsbFrequency },
+    { cpumMsrRd_IntelPlatformInfo },
+    { cpumMsrRd_IntelFlexRatio },
+    { cpumMsrRd_IntelPkgCStConfigControl },
+    { cpumMsrRd_IntelPmgIoCaptureBase },
+    { cpumMsrRd_IntelLastBranchFromToN },
+    { cpumMsrRd_IntelLastBranchFromN },
+    { cpumMsrRd_IntelLastBranchToN },
+    { cpumMsrRd_IntelLastBranchTos },
+    { cpumMsrRd_IntelBblCrCtl },
+    { cpumMsrRd_IntelBblCrCtl3 },
+    { cpumMsrRd_IntelI7TemperatureTarget },
+    { cpumMsrRd_IntelI7MsrOffCoreResponseN },
+    { cpumMsrRd_IntelI7MiscPwrMgmt },
+    { cpumMsrRd_IntelP6CrN },
+    { cpumMsrRd_IntelCpuId1FeatureMaskEcdx },
+    { cpumMsrRd_IntelCpuId1FeatureMaskEax },
+    { cpumMsrRd_IntelCpuId80000001FeatureMaskEcdx },
+    { cpumMsrRd_IntelI7SandyAesNiCtl },
+    { cpumMsrRd_IntelI7TurboRatioLimit },
+    { cpumMsrRd_IntelI7LbrSelect },
+    { cpumMsrRd_IntelI7SandyErrorControl },
+    { cpumMsrRd_IntelI7VirtualLegacyWireCap },
+    { cpumMsrRd_IntelI7PowerCtl },
+    { cpumMsrRd_IntelI7SandyPebsNumAlt },
+    { cpumMsrRd_IntelI7PebsLdLat },
+    { cpumMsrRd_IntelI7PkgCnResidencyN },
+    { cpumMsrRd_IntelI7CoreCnResidencyN },
+    { cpumMsrRd_IntelI7SandyVrCurrentConfig },
+    { cpumMsrRd_IntelI7SandyVrMiscConfig },
+    { cpumMsrRd_IntelI7SandyRaplPowerUnit },
+    { cpumMsrRd_IntelI7SandyPkgCnIrtlN },
+    { cpumMsrRd_IntelI7SandyPkgC2Residency },
+    { cpumMsrRd_IntelI7RaplPkgPowerLimit },
+    { cpumMsrRd_IntelI7RaplPkgEnergyStatus },
+    { cpumMsrRd_IntelI7RaplPkgPerfStatus },
+    { cpumMsrRd_IntelI7RaplPkgPowerInfo },
+    { cpumMsrRd_IntelI7RaplDramPowerLimit },
+    { cpumMsrRd_IntelI7RaplDramEnergyStatus },
+    { cpumMsrRd_IntelI7RaplDramPerfStatus },
+    { cpumMsrRd_IntelI7RaplDramPowerInfo },
+    { cpumMsrRd_IntelI7RaplPp0PowerLimit },
+    { cpumMsrRd_IntelI7RaplPp0EnergyStatus },
+    { cpumMsrRd_IntelI7RaplPp0Policy },
+    { cpumMsrRd_IntelI7RaplPp0PerfStatus },
+    { cpumMsrRd_IntelI7RaplPp1PowerLimit },
+    { cpumMsrRd_IntelI7RaplPp1EnergyStatus },
+    { cpumMsrRd_IntelI7RaplPp1Policy },
+    { cpumMsrRd_IntelI7IvyConfigTdpNominal },
+    { cpumMsrRd_IntelI7IvyConfigTdpLevel1 },
+    { cpumMsrRd_IntelI7IvyConfigTdpLevel2 },
+    { cpumMsrRd_IntelI7IvyConfigTdpControl },
+    { cpumMsrRd_IntelI7IvyTurboActivationRatio },
+    { cpumMsrRd_IntelI7UncPerfGlobalCtrl },
+    { cpumMsrRd_IntelI7UncPerfGlobalStatus },
+    { cpumMsrRd_IntelI7UncPerfGlobalOvfCtrl },
+    { cpumMsrRd_IntelI7UncPerfFixedCtrCtrl },
+    { cpumMsrRd_IntelI7UncPerfFixedCtr },
+    { cpumMsrRd_IntelI7UncCBoxConfig },
+    { cpumMsrRd_IntelI7UncArbPerfCtrN },
+    { cpumMsrRd_IntelI7UncArbPerfEvtSelN },
+    { cpumMsrRd_IntelI7SmiCount },
+    { cpumMsrRd_IntelCore2EmttmCrTablesN },
+    { cpumMsrRd_IntelCore2SmmCStMiscInfo },
+    { cpumMsrRd_IntelCore1ExtConfig },
+    { cpumMsrRd_IntelCore1DtsCalControl },
+    { cpumMsrRd_IntelCore2PeciControl },
+    { cpumMsrRd_IntelAtSilvCoreC1Recidency },
+
+    { cpumMsrRd_P6LastBranchFromIp },
+    { cpumMsrRd_P6LastBranchToIp },
+    { cpumMsrRd_P6LastIntFromIp },
+    { cpumMsrRd_P6LastIntToIp },
+
+    { cpumMsrRd_AmdFam15hTscRate },
+    { cpumMsrRd_AmdFam15hLwpCfg },
+    { cpumMsrRd_AmdFam15hLwpCbAddr },
+    { cpumMsrRd_AmdFam10hMc4MiscN },
+    { cpumMsrRd_AmdK8PerfCtlN },
+    { cpumMsrRd_AmdK8PerfCtrN },
+    { cpumMsrRd_AmdK8SysCfg },
+    { cpumMsrRd_AmdK8HwCr },
+    { cpumMsrRd_AmdK8IorrBaseN },
+    { cpumMsrRd_AmdK8IorrMaskN },
+    { cpumMsrRd_AmdK8TopOfMemN },
+    { cpumMsrRd_AmdK8NbCfg1 },
+    { cpumMsrRd_AmdK8McXcptRedir },
+    { cpumMsrRd_AmdK8CpuNameN },
+    { cpumMsrRd_AmdK8HwThermalCtrl },
+    { cpumMsrRd_AmdK8SwThermalCtrl },
+    { cpumMsrRd_AmdK8FidVidControl },
+    { cpumMsrRd_AmdK8FidVidStatus },
+    { cpumMsrRd_AmdK8McCtlMaskN },
+    { cpumMsrRd_AmdK8SmiOnIoTrapN },
+    { cpumMsrRd_AmdK8SmiOnIoTrapCtlSts },
+    { cpumMsrRd_AmdK8IntPendingMessage },
+    { cpumMsrRd_AmdK8SmiTriggerIoCycle },
+    { cpumMsrRd_AmdFam10hMmioCfgBaseAddr },
+    { cpumMsrRd_AmdFam10hTrapCtlMaybe },
+    { cpumMsrRd_AmdFam10hPStateCurLimit },
+    { cpumMsrRd_AmdFam10hPStateControl },
+    { cpumMsrRd_AmdFam10hPStateStatus },
+    { cpumMsrRd_AmdFam10hPStateN },
+    { cpumMsrRd_AmdFam10hCofVidControl },
+    { cpumMsrRd_AmdFam10hCofVidStatus },
+    { cpumMsrRd_AmdFam10hCStateIoBaseAddr },
+    { cpumMsrRd_AmdFam10hCpuWatchdogTimer },
+    { cpumMsrRd_AmdK8SmmBase },
+    { cpumMsrRd_AmdK8SmmAddr },
+    { cpumMsrRd_AmdK8SmmMask },
+    { cpumMsrRd_AmdK8VmCr },
+    { cpumMsrRd_AmdK8IgnNe },
+    { cpumMsrRd_AmdK8SmmCtl },
+    { cpumMsrRd_AmdK8VmHSavePa },
+    { cpumMsrRd_AmdFam10hVmLockKey },
+    { cpumMsrRd_AmdFam10hSmmLockKey },
+    { cpumMsrRd_AmdFam10hLocalSmiStatus },
+    { cpumMsrRd_AmdFam10hOsVisWrkIdLength },
+    { cpumMsrRd_AmdFam10hOsVisWrkStatus },
+    { cpumMsrRd_AmdFam16hL2IPerfCtlN },
+    { cpumMsrRd_AmdFam16hL2IPerfCtrN },
+    { cpumMsrRd_AmdFam15hNorthbridgePerfCtlN },
+    { cpumMsrRd_AmdFam15hNorthbridgePerfCtrN },
+    { cpumMsrRd_AmdK7MicrocodeCtl },
+    { cpumMsrRd_AmdK7ClusterIdMaybe },
+    { cpumMsrRd_AmdK8CpuIdCtlStd07hEbax },
+    { cpumMsrRd_AmdK8CpuIdCtlStd06hEcx },
+    { cpumMsrRd_AmdK8CpuIdCtlStd01hEdcx },
+    { cpumMsrRd_AmdK8CpuIdCtlExt01hEdcx },
+    { cpumMsrRd_AmdK8PatchLevel },
+    { cpumMsrRd_AmdK7DebugStatusMaybe },
+    { cpumMsrRd_AmdK7BHTraceBaseMaybe },
+    { cpumMsrRd_AmdK7BHTracePtrMaybe },
+    { cpumMsrRd_AmdK7BHTraceLimitMaybe },
+    { cpumMsrRd_AmdK7HardwareDebugToolCfgMaybe },
+    { cpumMsrRd_AmdK7FastFlushCountMaybe },
+    { cpumMsrRd_AmdK7NodeId },
+    { cpumMsrRd_AmdK7DrXAddrMaskN },
+    { cpumMsrRd_AmdK7Dr0DataMatchMaybe },
+    { cpumMsrRd_AmdK7Dr0DataMaskMaybe },
+    { cpumMsrRd_AmdK7LoadStoreCfg },
+    { cpumMsrRd_AmdK7InstrCacheCfg },
+    { cpumMsrRd_AmdK7DataCacheCfg },
+    { cpumMsrRd_AmdK7BusUnitCfg },
+    { cpumMsrRd_AmdK7DebugCtl2Maybe },
+    { cpumMsrRd_AmdFam15hFpuCfg },
+    { cpumMsrRd_AmdFam15hDecoderCfg },
+    { cpumMsrRd_AmdFam10hBusUnitCfg2 },
+    { cpumMsrRd_AmdFam15hCombUnitCfg },
+    { cpumMsrRd_AmdFam15hCombUnitCfg2 },
+    { cpumMsrRd_AmdFam15hCombUnitCfg3 },
+    { cpumMsrRd_AmdFam15hExecUnitCfg },
+    { cpumMsrRd_AmdFam15hLoadStoreCfg2 },
+    { cpumMsrRd_AmdFam10hIbsFetchCtl },
+    { cpumMsrRd_AmdFam10hIbsFetchLinAddr },
+    { cpumMsrRd_AmdFam10hIbsFetchPhysAddr },
+    { cpumMsrRd_AmdFam10hIbsOpExecCtl },
+    { cpumMsrRd_AmdFam10hIbsOpRip },
+    { cpumMsrRd_AmdFam10hIbsOpData },
+    { cpumMsrRd_AmdFam10hIbsOpData2 },
+    { cpumMsrRd_AmdFam10hIbsOpData3 },
+    { cpumMsrRd_AmdFam10hIbsDcLinAddr },
+    { cpumMsrRd_AmdFam10hIbsDcPhysAddr },
+    { cpumMsrRd_AmdFam10hIbsCtl },
+    { cpumMsrRd_AmdFam14hIbsBrTarget },
+
+    { cpumMsrRd_Gim },
+};
+
+
+/**
+ * MSR write function table.
+ */
+static const struct WRITEMSRCLANG11WEIRDNOTHROW { PFNCPUMWRMSR pfnWrMsr; } g_aCpumWrMsrFns[kCpumMsrWrFn_End] =
+{
+    { NULL }, /* Invalid */
+    { cpumMsrWr_IgnoreWrite },
+    { cpumMsrWr_ReadOnly },
+    { NULL }, /* Alias */
+    { cpumMsrWr_Ia32P5McAddr },
+    { cpumMsrWr_Ia32P5McType },
+    { cpumMsrWr_Ia32TimestampCounter },
+    { cpumMsrWr_Ia32ApicBase },
+    { cpumMsrWr_Ia32FeatureControl },
+    { cpumMsrWr_Ia32BiosSignId },
+    { cpumMsrWr_Ia32BiosUpdateTrigger },
+    { cpumMsrWr_Ia32SmmMonitorCtl },
+    { cpumMsrWr_Ia32PmcN },
+    { cpumMsrWr_Ia32MonitorFilterLineSize },
+    { cpumMsrWr_Ia32MPerf },
+    { cpumMsrWr_Ia32APerf },
+    { cpumMsrWr_Ia32MtrrPhysBaseN },
+    { cpumMsrWr_Ia32MtrrPhysMaskN },
+    { cpumMsrWr_Ia32MtrrFixed },
+    { cpumMsrWr_Ia32MtrrDefType },
+    { cpumMsrWr_Ia32Pat },
+    { cpumMsrWr_Ia32SysEnterCs },
+    { cpumMsrWr_Ia32SysEnterEsp },
+    { cpumMsrWr_Ia32SysEnterEip },
+    { cpumMsrWr_Ia32McgStatus },
+    { cpumMsrWr_Ia32McgCtl },
+    { cpumMsrWr_Ia32DebugCtl },
+    { cpumMsrWr_Ia32SmrrPhysBase },
+    { cpumMsrWr_Ia32SmrrPhysMask },
+    { cpumMsrWr_Ia32PlatformDcaCap },
+    { cpumMsrWr_Ia32Dca0Cap },
+    { cpumMsrWr_Ia32PerfEvtSelN },
+    { cpumMsrWr_Ia32PerfStatus },
+    { cpumMsrWr_Ia32PerfCtl },
+    { cpumMsrWr_Ia32FixedCtrN },
+    { cpumMsrWr_Ia32PerfCapabilities },
+    { cpumMsrWr_Ia32FixedCtrCtrl },
+    { cpumMsrWr_Ia32PerfGlobalStatus },
+    { cpumMsrWr_Ia32PerfGlobalCtrl },
+    { cpumMsrWr_Ia32PerfGlobalOvfCtrl },
+    { cpumMsrWr_Ia32PebsEnable },
+    { cpumMsrWr_Ia32ClockModulation },
+    { cpumMsrWr_Ia32ThermInterrupt },
+    { cpumMsrWr_Ia32ThermStatus },
+    { cpumMsrWr_Ia32Therm2Ctl },
+    { cpumMsrWr_Ia32MiscEnable },
+    { cpumMsrWr_Ia32McCtlStatusAddrMiscN },
+    { cpumMsrWr_Ia32McNCtl2 },
+    { cpumMsrWr_Ia32DsArea },
+    { cpumMsrWr_Ia32TscDeadline },
+    { cpumMsrWr_Ia32X2ApicN },
+    { cpumMsrWr_Ia32DebugInterface },
+    { cpumMsrWr_Ia32SpecCtrl },
+    { cpumMsrWr_Ia32PredCmd },
+    { cpumMsrWr_Ia32FlushCmd },
+
+    { cpumMsrWr_Amd64Efer },
+    { cpumMsrWr_Amd64SyscallTarget },
+    { cpumMsrWr_Amd64LongSyscallTarget },
+    { cpumMsrWr_Amd64CompSyscallTarget },
+    { cpumMsrWr_Amd64SyscallFlagMask },
+    { cpumMsrWr_Amd64FsBase },
+    { cpumMsrWr_Amd64GsBase },
+    { cpumMsrWr_Amd64KernelGsBase },
+    { cpumMsrWr_Amd64TscAux },
+
+    { cpumMsrWr_IntelEblCrPowerOn },
+    { cpumMsrWr_IntelP4EbcHardPowerOn },
+    { cpumMsrWr_IntelP4EbcSoftPowerOn },
+    { cpumMsrWr_IntelP4EbcFrequencyId },
+    { cpumMsrWr_IntelFlexRatio },
+    { cpumMsrWr_IntelPkgCStConfigControl },
+    { cpumMsrWr_IntelPmgIoCaptureBase },
+    { cpumMsrWr_IntelLastBranchFromToN },
+    { cpumMsrWr_IntelLastBranchFromN },
+    { cpumMsrWr_IntelLastBranchToN },
+    { cpumMsrWr_IntelLastBranchTos },
+    { cpumMsrWr_IntelBblCrCtl },
+    { cpumMsrWr_IntelBblCrCtl3 },
+    { cpumMsrWr_IntelI7TemperatureTarget },
+    { cpumMsrWr_IntelI7MsrOffCoreResponseN },
+    { cpumMsrWr_IntelI7MiscPwrMgmt },
+    { cpumMsrWr_IntelP6CrN },
+    { cpumMsrWr_IntelCpuId1FeatureMaskEcdx },
+    { cpumMsrWr_IntelCpuId1FeatureMaskEax },
+    { cpumMsrWr_IntelCpuId80000001FeatureMaskEcdx },
+    { cpumMsrWr_IntelI7SandyAesNiCtl },
+    { cpumMsrWr_IntelI7TurboRatioLimit },
+    { cpumMsrWr_IntelI7LbrSelect },
+    { cpumMsrWr_IntelI7SandyErrorControl },
+    { cpumMsrWr_IntelI7PowerCtl },
+    { cpumMsrWr_IntelI7SandyPebsNumAlt },
+    { cpumMsrWr_IntelI7PebsLdLat },
+    { cpumMsrWr_IntelI7SandyVrCurrentConfig },
+    { cpumMsrWr_IntelI7SandyVrMiscConfig },
+    { cpumMsrWr_IntelI7SandyRaplPowerUnit },
+    { cpumMsrWr_IntelI7SandyPkgCnIrtlN },
+    { cpumMsrWr_IntelI7SandyPkgC2Residency },
+    { cpumMsrWr_IntelI7RaplPkgPowerLimit },
+    { cpumMsrWr_IntelI7RaplDramPowerLimit },
+    { cpumMsrWr_IntelI7RaplPp0PowerLimit },
+    { cpumMsrWr_IntelI7RaplPp0Policy },
+    { cpumMsrWr_IntelI7RaplPp1PowerLimit },
+    { cpumMsrWr_IntelI7RaplPp1Policy },
+    { cpumMsrWr_IntelI7IvyConfigTdpControl },
+    { cpumMsrWr_IntelI7IvyTurboActivationRatio },
+    { cpumMsrWr_IntelI7UncPerfGlobalCtrl },
+    { cpumMsrWr_IntelI7UncPerfGlobalStatus },
+    { cpumMsrWr_IntelI7UncPerfGlobalOvfCtrl },
+    { cpumMsrWr_IntelI7UncPerfFixedCtrCtrl },
+    { cpumMsrWr_IntelI7UncPerfFixedCtr },
+    { cpumMsrWr_IntelI7UncArbPerfCtrN },
+    { cpumMsrWr_IntelI7UncArbPerfEvtSelN },
+    { cpumMsrWr_IntelCore2EmttmCrTablesN },
+    { cpumMsrWr_IntelCore2SmmCStMiscInfo },
+    { cpumMsrWr_IntelCore1ExtConfig },
+    { cpumMsrWr_IntelCore1DtsCalControl },
+    { cpumMsrWr_IntelCore2PeciControl },
+
+    { cpumMsrWr_P6LastIntFromIp },
+    { cpumMsrWr_P6LastIntToIp },
+
+    { cpumMsrWr_AmdFam15hTscRate },
+    { cpumMsrWr_AmdFam15hLwpCfg },
+    { cpumMsrWr_AmdFam15hLwpCbAddr },
+    { cpumMsrWr_AmdFam10hMc4MiscN },
+    { cpumMsrWr_AmdK8PerfCtlN },
+    { cpumMsrWr_AmdK8PerfCtrN },
+    { cpumMsrWr_AmdK8SysCfg },
+    { cpumMsrWr_AmdK8HwCr },
+    { cpumMsrWr_AmdK8IorrBaseN },
+    { cpumMsrWr_AmdK8IorrMaskN },
+    { cpumMsrWr_AmdK8TopOfMemN },
+    { cpumMsrWr_AmdK8NbCfg1 },
+    { cpumMsrWr_AmdK8McXcptRedir },
+    { cpumMsrWr_AmdK8CpuNameN },
+    { cpumMsrWr_AmdK8HwThermalCtrl },
+    { cpumMsrWr_AmdK8SwThermalCtrl },
+    { cpumMsrWr_AmdK8FidVidControl },
+    { cpumMsrWr_AmdK8McCtlMaskN },
+    { cpumMsrWr_AmdK8SmiOnIoTrapN },
+    { cpumMsrWr_AmdK8SmiOnIoTrapCtlSts },
+    { cpumMsrWr_AmdK8IntPendingMessage },
+    { cpumMsrWr_AmdK8SmiTriggerIoCycle },
+    { cpumMsrWr_AmdFam10hMmioCfgBaseAddr },
+    { cpumMsrWr_AmdFam10hTrapCtlMaybe },
+    { cpumMsrWr_AmdFam10hPStateControl },
+    { cpumMsrWr_AmdFam10hPStateStatus },
+    { cpumMsrWr_AmdFam10hPStateN },
+    { cpumMsrWr_AmdFam10hCofVidControl },
+    { cpumMsrWr_AmdFam10hCofVidStatus },
+    { cpumMsrWr_AmdFam10hCStateIoBaseAddr },
+    { cpumMsrWr_AmdFam10hCpuWatchdogTimer },
+    { cpumMsrWr_AmdK8SmmBase },
+    { cpumMsrWr_AmdK8SmmAddr },
+    { cpumMsrWr_AmdK8SmmMask },
+    { cpumMsrWr_AmdK8VmCr },
+    { cpumMsrWr_AmdK8IgnNe },
+    { cpumMsrWr_AmdK8SmmCtl },
+    { cpumMsrWr_AmdK8VmHSavePa },
+    { cpumMsrWr_AmdFam10hVmLockKey },
+    { cpumMsrWr_AmdFam10hSmmLockKey },
+    { cpumMsrWr_AmdFam10hLocalSmiStatus },
+    { cpumMsrWr_AmdFam10hOsVisWrkIdLength },
+    { cpumMsrWr_AmdFam10hOsVisWrkStatus },
+    { cpumMsrWr_AmdFam16hL2IPerfCtlN },
+    { cpumMsrWr_AmdFam16hL2IPerfCtrN },
+    { cpumMsrWr_AmdFam15hNorthbridgePerfCtlN },
+    { cpumMsrWr_AmdFam15hNorthbridgePerfCtrN },
+    { cpumMsrWr_AmdK7MicrocodeCtl },
+    { cpumMsrWr_AmdK7ClusterIdMaybe },
+    { cpumMsrWr_AmdK8CpuIdCtlStd07hEbax },
+    { cpumMsrWr_AmdK8CpuIdCtlStd06hEcx },
+    { cpumMsrWr_AmdK8CpuIdCtlStd01hEdcx },
+    { cpumMsrWr_AmdK8CpuIdCtlExt01hEdcx },
+    { cpumMsrWr_AmdK8PatchLoader },
+    { cpumMsrWr_AmdK7DebugStatusMaybe },
+    { cpumMsrWr_AmdK7BHTraceBaseMaybe },
+    { cpumMsrWr_AmdK7BHTracePtrMaybe },
+    { cpumMsrWr_AmdK7BHTraceLimitMaybe },
+    { cpumMsrWr_AmdK7HardwareDebugToolCfgMaybe },
+    { cpumMsrWr_AmdK7FastFlushCountMaybe },
+    { cpumMsrWr_AmdK7NodeId },
+    { cpumMsrWr_AmdK7DrXAddrMaskN },
+    { cpumMsrWr_AmdK7Dr0DataMatchMaybe },
+    { cpumMsrWr_AmdK7Dr0DataMaskMaybe },
+    { cpumMsrWr_AmdK7LoadStoreCfg },
+    { cpumMsrWr_AmdK7InstrCacheCfg },
+    { cpumMsrWr_AmdK7DataCacheCfg },
+    { cpumMsrWr_AmdK7BusUnitCfg },
+    { cpumMsrWr_AmdK7DebugCtl2Maybe },
+    { cpumMsrWr_AmdFam15hFpuCfg },
+    { cpumMsrWr_AmdFam15hDecoderCfg },
+    { cpumMsrWr_AmdFam10hBusUnitCfg2 },
+    { cpumMsrWr_AmdFam15hCombUnitCfg },
+    { cpumMsrWr_AmdFam15hCombUnitCfg2 },
+    { cpumMsrWr_AmdFam15hCombUnitCfg3 },
+    { cpumMsrWr_AmdFam15hExecUnitCfg },
+    { cpumMsrWr_AmdFam15hLoadStoreCfg2 },
+    { cpumMsrWr_AmdFam10hIbsFetchCtl },
+    { cpumMsrWr_AmdFam10hIbsFetchLinAddr },
+    { cpumMsrWr_AmdFam10hIbsFetchPhysAddr },
+    { cpumMsrWr_AmdFam10hIbsOpExecCtl },
+    { cpumMsrWr_AmdFam10hIbsOpRip },
+    { cpumMsrWr_AmdFam10hIbsOpData },
+    { cpumMsrWr_AmdFam10hIbsOpData2 },
+    { cpumMsrWr_AmdFam10hIbsOpData3 },
+    { cpumMsrWr_AmdFam10hIbsDcLinAddr },
+    { cpumMsrWr_AmdFam10hIbsDcPhysAddr },
+    { cpumMsrWr_AmdFam10hIbsCtl },
+    { cpumMsrWr_AmdFam14hIbsBrTarget },
+
+    { cpumMsrWr_Gim },
+};
+
+
+/**
+ * Looks up the range for the given MSR.
+ *
+ * @returns Pointer to the range if found, NULL if not.
+ * @param   pVM                The cross context VM structure.
+ * @param   idMsr              The MSR to look up.
+ */
+# ifndef IN_RING3
+static
+# endif
+PCPUMMSRRANGE cpumLookupMsrRange(PVM pVM, uint32_t idMsr)
+{
+    /*
+     * Binary lookup.
+     */
+    uint32_t        cRanges   = RT_MIN(pVM->cpum.s.GuestInfo.cMsrRanges, RT_ELEMENTS(pVM->cpum.s.GuestInfo.aMsrRanges));
+    if (!cRanges)
+        return NULL;
+    PCPUMMSRRANGE   paRanges  = pVM->cpum.s.GuestInfo.aMsrRanges;
+    for (;;)
+    {
+        uint32_t i = cRanges / 2;
+        if (idMsr < paRanges[i].uFirst)
+        {
+            if (i == 0)
+                break;
+            cRanges = i;
+        }
+        else if (idMsr > paRanges[i].uLast)
+        {
+            i++;
+            if (i >= cRanges)
+                break;
+            cRanges -= i;
+            paRanges = &paRanges[i];
+        }
+        else
+        {
+            if (paRanges[i].enmRdFn == kCpumMsrRdFn_MsrAlias)
+                return cpumLookupMsrRange(pVM, paRanges[i].uValue);
+            return &paRanges[i];
+        }
+    }
+
+# ifdef VBOX_STRICT
+    /*
+     * Linear lookup to verify the above binary search.
+     */
+    uint32_t        cLeft = RT_MIN(pVM->cpum.s.GuestInfo.cMsrRanges, RT_ELEMENTS(pVM->cpum.s.GuestInfo.aMsrRanges));
+    PCPUMMSRRANGE   pCur  = pVM->cpum.s.GuestInfo.aMsrRanges;
+    while (cLeft-- > 0)
+    {
+        if (idMsr >= pCur->uFirst && idMsr <= pCur->uLast)
+        {
+            AssertFailed();
+            if (pCur->enmRdFn == kCpumMsrRdFn_MsrAlias)
+                return cpumLookupMsrRange(pVM, pCur->uValue);
+            return pCur;
+        }
+        pCur++;
+    }
+# endif
+    return NULL;
+}
+
+
+/**
+ * Query a guest MSR.
+ *
+ * The caller is responsible for checking privilege if the call is the result of
+ * a RDMSR instruction.  We'll do the rest.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_CPUM_R3_MSR_READ if the MSR read could not be serviced in the
+ *          current context (raw-mode or ring-0).
+ * @retval  VERR_CPUM_RAISE_GP_0 on failure (invalid MSR), the caller is
+ *          expected to take the appropriate actions. @a *puValue is set to 0.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   idMsr               The MSR.
+ * @param   puValue             Where to return the value.
+ *
+ * @remarks This will always return the right values, even when we're in the
+ *          recompiler.
+ */
+VMMDECL(VBOXSTRICTRC) CPUMQueryGuestMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *puValue)
+{
+    *puValue = 0;
+
+    VBOXSTRICTRC    rcStrict;
+    PVM             pVM    = pVCpu->CTX_SUFF(pVM);
+    PCPUMMSRRANGE   pRange = cpumLookupMsrRange(pVM, idMsr);
+    if (pRange)
+    {
+        CPUMMSRRDFN  enmRdFn = (CPUMMSRRDFN)pRange->enmRdFn;
+        AssertReturn(enmRdFn > kCpumMsrRdFn_Invalid && enmRdFn < kCpumMsrRdFn_End, VERR_CPUM_IPE_1);
+
+        PFNCPUMRDMSR pfnRdMsr = g_aCpumRdMsrFns[enmRdFn].pfnRdMsr;
+        AssertReturn(pfnRdMsr, VERR_CPUM_IPE_2);
+
+        STAM_COUNTER_INC(&pRange->cReads);
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReads);
+
+        rcStrict = pfnRdMsr(pVCpu, idMsr, pRange, puValue);
+        if (rcStrict == VINF_SUCCESS)
+            Log2(("CPUM: RDMSR %#x (%s) -> %#llx\n", idMsr, pRange->szName, *puValue));
+        else if (rcStrict == VERR_CPUM_RAISE_GP_0)
+        {
+            Log(("CPUM: RDMSR %#x (%s) -> #GP(0)\n", idMsr, pRange->szName));
+            STAM_COUNTER_INC(&pRange->cGps);
+            STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReadsRaiseGp);
+        }
+#ifndef IN_RING3
+        else if (rcStrict == VINF_CPUM_R3_MSR_READ)
+            Log(("CPUM: RDMSR %#x (%s) -> ring-3\n", idMsr, pRange->szName));
+#endif
+        else
+        {
+            Log(("CPUM: RDMSR %#x (%s) -> rcStrict=%Rrc\n", idMsr, pRange->szName, VBOXSTRICTRC_VAL(rcStrict)));
+            AssertMsgStmt(RT_FAILURE_NP(rcStrict), ("%Rrc idMsr=%#x\n", VBOXSTRICTRC_VAL(rcStrict), idMsr),
+                          rcStrict = VERR_IPE_UNEXPECTED_INFO_STATUS);
+            Assert(rcStrict != VERR_EM_INTERPRETER);
+        }
+    }
+    else
+    {
+        Log(("CPUM: Unknown RDMSR %#x -> #GP(0)\n", idMsr));
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReads);
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrReadsUnknown);
+        rcStrict = VERR_CPUM_RAISE_GP_0;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Writes to a guest MSR.
+ *
+ * The caller is responsible for checking privilege if the call is the result of
+ * a WRMSR instruction.  We'll do the rest.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_CPUM_R3_MSR_WRITE if the MSR write could not be serviced in the
+ *          current context (raw-mode or ring-0).
+ * @retval  VERR_CPUM_RAISE_GP_0 on failure, the caller is expected to take the
+ *          appropriate actions.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR id.
+ * @param   uValue      The value to set.
+ *
+ * @remarks Everyone changing MSR values, including the recompiler, shall do it
+ *          by calling this method.  This makes sure we have current values and
+ *          that we trigger all the right actions when something changes.
+ *
+ *          For performance reasons, this actually isn't entirely true for some
+ *          MSRs when in HM mode.  The code here and in HM must be aware of
+ *          this.
+ */
+VMMDECL(VBOXSTRICTRC) CPUMSetGuestMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t uValue)
+{
+    VBOXSTRICTRC    rcStrict;
+    PVM             pVM    = pVCpu->CTX_SUFF(pVM);
+    PCPUMMSRRANGE   pRange = cpumLookupMsrRange(pVM, idMsr);
+    if (pRange)
+    {
+        STAM_COUNTER_INC(&pRange->cWrites);
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWrites);
+
+        if (!(uValue & pRange->fWrGpMask))
+        {
+            CPUMMSRWRFN  enmWrFn = (CPUMMSRWRFN)pRange->enmWrFn;
+            AssertReturn(enmWrFn > kCpumMsrWrFn_Invalid && enmWrFn < kCpumMsrWrFn_End, VERR_CPUM_IPE_1);
+
+            PFNCPUMWRMSR pfnWrMsr = g_aCpumWrMsrFns[enmWrFn].pfnWrMsr;
+            AssertReturn(pfnWrMsr, VERR_CPUM_IPE_2);
+
+            uint64_t uValueAdjusted = uValue & ~pRange->fWrIgnMask;
+            if (uValueAdjusted != uValue)
+            {
+                STAM_COUNTER_INC(&pRange->cIgnoredBits);
+                STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesToIgnoredBits);
+            }
+
+            rcStrict = pfnWrMsr(pVCpu, idMsr, pRange, uValueAdjusted, uValue);
+            if (rcStrict == VINF_SUCCESS)
+                Log2(("CPUM: WRMSR %#x (%s), %#llx [%#llx]\n", idMsr, pRange->szName, uValueAdjusted, uValue));
+            else if (rcStrict == VERR_CPUM_RAISE_GP_0)
+            {
+                Log(("CPUM: WRMSR %#x (%s), %#llx [%#llx] -> #GP(0)\n", idMsr, pRange->szName, uValueAdjusted, uValue));
+                STAM_COUNTER_INC(&pRange->cGps);
+                STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesRaiseGp);
+            }
+#ifndef IN_RING3
+            else if (rcStrict == VINF_CPUM_R3_MSR_WRITE)
+                Log(("CPUM: WRMSR %#x (%s), %#llx [%#llx] -> ring-3\n", idMsr, pRange->szName, uValueAdjusted, uValue));
+#endif
+            else
+            {
+                Log(("CPUM: WRMSR %#x (%s), %#llx [%#llx] -> rcStrict=%Rrc\n",
+                     idMsr, pRange->szName, uValueAdjusted, uValue, VBOXSTRICTRC_VAL(rcStrict)));
+                AssertMsgStmt(RT_FAILURE_NP(rcStrict), ("%Rrc idMsr=%#x\n", VBOXSTRICTRC_VAL(rcStrict), idMsr),
+                              rcStrict = VERR_IPE_UNEXPECTED_INFO_STATUS);
+                Assert(rcStrict != VERR_EM_INTERPRETER);
+            }
+        }
+        else
+        {
+            Log(("CPUM: WRMSR %#x (%s), %#llx -> #GP(0) - invalid bits %#llx\n",
+                 idMsr, pRange->szName, uValue, uValue & pRange->fWrGpMask));
+            STAM_COUNTER_INC(&pRange->cGps);
+            STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesRaiseGp);
+            rcStrict = VERR_CPUM_RAISE_GP_0;
+        }
+    }
+    else
+    {
+        Log(("CPUM: Unknown WRMSR %#x, %#llx -> #GP(0)\n", idMsr, uValue));
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWrites);
+        STAM_REL_COUNTER_INC(&pVM->cpum.s.cMsrWritesUnknown);
+        rcStrict = VERR_CPUM_RAISE_GP_0;
+    }
+    return rcStrict;
+}
+
+
+#if defined(VBOX_STRICT) && defined(IN_RING3)
+/**
+ * Performs some checks on the static data related to MSRs.
+ *
+ * @returns VINF_SUCCESS on success, error on failure.
+ */
+int cpumR3MsrStrictInitChecks(void)
+{
+#define CPUM_ASSERT_RD_MSR_FN(a_Register) \
+        AssertReturn(g_aCpumRdMsrFns[kCpumMsrRdFn_##a_Register].pfnRdMsr == cpumMsrRd_##a_Register, VERR_CPUM_IPE_2);
+#define CPUM_ASSERT_WR_MSR_FN(a_Register) \
+        AssertReturn(g_aCpumWrMsrFns[kCpumMsrWrFn_##a_Register].pfnWrMsr == cpumMsrWr_##a_Register, VERR_CPUM_IPE_2);
+
+    AssertReturn(g_aCpumRdMsrFns[kCpumMsrRdFn_Invalid].pfnRdMsr == NULL, VERR_CPUM_IPE_2);
+    CPUM_ASSERT_RD_MSR_FN(FixedValue);
+    CPUM_ASSERT_RD_MSR_FN(WriteOnly);
+    CPUM_ASSERT_RD_MSR_FN(Ia32P5McAddr);
+    CPUM_ASSERT_RD_MSR_FN(Ia32P5McType);
+    CPUM_ASSERT_RD_MSR_FN(Ia32TimestampCounter);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PlatformId);
+    CPUM_ASSERT_RD_MSR_FN(Ia32ApicBase);
+    CPUM_ASSERT_RD_MSR_FN(Ia32FeatureControl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32BiosSignId);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SmmMonitorCtl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PmcN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MonitorFilterLineSize);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MPerf);
+    CPUM_ASSERT_RD_MSR_FN(Ia32APerf);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MtrrCap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MtrrPhysBaseN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MtrrPhysMaskN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MtrrFixed);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MtrrDefType);
+    CPUM_ASSERT_RD_MSR_FN(Ia32Pat);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SysEnterCs);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SysEnterEsp);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SysEnterEip);
+    CPUM_ASSERT_RD_MSR_FN(Ia32McgCap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32McgStatus);
+    CPUM_ASSERT_RD_MSR_FN(Ia32McgCtl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32DebugCtl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SmrrPhysBase);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SmrrPhysMask);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PlatformDcaCap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32CpuDcaCap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32Dca0Cap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfEvtSelN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfStatus);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfCtl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32FixedCtrN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfCapabilities);
+    CPUM_ASSERT_RD_MSR_FN(Ia32FixedCtrCtrl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfGlobalStatus);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfGlobalCtrl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PerfGlobalOvfCtrl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32PebsEnable);
+    CPUM_ASSERT_RD_MSR_FN(Ia32ClockModulation);
+    CPUM_ASSERT_RD_MSR_FN(Ia32ThermInterrupt);
+    CPUM_ASSERT_RD_MSR_FN(Ia32ThermStatus);
+    CPUM_ASSERT_RD_MSR_FN(Ia32MiscEnable);
+    CPUM_ASSERT_RD_MSR_FN(Ia32McCtlStatusAddrMiscN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32McNCtl2);
+    CPUM_ASSERT_RD_MSR_FN(Ia32DsArea);
+    CPUM_ASSERT_RD_MSR_FN(Ia32TscDeadline);
+    CPUM_ASSERT_RD_MSR_FN(Ia32X2ApicN);
+    CPUM_ASSERT_RD_MSR_FN(Ia32DebugInterface);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxBasic);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxPinbasedCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxProcbasedCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxExitCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxEntryCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxMisc);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr0Fixed0);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr0Fixed1);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr4Fixed0);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxCr4Fixed1);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxVmcsEnum);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxProcBasedCtls2);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxEptVpidCap);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxTruePinbasedCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxTrueProcbasedCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxTrueExitCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxTrueEntryCtls);
+    CPUM_ASSERT_RD_MSR_FN(Ia32VmxVmFunc);
+    CPUM_ASSERT_RD_MSR_FN(Ia32SpecCtrl);
+    CPUM_ASSERT_RD_MSR_FN(Ia32ArchCapabilities);
+
+    CPUM_ASSERT_RD_MSR_FN(Amd64Efer);
+    CPUM_ASSERT_RD_MSR_FN(Amd64SyscallTarget);
+    CPUM_ASSERT_RD_MSR_FN(Amd64LongSyscallTarget);
+    CPUM_ASSERT_RD_MSR_FN(Amd64CompSyscallTarget);
+    CPUM_ASSERT_RD_MSR_FN(Amd64SyscallFlagMask);
+    CPUM_ASSERT_RD_MSR_FN(Amd64FsBase);
+    CPUM_ASSERT_RD_MSR_FN(Amd64GsBase);
+    CPUM_ASSERT_RD_MSR_FN(Amd64KernelGsBase);
+    CPUM_ASSERT_RD_MSR_FN(Amd64TscAux);
+
+    CPUM_ASSERT_RD_MSR_FN(IntelEblCrPowerOn);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7CoreThreadCount);
+    CPUM_ASSERT_RD_MSR_FN(IntelP4EbcHardPowerOn);
+    CPUM_ASSERT_RD_MSR_FN(IntelP4EbcSoftPowerOn);
+    CPUM_ASSERT_RD_MSR_FN(IntelP4EbcFrequencyId);
+    CPUM_ASSERT_RD_MSR_FN(IntelP6FsbFrequency);
+    CPUM_ASSERT_RD_MSR_FN(IntelPlatformInfo);
+    CPUM_ASSERT_RD_MSR_FN(IntelFlexRatio);
+    CPUM_ASSERT_RD_MSR_FN(IntelPkgCStConfigControl);
+    CPUM_ASSERT_RD_MSR_FN(IntelPmgIoCaptureBase);
+    CPUM_ASSERT_RD_MSR_FN(IntelLastBranchFromToN);
+    CPUM_ASSERT_RD_MSR_FN(IntelLastBranchFromN);
+    CPUM_ASSERT_RD_MSR_FN(IntelLastBranchToN);
+    CPUM_ASSERT_RD_MSR_FN(IntelLastBranchTos);
+    CPUM_ASSERT_RD_MSR_FN(IntelBblCrCtl);
+    CPUM_ASSERT_RD_MSR_FN(IntelBblCrCtl3);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7TemperatureTarget);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7MsrOffCoreResponseN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7MiscPwrMgmt);
+    CPUM_ASSERT_RD_MSR_FN(IntelP6CrN);
+    CPUM_ASSERT_RD_MSR_FN(IntelCpuId1FeatureMaskEcdx);
+    CPUM_ASSERT_RD_MSR_FN(IntelCpuId1FeatureMaskEax);
+    CPUM_ASSERT_RD_MSR_FN(IntelCpuId80000001FeatureMaskEcdx);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyAesNiCtl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7TurboRatioLimit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7LbrSelect);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyErrorControl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7VirtualLegacyWireCap);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7PowerCtl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyPebsNumAlt);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7PebsLdLat);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7PkgCnResidencyN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7CoreCnResidencyN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyVrCurrentConfig);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyVrMiscConfig);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyRaplPowerUnit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyPkgCnIrtlN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SandyPkgC2Residency);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgPowerLimit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgEnergyStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgPerfStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPkgPowerInfo);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramPowerLimit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramEnergyStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramPerfStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplDramPowerInfo);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0PowerLimit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0EnergyStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0Policy);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp0PerfStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp1PowerLimit);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp1EnergyStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7RaplPp1Policy);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpNominal);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpLevel1);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpLevel2);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7IvyConfigTdpControl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7IvyTurboActivationRatio);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfGlobalCtrl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfGlobalStatus);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfGlobalOvfCtrl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfFixedCtrCtrl);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncPerfFixedCtr);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncCBoxConfig);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncArbPerfCtrN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7UncArbPerfEvtSelN);
+    CPUM_ASSERT_RD_MSR_FN(IntelI7SmiCount);
+    CPUM_ASSERT_RD_MSR_FN(IntelCore2EmttmCrTablesN);
+    CPUM_ASSERT_RD_MSR_FN(IntelCore2SmmCStMiscInfo);
+    CPUM_ASSERT_RD_MSR_FN(IntelCore1ExtConfig);
+    CPUM_ASSERT_RD_MSR_FN(IntelCore1DtsCalControl);
+    CPUM_ASSERT_RD_MSR_FN(IntelCore2PeciControl);
+    CPUM_ASSERT_RD_MSR_FN(IntelAtSilvCoreC1Recidency);
+
+    CPUM_ASSERT_RD_MSR_FN(P6LastBranchFromIp);
+    CPUM_ASSERT_RD_MSR_FN(P6LastBranchToIp);
+    CPUM_ASSERT_RD_MSR_FN(P6LastIntFromIp);
+    CPUM_ASSERT_RD_MSR_FN(P6LastIntToIp);
+
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hTscRate);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hLwpCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hLwpCbAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hMc4MiscN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8PerfCtlN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8PerfCtrN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SysCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8HwCr);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8IorrBaseN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8IorrMaskN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8TopOfMemN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8NbCfg1);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8McXcptRedir);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8CpuNameN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8HwThermalCtrl);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SwThermalCtrl);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8FidVidControl);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8FidVidStatus);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8McCtlMaskN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmiOnIoTrapN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmiOnIoTrapCtlSts);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8IntPendingMessage);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmiTriggerIoCycle);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hMmioCfgBaseAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hTrapCtlMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateCurLimit);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateControl);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateStatus);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hPStateN);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hCofVidControl);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hCofVidStatus);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hCStateIoBaseAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hCpuWatchdogTimer);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmmBase);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmmAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmmMask);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8VmCr);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8IgnNe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8SmmCtl);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8VmHSavePa);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hVmLockKey);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hSmmLockKey);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hLocalSmiStatus);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hOsVisWrkIdLength);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hOsVisWrkStatus);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam16hL2IPerfCtlN);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam16hL2IPerfCtrN);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hNorthbridgePerfCtlN);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hNorthbridgePerfCtrN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7MicrocodeCtl);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7ClusterIdMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlStd07hEbax);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlStd06hEcx);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlStd01hEdcx);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8CpuIdCtlExt01hEdcx);
+    CPUM_ASSERT_RD_MSR_FN(AmdK8PatchLevel);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7DebugStatusMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7BHTraceBaseMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7BHTracePtrMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7BHTraceLimitMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7HardwareDebugToolCfgMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7FastFlushCountMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7NodeId);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7DrXAddrMaskN);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7Dr0DataMatchMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7Dr0DataMaskMaybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7LoadStoreCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7InstrCacheCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7DataCacheCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7BusUnitCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdK7DebugCtl2Maybe);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hFpuCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hDecoderCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hBusUnitCfg2);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hCombUnitCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hCombUnitCfg2);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hCombUnitCfg3);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hExecUnitCfg);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam15hLoadStoreCfg2);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsFetchCtl);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsFetchLinAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsFetchPhysAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpExecCtl);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpRip);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpData);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpData2);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsOpData3);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsDcLinAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsDcPhysAddr);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam10hIbsCtl);
+    CPUM_ASSERT_RD_MSR_FN(AmdFam14hIbsBrTarget);
+
+    CPUM_ASSERT_RD_MSR_FN(Gim)
+
+    AssertReturn(g_aCpumWrMsrFns[kCpumMsrWrFn_Invalid].pfnWrMsr == NULL, VERR_CPUM_IPE_2);
+    CPUM_ASSERT_WR_MSR_FN(Ia32P5McAddr);
+    CPUM_ASSERT_WR_MSR_FN(Ia32P5McType);
+    CPUM_ASSERT_WR_MSR_FN(Ia32TimestampCounter);
+    CPUM_ASSERT_WR_MSR_FN(Ia32ApicBase);
+    CPUM_ASSERT_WR_MSR_FN(Ia32FeatureControl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32BiosSignId);
+    CPUM_ASSERT_WR_MSR_FN(Ia32BiosUpdateTrigger);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SmmMonitorCtl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PmcN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MonitorFilterLineSize);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MPerf);
+    CPUM_ASSERT_WR_MSR_FN(Ia32APerf);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MtrrPhysBaseN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MtrrPhysMaskN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MtrrFixed);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MtrrDefType);
+    CPUM_ASSERT_WR_MSR_FN(Ia32Pat);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SysEnterCs);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SysEnterEsp);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SysEnterEip);
+    CPUM_ASSERT_WR_MSR_FN(Ia32McgStatus);
+    CPUM_ASSERT_WR_MSR_FN(Ia32McgCtl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32DebugCtl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SmrrPhysBase);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SmrrPhysMask);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PlatformDcaCap);
+    CPUM_ASSERT_WR_MSR_FN(Ia32Dca0Cap);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfEvtSelN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfStatus);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfCtl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32FixedCtrN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfCapabilities);
+    CPUM_ASSERT_WR_MSR_FN(Ia32FixedCtrCtrl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfGlobalStatus);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfGlobalCtrl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PerfGlobalOvfCtrl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PebsEnable);
+    CPUM_ASSERT_WR_MSR_FN(Ia32ClockModulation);
+    CPUM_ASSERT_WR_MSR_FN(Ia32ThermInterrupt);
+    CPUM_ASSERT_WR_MSR_FN(Ia32ThermStatus);
+    CPUM_ASSERT_WR_MSR_FN(Ia32MiscEnable);
+    CPUM_ASSERT_WR_MSR_FN(Ia32McCtlStatusAddrMiscN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32McNCtl2);
+    CPUM_ASSERT_WR_MSR_FN(Ia32DsArea);
+    CPUM_ASSERT_WR_MSR_FN(Ia32TscDeadline);
+    CPUM_ASSERT_WR_MSR_FN(Ia32X2ApicN);
+    CPUM_ASSERT_WR_MSR_FN(Ia32DebugInterface);
+    CPUM_ASSERT_WR_MSR_FN(Ia32SpecCtrl);
+    CPUM_ASSERT_WR_MSR_FN(Ia32PredCmd);
+    CPUM_ASSERT_WR_MSR_FN(Ia32FlushCmd);
+
+    CPUM_ASSERT_WR_MSR_FN(Amd64Efer);
+    CPUM_ASSERT_WR_MSR_FN(Amd64SyscallTarget);
+    CPUM_ASSERT_WR_MSR_FN(Amd64LongSyscallTarget);
+    CPUM_ASSERT_WR_MSR_FN(Amd64CompSyscallTarget);
+    CPUM_ASSERT_WR_MSR_FN(Amd64SyscallFlagMask);
+    CPUM_ASSERT_WR_MSR_FN(Amd64FsBase);
+    CPUM_ASSERT_WR_MSR_FN(Amd64GsBase);
+    CPUM_ASSERT_WR_MSR_FN(Amd64KernelGsBase);
+    CPUM_ASSERT_WR_MSR_FN(Amd64TscAux);
+
+    CPUM_ASSERT_WR_MSR_FN(IntelEblCrPowerOn);
+    CPUM_ASSERT_WR_MSR_FN(IntelP4EbcHardPowerOn);
+    CPUM_ASSERT_WR_MSR_FN(IntelP4EbcSoftPowerOn);
+    CPUM_ASSERT_WR_MSR_FN(IntelP4EbcFrequencyId);
+    CPUM_ASSERT_WR_MSR_FN(IntelFlexRatio);
+    CPUM_ASSERT_WR_MSR_FN(IntelPkgCStConfigControl);
+    CPUM_ASSERT_WR_MSR_FN(IntelPmgIoCaptureBase);
+    CPUM_ASSERT_WR_MSR_FN(IntelLastBranchFromToN);
+    CPUM_ASSERT_WR_MSR_FN(IntelLastBranchFromN);
+    CPUM_ASSERT_WR_MSR_FN(IntelLastBranchToN);
+    CPUM_ASSERT_WR_MSR_FN(IntelLastBranchTos);
+    CPUM_ASSERT_WR_MSR_FN(IntelBblCrCtl);
+    CPUM_ASSERT_WR_MSR_FN(IntelBblCrCtl3);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7TemperatureTarget);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7MsrOffCoreResponseN);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7MiscPwrMgmt);
+    CPUM_ASSERT_WR_MSR_FN(IntelP6CrN);
+    CPUM_ASSERT_WR_MSR_FN(IntelCpuId1FeatureMaskEcdx);
+    CPUM_ASSERT_WR_MSR_FN(IntelCpuId1FeatureMaskEax);
+    CPUM_ASSERT_WR_MSR_FN(IntelCpuId80000001FeatureMaskEcdx);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyAesNiCtl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7TurboRatioLimit);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7LbrSelect);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyErrorControl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7PowerCtl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyPebsNumAlt);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7PebsLdLat);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyVrCurrentConfig);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyVrMiscConfig);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyPkgCnIrtlN);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7SandyPkgC2Residency);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPkgPowerLimit);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplDramPowerLimit);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp0PowerLimit);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp0Policy);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp1PowerLimit);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7RaplPp1Policy);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7IvyConfigTdpControl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7IvyTurboActivationRatio);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfGlobalCtrl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfGlobalStatus);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfGlobalOvfCtrl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfFixedCtrCtrl);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncPerfFixedCtr);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncArbPerfCtrN);
+    CPUM_ASSERT_WR_MSR_FN(IntelI7UncArbPerfEvtSelN);
+    CPUM_ASSERT_WR_MSR_FN(IntelCore2EmttmCrTablesN);
+    CPUM_ASSERT_WR_MSR_FN(IntelCore2SmmCStMiscInfo);
+    CPUM_ASSERT_WR_MSR_FN(IntelCore1ExtConfig);
+    CPUM_ASSERT_WR_MSR_FN(IntelCore1DtsCalControl);
+    CPUM_ASSERT_WR_MSR_FN(IntelCore2PeciControl);
+
+    CPUM_ASSERT_WR_MSR_FN(P6LastIntFromIp);
+    CPUM_ASSERT_WR_MSR_FN(P6LastIntToIp);
+
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hTscRate);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hLwpCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hLwpCbAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hMc4MiscN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8PerfCtlN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8PerfCtrN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SysCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8HwCr);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8IorrBaseN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8IorrMaskN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8TopOfMemN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8NbCfg1);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8McXcptRedir);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8CpuNameN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8HwThermalCtrl);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SwThermalCtrl);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8FidVidControl);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8McCtlMaskN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmiOnIoTrapN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmiOnIoTrapCtlSts);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8IntPendingMessage);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmiTriggerIoCycle);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hMmioCfgBaseAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hTrapCtlMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hPStateControl);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hPStateStatus);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hPStateN);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hCofVidControl);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hCofVidStatus);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hCStateIoBaseAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hCpuWatchdogTimer);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmmBase);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmmAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmmMask);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8VmCr);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8IgnNe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8SmmCtl);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8VmHSavePa);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hVmLockKey);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hSmmLockKey);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hLocalSmiStatus);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hOsVisWrkIdLength);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hOsVisWrkStatus);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam16hL2IPerfCtlN);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam16hL2IPerfCtrN);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hNorthbridgePerfCtlN);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hNorthbridgePerfCtrN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7MicrocodeCtl);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7ClusterIdMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlStd07hEbax);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlStd06hEcx);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlStd01hEdcx);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8CpuIdCtlExt01hEdcx);
+    CPUM_ASSERT_WR_MSR_FN(AmdK8PatchLoader);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7DebugStatusMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7BHTraceBaseMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7BHTracePtrMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7BHTraceLimitMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7HardwareDebugToolCfgMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7FastFlushCountMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7NodeId);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7DrXAddrMaskN);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7Dr0DataMatchMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7Dr0DataMaskMaybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7LoadStoreCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7InstrCacheCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7DataCacheCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7BusUnitCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdK7DebugCtl2Maybe);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hFpuCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hDecoderCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hBusUnitCfg2);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hCombUnitCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hCombUnitCfg2);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hCombUnitCfg3);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hExecUnitCfg);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam15hLoadStoreCfg2);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsFetchCtl);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsFetchLinAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsFetchPhysAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpExecCtl);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpRip);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpData);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpData2);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsOpData3);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsDcLinAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsDcPhysAddr);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam10hIbsCtl);
+    CPUM_ASSERT_WR_MSR_FN(AmdFam14hIbsBrTarget);
+
+    CPUM_ASSERT_WR_MSR_FN(Gim);
+
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_STRICT && IN_RING3 */
+
+
+/**
+ * Gets the scalable bus frequency.
+ *
+ * The bus frequency is used as a base in several MSRs that gives the CPU and
+ * other frequency ratios.
+ *
+ * @returns Scalable bus frequency in Hz. Will not return CPUM_SBUSFREQ_UNKNOWN.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMDECL(uint64_t) CPUMGetGuestScalableBusFrequency(PVM pVM)
+{
+    uint64_t uFreq = pVM->cpum.s.GuestInfo.uScalableBusFreq;
+    if (uFreq == CPUM_SBUSFREQ_UNKNOWN)
+        uFreq = CPUM_SBUSFREQ_100MHZ;
+    return uFreq;
+}
+
+
+/**
+ * Sets the guest EFER MSR without performing any additional checks.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uOldEfer    The previous EFER MSR value.
+ * @param   uValidEfer  The new, validated EFER MSR value.
+ *
+ * @remarks One would normally call CPUMIsGuestEferMsrWriteValid() before calling
+ *          this function to change the EFER in order to perform an EFER transition.
+ */
+VMMDECL(void) CPUMSetGuestEferMsrNoChecks(PVMCPUCC pVCpu, uint64_t uOldEfer, uint64_t uValidEfer)
+{
+    pVCpu->cpum.s.Guest.msrEFER = uValidEfer;
+
+    /* AMD64 Architecture Programmer's Manual: 15.15 TLB Control; flush the TLB
+       if MSR_K6_EFER_NXE, MSR_K6_EFER_LME or MSR_K6_EFER_LMA are changed. */
+    if (   (uOldEfer                    & (MSR_K6_EFER_NXE | MSR_K6_EFER_LME | MSR_K6_EFER_LMA))
+        != (pVCpu->cpum.s.Guest.msrEFER & (MSR_K6_EFER_NXE | MSR_K6_EFER_LME | MSR_K6_EFER_LMA)))
+    {
+        /// @todo PGMFlushTLB(pVCpu, cr3, true /*fGlobal*/);
+        HMFlushTlb(pVCpu);
+
+        /* Notify PGM about NXE changes. */
+        if (   (uOldEfer                    & MSR_K6_EFER_NXE)
+            != (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_NXE))
+            PGMNotifyNxeChanged(pVCpu, !(uOldEfer & MSR_K6_EFER_NXE));
+    }
+}
+
+
+/**
+ * Checks if a guest PAT MSR write is valid.
+ *
+ * @returns @c true if the PAT bit combination is valid, @c false otherwise.
+ * @param   uValue      The PAT MSR value.
+ */
+VMMDECL(bool) CPUMIsPatMsrValid(uint64_t uValue)
+{
+    for (uint32_t cShift = 0; cShift < 63; cShift += 8)
+    {
+        /* Check all eight bits because the top 5 bits of each byte are reserved. */
+        uint8_t uType = (uint8_t)(uValue >> cShift);
+        if ((uType >= 8) || (uType == 2) || (uType == 3))
+        {
+            Log(("CPUM: Invalid PAT type at %u:%u in IA32_PAT: %#llx (%#llx)\n", cShift + 7, cShift, uValue, uType));
+            return false;
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Validates an EFER MSR write and provides the new, validated EFER MSR.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   uCr0            The CR0 of the CPU corresponding to the EFER MSR.
+ * @param   uOldEfer        Value of the previous EFER MSR on the CPU if any.
+ * @param   uNewEfer        The new EFER MSR value being written.
+ * @param   puValidEfer     Where to store the validated EFER (only updated if
+ *                          this function returns VINF_SUCCESS).
+ */
+VMMDECL(int) CPUMIsGuestEferMsrWriteValid(PVM pVM, uint64_t uCr0, uint64_t uOldEfer, uint64_t uNewEfer, uint64_t *puValidEfer)
+{
+    /* #GP(0) If anything outside the allowed bits is set. */
+    uint64_t fMask = CPUMGetGuestEferMsrValidMask(pVM);
+    if (uNewEfer & ~fMask)
+    {
+        Log(("CPUM: Settings disallowed EFER bit. uNewEfer=%#RX64 fAllowed=%#RX64 -> #GP(0)\n", uNewEfer, fMask));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    /* Check for illegal MSR_K6_EFER_LME transitions: not allowed to change LME if
+       paging is enabled. (AMD Arch. Programmer's Manual Volume 2: Table 14-5) */
+    if (   (uOldEfer & MSR_K6_EFER_LME) != (uNewEfer & MSR_K6_EFER_LME)
+        && (uCr0 & X86_CR0_PG))
+    {
+        Log(("CPUM: Illegal MSR_K6_EFER_LME change: paging is enabled!!\n"));
+        return VERR_CPUM_RAISE_GP_0;
+    }
+
+    /* There are a few more: e.g. MSR_K6_EFER_LMSLE. */
+    AssertMsg(!(uNewEfer & ~(  MSR_K6_EFER_NXE
+                             | MSR_K6_EFER_LME
+                             | MSR_K6_EFER_LMA /* ignored anyway */
+                             | MSR_K6_EFER_SCE
+                             | MSR_K6_EFER_FFXSR
+                             | MSR_K6_EFER_SVME)),
+              ("Unexpected value %#RX64\n", uNewEfer));
+
+    /* Ignore EFER.LMA, it's updated when setting CR0. */
+    fMask &= ~MSR_K6_EFER_LMA;
+
+    *puValidEfer = (uOldEfer & ~fMask) | (uNewEfer & fMask);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the mask of valid EFER bits depending on supported guest-CPU features.
+ *
+ * @returns Mask of valid EFER bits.
+ * @param   pVM     The cross context VM structure.
+ *
+ * @remarks EFER.LMA is included as part of the valid mask. It's not invalid but
+ *          rather a read-only bit.
+ */
+VMMDECL(uint64_t) CPUMGetGuestEferMsrValidMask(PVM pVM)
+{
+    uint32_t const  fExtFeatures = pVM->cpum.s.aGuestCpuIdPatmExt[0].uEax >= 0x80000001
+                                 ? pVM->cpum.s.aGuestCpuIdPatmExt[1].uEdx
+                                 : 0;
+    uint64_t        fMask        = 0;
+    uint64_t const  fIgnoreMask  = MSR_K6_EFER_LMA;
+
+    /* Filter out those bits the guest is allowed to change. (e.g. LMA is read-only) */
+    if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_NX)
+        fMask |= MSR_K6_EFER_NXE;
+    if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
+        fMask |= MSR_K6_EFER_LME;
+    if (fExtFeatures & X86_CPUID_EXT_FEATURE_EDX_SYSCALL)
+        fMask |= MSR_K6_EFER_SCE;
+    if (fExtFeatures & X86_CPUID_AMD_FEATURE_EDX_FFXSR)
+        fMask |= MSR_K6_EFER_FFXSR;
+    if (pVM->cpum.s.GuestFeatures.fSvm)
+        fMask |= MSR_K6_EFER_SVME;
+
+    return (fIgnoreMask | fMask);
+}
+
+
+/**
+ * Fast way for HM to access the MSR_K8_TSC_AUX register.
+ *
+ * @returns The register value.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestTscAux(PVMCPUCC pVCpu)
+{
+    Assert(!(pVCpu->cpum.s.Guest.fExtrn & CPUMCTX_EXTRN_TSC_AUX));
+    return pVCpu->cpum.s.GuestMsrs.msr.TscAux;
+}
+
+
+/**
+ * Fast way for HM to access the MSR_K8_TSC_AUX register.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   uValue  The new value.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(void) CPUMSetGuestTscAux(PVMCPUCC pVCpu, uint64_t uValue)
+{
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_TSC_AUX;
+    pVCpu->cpum.s.GuestMsrs.msr.TscAux = uValue;
+}
+
+
+/**
+ * Fast way for HM to access the IA32_SPEC_CTRL register.
+ *
+ * @returns The register value.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestSpecCtrl(PVMCPUCC pVCpu)
+{
+    return pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl;
+}
+
+
+/**
+ * Fast way for HM to access the IA32_SPEC_CTRL register.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   uValue  The new value.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(void) CPUMSetGuestSpecCtrl(PVMCPUCC pVCpu, uint64_t uValue)
+{
+    pVCpu->cpum.s.GuestMsrs.msr.SpecCtrl = uValue;
+}
+
diff --git a/src/VBox/VMM/VMMAll/CPUMAllRegs.cpp b/src/VBox/VMM/VMMAll/CPUMAllRegs.cpp
new file mode 100644
index 00000000..753aaa71
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/CPUMAllRegs.cpp
@@ -0,0 +1,3039 @@
+/* $Id: CPUMAllRegs.cpp $ */
+/** @file
+ * CPUM - CPU Monitor(/Manager) - Getters and Setters.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_CPUM
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/hm.h>
+#include "CPUMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/dis.h>
+#include <VBox/log.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/tm.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+# include <iprt/asm-amd64-x86.h>
+#endif
+#ifdef IN_RING3
+# include <iprt/thread.h>
+#endif
+
+/** Disable stack frame pointer generation here. */
+#if defined(_MSC_VER) && !defined(DEBUG) && defined(RT_ARCH_X86)
+# pragma optimize("y", off)
+#endif
+
+AssertCompile2MemberOffsets(VM, cpum.s.GuestFeatures, cpum.ro.GuestFeatures);
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/**
+ * Converts a CPUMCPU::Guest pointer into a VMCPU pointer.
+ *
+ * @returns Pointer to the Virtual CPU.
+ * @param   a_pGuestCtx     Pointer to the guest context.
+ */
+#define CPUM_GUEST_CTX_TO_VMCPU(a_pGuestCtx) RT_FROM_MEMBER(a_pGuestCtx, VMCPU, cpum.s.Guest)
+
+/**
+ * Lazily loads the hidden parts of a selector register when using raw-mode.
+ */
+#define CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(a_pVCpu, a_pSReg) \
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(a_pVCpu, a_pSReg))
+
+/** @def CPUM_INT_ASSERT_NOT_EXTRN
+ * Macro for asserting that @a a_fNotExtrn are present.
+ *
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling EMT.
+ * @param   a_fNotExtrn     Mask of CPUMCTX_EXTRN_XXX bits to check.
+ */
+#define CPUM_INT_ASSERT_NOT_EXTRN(a_pVCpu, a_fNotExtrn) \
+    AssertMsg(!((a_pVCpu)->cpum.s.Guest.fExtrn & (a_fNotExtrn)), \
+              ("%#RX64; a_fNotExtrn=%#RX64\n", (a_pVCpu)->cpum.s.Guest.fExtrn, (a_fNotExtrn)))
+
+
+VMMDECL(void) CPUMSetHyperCR3(PVMCPU pVCpu, uint32_t cr3)
+{
+    pVCpu->cpum.s.Hyper.cr3 = cr3;
+}
+
+VMMDECL(uint32_t) CPUMGetHyperCR3(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.cr3;
+}
+
+
+/** @def MAYBE_LOAD_DRx
+ * Macro for updating DRx values in raw-mode and ring-0 contexts.
+ */
+#ifdef IN_RING0
+# define MAYBE_LOAD_DRx(a_pVCpu, a_fnLoad, a_uValue) do { a_fnLoad(a_uValue); } while (0)
+#else
+# define MAYBE_LOAD_DRx(a_pVCpu, a_fnLoad, a_uValue) do { } while (0)
+#endif
+
+VMMDECL(void) CPUMSetHyperDR0(PVMCPU pVCpu, RTGCUINTREG uDr0)
+{
+    pVCpu->cpum.s.Hyper.dr[0] = uDr0;
+    MAYBE_LOAD_DRx(pVCpu, ASMSetDR0, uDr0);
+}
+
+
+VMMDECL(void) CPUMSetHyperDR1(PVMCPU pVCpu, RTGCUINTREG uDr1)
+{
+    pVCpu->cpum.s.Hyper.dr[1] = uDr1;
+    MAYBE_LOAD_DRx(pVCpu, ASMSetDR1, uDr1);
+}
+
+
+VMMDECL(void) CPUMSetHyperDR2(PVMCPU pVCpu, RTGCUINTREG uDr2)
+{
+    pVCpu->cpum.s.Hyper.dr[2] = uDr2;
+    MAYBE_LOAD_DRx(pVCpu, ASMSetDR2, uDr2);
+}
+
+
+VMMDECL(void) CPUMSetHyperDR3(PVMCPU pVCpu, RTGCUINTREG uDr3)
+{
+    pVCpu->cpum.s.Hyper.dr[3] = uDr3;
+    MAYBE_LOAD_DRx(pVCpu, ASMSetDR3, uDr3);
+}
+
+
+VMMDECL(void) CPUMSetHyperDR6(PVMCPU pVCpu, RTGCUINTREG uDr6)
+{
+    pVCpu->cpum.s.Hyper.dr[6] = uDr6;
+}
+
+
+VMMDECL(void) CPUMSetHyperDR7(PVMCPU pVCpu, RTGCUINTREG uDr7)
+{
+    pVCpu->cpum.s.Hyper.dr[7] = uDr7;
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR0(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[0];
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR1(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[1];
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR2(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[2];
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR3(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[3];
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR6(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[6];
+}
+
+
+VMMDECL(RTGCUINTREG) CPUMGetHyperDR7(PVMCPU pVCpu)
+{
+    return pVCpu->cpum.s.Hyper.dr[7];
+}
+
+
+/**
+ * Checks that the special cookie stored in unused reserved RFLAGS bits
+ *
+ * @retval  true if cookie is ok.
+ * @retval  false if cookie is not ok.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) CPUMAssertGuestRFlagsCookie(PVM pVM, PVMCPU pVCpu)
+{
+    AssertLogRelMsgReturn(      (  pVCpu->cpum.s.Guest.rflags.uBoth
+                                 & ~(uint64_t)(CPUMX86EFLAGS_HW_MASK_64 | CPUMX86EFLAGS_INT_MASK_64))
+                             == pVM->cpum.s.fReservedRFlagsCookie
+                          && (pVCpu->cpum.s.Guest.rflags.uBoth & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK
+                          && (pVCpu->cpum.s.Guest.rflags.uBoth & X86_EFL_RAZ_MASK & CPUMX86EFLAGS_HW_MASK_64) == 0,
+                          ("rflags=%#RX64 vs fReservedRFlagsCookie=%#RX64\n",
+                           pVCpu->cpum.s.Guest.rflags.uBoth, pVM->cpum.s.fReservedRFlagsCookie),
+                          false);
+    return true;
+}
+
+
+/**
+ * Queries the pointer to the internal CPUMCTX structure.
+ *
+ * @returns The CPUMCTX pointer.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(PCPUMCTX) CPUMQueryGuestCtxPtr(PVMCPU pVCpu)
+{
+    return &pVCpu->cpum.s.Guest;
+}
+
+
+/**
+ * Queries the pointer to the internal CPUMCTXMSRS structure.
+ *
+ * This is for NEM only.
+ *
+ * @returns The CPUMCTX pointer.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(PCPUMCTXMSRS) CPUMQueryGuestCtxMsrsPtr(PVMCPU pVCpu)
+{
+    return &pVCpu->cpum.s.GuestMsrs;
+}
+
+
+VMMDECL(int) CPUMSetGuestGDTR(PVMCPU pVCpu, uint64_t GCPtrBase, uint16_t cbLimit)
+{
+    pVCpu->cpum.s.Guest.gdtr.cbGdt = cbLimit;
+    pVCpu->cpum.s.Guest.gdtr.pGdt  = GCPtrBase;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_GDTR;
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_GDTR;
+    return VINF_SUCCESS; /* formality, consider it void. */
+}
+
+
+VMMDECL(int) CPUMSetGuestIDTR(PVMCPU pVCpu, uint64_t GCPtrBase, uint16_t cbLimit)
+{
+    pVCpu->cpum.s.Guest.idtr.cbIdt = cbLimit;
+    pVCpu->cpum.s.Guest.idtr.pIdt  = GCPtrBase;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_IDTR;
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_IDTR;
+    return VINF_SUCCESS; /* formality, consider it void. */
+}
+
+
+VMMDECL(int) CPUMSetGuestTR(PVMCPU pVCpu, uint16_t tr)
+{
+    pVCpu->cpum.s.Guest.tr.Sel  = tr;
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_TR;
+    return VINF_SUCCESS; /* formality, consider it void. */
+}
+
+
+VMMDECL(int) CPUMSetGuestLDTR(PVMCPU pVCpu, uint16_t ldtr)
+{
+    pVCpu->cpum.s.Guest.ldtr.Sel      = ldtr;
+    /* The caller will set more hidden bits if it has them. */
+    pVCpu->cpum.s.Guest.ldtr.ValidSel = 0;
+    pVCpu->cpum.s.Guest.ldtr.fFlags   = 0;
+    pVCpu->cpum.s.fChanged  |= CPUM_CHANGED_LDTR;
+    return VINF_SUCCESS; /* formality, consider it void. */
+}
+
+
+/**
+ * Set the guest CR0.
+ *
+ * When called in GC, the hyper CR0 may be updated if that is
+ * required. The caller only has to take special action if AM,
+ * WP, PG or PE changes.
+ *
+ * @returns VINF_SUCCESS (consider it void).
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   cr0     The new CR0 value.
+ */
+VMMDECL(int) CPUMSetGuestCR0(PVMCPUCC pVCpu, uint64_t cr0)
+{
+    /*
+     * Check for changes causing TLB flushes (for REM).
+     * The caller is responsible for calling PGM when appropriate.
+     */
+    if (    (cr0                     & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE))
+        !=  (pVCpu->cpum.s.Guest.cr0 & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE)))
+        pVCpu->cpum.s.fChanged |= CPUM_CHANGED_GLOBAL_TLB_FLUSH;
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CR0;
+
+    /*
+     * Let PGM know if the WP goes from 0 to 1 (netware WP0+RO+US hack)
+     */
+    if (((cr0 ^ pVCpu->cpum.s.Guest.cr0) & X86_CR0_WP) && (cr0 & X86_CR0_WP))
+        PGMCr0WpEnabled(pVCpu);
+
+    /* The ET flag is settable on a 386 and hardwired on 486+. */
+    if (   !(cr0 & X86_CR0_ET)
+        && pVCpu->CTX_SUFF(pVM)->cpum.s.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386)
+        cr0 |= X86_CR0_ET;
+
+    pVCpu->cpum.s.Guest.cr0 = cr0;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR0;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestCR2(PVMCPU pVCpu, uint64_t cr2)
+{
+    pVCpu->cpum.s.Guest.cr2 = cr2;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR2;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestCR3(PVMCPU pVCpu, uint64_t cr3)
+{
+    pVCpu->cpum.s.Guest.cr3 = cr3;
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CR3;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR3;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestCR4(PVMCPU pVCpu, uint64_t cr4)
+{
+    /* Note! We don't bother with OSXSAVE and legacy CPUID patches. */
+
+    if (   (cr4                     & (X86_CR4_PGE | X86_CR4_PAE | X86_CR4_PSE))
+        != (pVCpu->cpum.s.Guest.cr4 & (X86_CR4_PGE | X86_CR4_PAE | X86_CR4_PSE)))
+        pVCpu->cpum.s.fChanged |= CPUM_CHANGED_GLOBAL_TLB_FLUSH;
+
+    pVCpu->cpum.s.fChanged |= CPUM_CHANGED_CR4;
+    pVCpu->cpum.s.Guest.cr4 = cr4;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR4;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEFlags(PVMCPU pVCpu, uint32_t eflags)
+{
+    pVCpu->cpum.s.Guest.eflags.u = eflags;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEIP(PVMCPU pVCpu, uint32_t eip)
+{
+    pVCpu->cpum.s.Guest.eip = eip;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEAX(PVMCPU pVCpu, uint32_t eax)
+{
+    pVCpu->cpum.s.Guest.eax = eax;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEBX(PVMCPU pVCpu, uint32_t ebx)
+{
+    pVCpu->cpum.s.Guest.ebx = ebx;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestECX(PVMCPU pVCpu, uint32_t ecx)
+{
+    pVCpu->cpum.s.Guest.ecx = ecx;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEDX(PVMCPU pVCpu, uint32_t edx)
+{
+    pVCpu->cpum.s.Guest.edx = edx;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestESP(PVMCPU pVCpu, uint32_t esp)
+{
+    pVCpu->cpum.s.Guest.esp = esp;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEBP(PVMCPU pVCpu, uint32_t ebp)
+{
+    pVCpu->cpum.s.Guest.ebp = ebp;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestESI(PVMCPU pVCpu, uint32_t esi)
+{
+    pVCpu->cpum.s.Guest.esi = esi;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestEDI(PVMCPU pVCpu, uint32_t edi)
+{
+    pVCpu->cpum.s.Guest.edi = edi;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestSS(PVMCPU pVCpu, uint16_t ss)
+{
+    pVCpu->cpum.s.Guest.ss.Sel = ss;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestCS(PVMCPU pVCpu, uint16_t cs)
+{
+    pVCpu->cpum.s.Guest.cs.Sel = cs;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestDS(PVMCPU pVCpu, uint16_t ds)
+{
+    pVCpu->cpum.s.Guest.ds.Sel = ds;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestES(PVMCPU pVCpu, uint16_t es)
+{
+    pVCpu->cpum.s.Guest.es.Sel = es;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestFS(PVMCPU pVCpu, uint16_t fs)
+{
+    pVCpu->cpum.s.Guest.fs.Sel = fs;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(int) CPUMSetGuestGS(PVMCPU pVCpu, uint16_t gs)
+{
+    pVCpu->cpum.s.Guest.gs.Sel = gs;
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(void) CPUMSetGuestEFER(PVMCPU pVCpu, uint64_t val)
+{
+    pVCpu->cpum.s.Guest.msrEFER = val;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_EFER;
+}
+
+
+VMMDECL(RTGCPTR) CPUMGetGuestIDTR(PCVMCPU pVCpu, uint16_t *pcbLimit)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_IDTR);
+    if (pcbLimit)
+        *pcbLimit = pVCpu->cpum.s.Guest.idtr.cbIdt;
+    return pVCpu->cpum.s.Guest.idtr.pIdt;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestTR(PCVMCPU pVCpu, PCPUMSELREGHID pHidden)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_TR);
+    if (pHidden)
+        *pHidden = pVCpu->cpum.s.Guest.tr;
+    return pVCpu->cpum.s.Guest.tr.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestCS(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CS);
+    return pVCpu->cpum.s.Guest.cs.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestDS(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DS);
+    return pVCpu->cpum.s.Guest.ds.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestES(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_ES);
+    return pVCpu->cpum.s.Guest.es.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestFS(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_FS);
+    return pVCpu->cpum.s.Guest.fs.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestGS(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_GS);
+    return pVCpu->cpum.s.Guest.gs.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestSS(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_SS);
+    return pVCpu->cpum.s.Guest.ss.Sel;
+}
+
+
+VMMDECL(uint64_t)   CPUMGetGuestFlatPC(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
+    CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs);
+    if (   !CPUMIsGuestInLongMode(pVCpu)
+        || !pVCpu->cpum.s.Guest.cs.Attr.n.u1Long)
+        return pVCpu->cpum.s.Guest.eip + (uint32_t)pVCpu->cpum.s.Guest.cs.u64Base;
+    return pVCpu->cpum.s.Guest.rip + pVCpu->cpum.s.Guest.cs.u64Base;
+}
+
+
+VMMDECL(uint64_t)   CPUMGetGuestFlatSP(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
+    CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.ss);
+    if (   !CPUMIsGuestInLongMode(pVCpu)
+        || !pVCpu->cpum.s.Guest.cs.Attr.n.u1Long)
+        return pVCpu->cpum.s.Guest.eip + (uint32_t)pVCpu->cpum.s.Guest.ss.u64Base;
+    return pVCpu->cpum.s.Guest.rip + pVCpu->cpum.s.Guest.ss.u64Base;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestLDTR(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_LDTR);
+    return pVCpu->cpum.s.Guest.ldtr.Sel;
+}
+
+
+VMMDECL(RTSEL) CPUMGetGuestLdtrEx(PCVMCPU pVCpu, uint64_t *pGCPtrBase, uint32_t *pcbLimit)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_LDTR);
+    *pGCPtrBase = pVCpu->cpum.s.Guest.ldtr.u64Base;
+    *pcbLimit   = pVCpu->cpum.s.Guest.ldtr.u32Limit;
+    return pVCpu->cpum.s.Guest.ldtr.Sel;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestCR0(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return pVCpu->cpum.s.Guest.cr0;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestCR2(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR2);
+    return pVCpu->cpum.s.Guest.cr2;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestCR3(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR3);
+    return pVCpu->cpum.s.Guest.cr3;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestCR4(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4);
+    return pVCpu->cpum.s.Guest.cr4;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestCR8(PCVMCPUCC pVCpu)
+{
+    uint64_t u64;
+    int rc = CPUMGetGuestCRx(pVCpu, DISCREG_CR8, &u64);
+    if (RT_FAILURE(rc))
+        u64 = 0;
+    return u64;
+}
+
+
+VMMDECL(void) CPUMGetGuestGDTR(PCVMCPU pVCpu, PVBOXGDTR pGDTR)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_GDTR);
+    *pGDTR = pVCpu->cpum.s.Guest.gdtr;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEIP(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP);
+    return pVCpu->cpum.s.Guest.eip;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestRIP(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP);
+    return pVCpu->cpum.s.Guest.rip;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEAX(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RAX);
+    return pVCpu->cpum.s.Guest.eax;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEBX(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RBX);
+    return pVCpu->cpum.s.Guest.ebx;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestECX(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RCX);
+    return pVCpu->cpum.s.Guest.ecx;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEDX(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RDX);
+    return pVCpu->cpum.s.Guest.edx;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestESI(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RSI);
+    return pVCpu->cpum.s.Guest.esi;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEDI(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RDI);
+    return pVCpu->cpum.s.Guest.edi;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestESP(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RSP);
+    return pVCpu->cpum.s.Guest.esp;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEBP(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RBP);
+    return pVCpu->cpum.s.Guest.ebp;
+}
+
+
+VMMDECL(uint32_t) CPUMGetGuestEFlags(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RFLAGS);
+    return pVCpu->cpum.s.Guest.eflags.u;
+}
+
+
+VMMDECL(int) CPUMGetGuestCRx(PCVMCPUCC pVCpu, unsigned iReg, uint64_t *pValue)
+{
+    switch (iReg)
+    {
+        case DISCREG_CR0:
+            CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+            *pValue = pVCpu->cpum.s.Guest.cr0;
+            break;
+
+        case DISCREG_CR2:
+            CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR2);
+            *pValue = pVCpu->cpum.s.Guest.cr2;
+            break;
+
+        case DISCREG_CR3:
+            CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR3);
+            *pValue = pVCpu->cpum.s.Guest.cr3;
+            break;
+
+        case DISCREG_CR4:
+            CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4);
+            *pValue = pVCpu->cpum.s.Guest.cr4;
+            break;
+
+        case DISCREG_CR8:
+        {
+            CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
+            uint8_t u8Tpr;
+            int rc = APICGetTpr(pVCpu, &u8Tpr, NULL /* pfPending */, NULL /* pu8PendingIrq */);
+            if (RT_FAILURE(rc))
+            {
+                AssertMsg(rc == VERR_PDM_NO_APIC_INSTANCE, ("%Rrc\n", rc));
+                *pValue = 0;
+                return rc;
+            }
+            *pValue = u8Tpr >> 4; /* bits 7-4 contain the task priority that go in cr8, bits 3-0 */
+            break;
+        }
+
+        default:
+            return VERR_INVALID_PARAMETER;
+    }
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR0(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+    return pVCpu->cpum.s.Guest.dr[0];
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR1(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+    return pVCpu->cpum.s.Guest.dr[1];
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR2(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+    return pVCpu->cpum.s.Guest.dr[2];
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR3(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+    return pVCpu->cpum.s.Guest.dr[3];
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR6(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR6);
+    return pVCpu->cpum.s.Guest.dr[6];
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestDR7(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR7);
+    return pVCpu->cpum.s.Guest.dr[7];
+}
+
+
+VMMDECL(int) CPUMGetGuestDRx(PCVMCPU pVCpu, uint32_t iReg, uint64_t *pValue)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR_MASK);
+    AssertReturn(iReg <= DISDREG_DR7, VERR_INVALID_PARAMETER);
+    /* DR4 is an alias for DR6, and DR5 is an alias for DR7. */
+    if (iReg == 4 || iReg == 5)
+        iReg += 2;
+    *pValue = pVCpu->cpum.s.Guest.dr[iReg];
+    return VINF_SUCCESS;
+}
+
+
+VMMDECL(uint64_t) CPUMGetGuestEFER(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_EFER);
+    return pVCpu->cpum.s.Guest.msrEFER;
+}
+
+
+/**
+ * Looks up a CPUID leaf in the CPUID leaf array, no subleaf.
+ *
+ * @returns Pointer to the leaf if found, NULL if not.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   uLeaf               The leaf to get.
+ */
+PCPUMCPUIDLEAF cpumCpuIdGetLeaf(PVM pVM, uint32_t uLeaf)
+{
+    unsigned            iEnd     = RT_MIN(pVM->cpum.s.GuestInfo.cCpuIdLeaves, RT_ELEMENTS(pVM->cpum.s.GuestInfo.aCpuIdLeaves));
+    if (iEnd)
+    {
+        unsigned        iStart   = 0;
+        PCPUMCPUIDLEAF  paLeaves = pVM->cpum.s.GuestInfo.aCpuIdLeaves;
+        for (;;)
+        {
+            unsigned i = iStart + (iEnd - iStart) / 2U;
+            if (uLeaf < paLeaves[i].uLeaf)
+            {
+                if (i <= iStart)
+                    return NULL;
+                iEnd = i;
+            }
+            else if (uLeaf > paLeaves[i].uLeaf)
+            {
+                i += 1;
+                if (i >= iEnd)
+                    return NULL;
+                iStart = i;
+            }
+            else
+            {
+                if (RT_LIKELY(paLeaves[i].fSubLeafMask == 0 && paLeaves[i].uSubLeaf == 0))
+                    return &paLeaves[i];
+
+                /* This shouldn't normally happen. But in case the it does due
+                   to user configuration overrids or something, just return the
+                   first sub-leaf. */
+                AssertMsgFailed(("uLeaf=%#x fSubLeafMask=%#x uSubLeaf=%#x\n",
+                                 uLeaf, paLeaves[i].fSubLeafMask, paLeaves[i].uSubLeaf));
+                while (   paLeaves[i].uSubLeaf != 0
+                       && i > 0
+                       && uLeaf == paLeaves[i - 1].uLeaf)
+                    i--;
+                return &paLeaves[i];
+            }
+        }
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Looks up a CPUID leaf in the CPUID leaf array.
+ *
+ * @returns Pointer to the leaf if found, NULL if not.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   uLeaf               The leaf to get.
+ * @param   uSubLeaf            The subleaf, if applicable.  Just pass 0 if it
+ *                              isn't.
+ * @param   pfExactSubLeafHit   Whether we've got an exact subleaf hit or not.
+ */
+PCPUMCPUIDLEAF cpumCpuIdGetLeafEx(PVM pVM, uint32_t uLeaf, uint32_t uSubLeaf, bool *pfExactSubLeafHit)
+{
+    unsigned            iEnd     = RT_MIN(pVM->cpum.s.GuestInfo.cCpuIdLeaves, RT_ELEMENTS(pVM->cpum.s.GuestInfo.aCpuIdLeaves));
+    if (iEnd)
+    {
+        unsigned        iStart   = 0;
+        PCPUMCPUIDLEAF  paLeaves = pVM->cpum.s.GuestInfo.aCpuIdLeaves;
+        for (;;)
+        {
+            unsigned i = iStart + (iEnd - iStart) / 2U;
+            if (uLeaf < paLeaves[i].uLeaf)
+            {
+                if (i <= iStart)
+                    return NULL;
+                iEnd = i;
+            }
+            else if (uLeaf > paLeaves[i].uLeaf)
+            {
+                i += 1;
+                if (i >= iEnd)
+                    return NULL;
+                iStart = i;
+            }
+            else
+            {
+                uSubLeaf &= paLeaves[i].fSubLeafMask;
+                if (uSubLeaf == paLeaves[i].uSubLeaf)
+                    *pfExactSubLeafHit = true;
+                else
+                {
+                    /* Find the right subleaf.  We return the last one before
+                       uSubLeaf if we don't find an exact match. */
+                    if (uSubLeaf < paLeaves[i].uSubLeaf)
+                        while (   i > 0
+                               && uLeaf    == paLeaves[i - 1].uLeaf
+                               && uSubLeaf <= paLeaves[i - 1].uSubLeaf)
+                            i--;
+                    else
+                        while (   i + 1 < pVM->cpum.s.GuestInfo.cCpuIdLeaves
+                               && uLeaf    == paLeaves[i + 1].uLeaf
+                               && uSubLeaf >= paLeaves[i + 1].uSubLeaf)
+                            i++;
+                    *pfExactSubLeafHit = uSubLeaf == paLeaves[i].uSubLeaf;
+                }
+                return &paLeaves[i];
+            }
+        }
+    }
+
+    *pfExactSubLeafHit = false;
+    return NULL;
+}
+
+
+/**
+ * Gets a CPUID leaf.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uLeaf       The CPUID leaf to get.
+ * @param   uSubLeaf    The CPUID sub-leaf to get, if applicable.
+ * @param   f64BitMode  A tristate indicate if the caller is in 64-bit mode or
+ *                      not: 1=true, 0=false, 1=whatever.  This affect how the
+ *                      X86_CPUID_EXT_FEATURE_EDX_SYSCALL flag is returned on
+ *                      Intel CPUs, where it's only returned in 64-bit mode.
+ * @param   pEax        Where to store the EAX value.
+ * @param   pEbx        Where to store the EBX value.
+ * @param   pEcx        Where to store the ECX value.
+ * @param   pEdx        Where to store the EDX value.
+ */
+VMMDECL(void) CPUMGetGuestCpuId(PVMCPUCC pVCpu, uint32_t uLeaf, uint32_t uSubLeaf, int f64BitMode,
+                                uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx)
+{
+    bool            fExactSubLeafHit;
+    PVM             pVM   = pVCpu->CTX_SUFF(pVM);
+    PCCPUMCPUIDLEAF pLeaf = cpumCpuIdGetLeafEx(pVM, uLeaf, uSubLeaf, &fExactSubLeafHit);
+    if (pLeaf)
+    {
+        AssertMsg(pLeaf->uLeaf == uLeaf, ("%#x %#x\n", pLeaf->uLeaf, uLeaf));
+        if (fExactSubLeafHit)
+        {
+            *pEax = pLeaf->uEax;
+            *pEbx = pLeaf->uEbx;
+            *pEcx = pLeaf->uEcx;
+            *pEdx = pLeaf->uEdx;
+
+            /*
+             * Deal with CPU specific information.
+             */
+            if (pLeaf->fFlags & (  CPUMCPUIDLEAF_F_CONTAINS_APIC_ID
+                                 | CPUMCPUIDLEAF_F_CONTAINS_OSXSAVE
+                                 | CPUMCPUIDLEAF_F_CONTAINS_APIC ))
+            {
+                if (uLeaf == 1)
+                {
+                    /* EBX: Bits 31-24: Initial APIC ID. */
+                    Assert(pVCpu->idCpu <= 255);
+                    AssertMsg((pLeaf->uEbx >> 24) == 0, ("%#x\n", pLeaf->uEbx)); /* raw-mode assumption */
+                    *pEbx = (pLeaf->uEbx & UINT32_C(0x00ffffff)) | (pVCpu->idCpu << 24);
+
+                    /* EDX: Bit 9: AND with APICBASE.EN. */
+                    if (!pVCpu->cpum.s.fCpuIdApicFeatureVisible && (pLeaf->fFlags & CPUMCPUIDLEAF_F_CONTAINS_APIC))
+                        *pEdx &= ~X86_CPUID_FEATURE_EDX_APIC;
+
+                    /* ECX: Bit 27: CR4.OSXSAVE mirror. */
+                    *pEcx = (pLeaf->uEcx & ~X86_CPUID_FEATURE_ECX_OSXSAVE)
+                          | (pVCpu->cpum.s.Guest.cr4 & X86_CR4_OSXSAVE ? X86_CPUID_FEATURE_ECX_OSXSAVE : 0);
+                }
+                else if (uLeaf == 0xb)
+                {
+                    /* EDX: Initial extended APIC ID. */
+                    AssertMsg(pLeaf->uEdx == 0, ("%#x\n", pLeaf->uEdx)); /* raw-mode assumption */
+                    *pEdx = pVCpu->idCpu;
+                    Assert(!(pLeaf->fFlags & ~(CPUMCPUIDLEAF_F_CONTAINS_APIC_ID | CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES)));
+                }
+                else if (uLeaf == UINT32_C(0x8000001e))
+                {
+                    /* EAX: Initial extended APIC ID. */
+                    AssertMsg(pLeaf->uEax == 0, ("%#x\n", pLeaf->uEax)); /* raw-mode assumption */
+                    *pEax = pVCpu->idCpu;
+                    Assert(!(pLeaf->fFlags & ~CPUMCPUIDLEAF_F_CONTAINS_APIC_ID));
+                }
+                else if (uLeaf == UINT32_C(0x80000001))
+                {
+                    /* EDX: Bit 9: AND with APICBASE.EN. */
+                    if (!pVCpu->cpum.s.fCpuIdApicFeatureVisible)
+                        *pEdx &= ~X86_CPUID_AMD_FEATURE_EDX_APIC;
+                    Assert(!(pLeaf->fFlags & ~CPUMCPUIDLEAF_F_CONTAINS_APIC));
+                }
+                else
+                    AssertMsgFailed(("uLeaf=%#x\n", uLeaf));
+            }
+
+            /* Intel CPUs supresses the SYSCALL bit when not executing in 64-bit mode: */
+            if (   uLeaf == UINT32_C(0x80000001)
+                && f64BitMode == false
+                && (*pEdx & X86_CPUID_EXT_FEATURE_EDX_SYSCALL)
+                && (   pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_INTEL
+                    || pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_VIA /*?*/
+                    || pVM->cpum.s.GuestFeatures.enmCpuVendor == CPUMCPUVENDOR_SHANGHAI /*?*/ ) )
+                *pEdx &= ~X86_CPUID_EXT_FEATURE_EDX_SYSCALL;
+
+        }
+        /*
+         * Out of range sub-leaves aren't quite as easy and pretty as we emulate
+         * them here, but we do the best we can here...
+         */
+        else
+        {
+            *pEax = *pEbx = *pEcx = *pEdx = 0;
+            if (pLeaf->fFlags & CPUMCPUIDLEAF_F_INTEL_TOPOLOGY_SUBLEAVES)
+            {
+                *pEcx = uSubLeaf & 0xff;
+                *pEdx = pVCpu->idCpu;
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Different CPUs have different ways of dealing with unknown CPUID leaves.
+         */
+        switch (pVM->cpum.s.GuestInfo.enmUnknownCpuIdMethod)
+        {
+            default:
+                AssertFailed();
+                RT_FALL_THRU();
+            case CPUMUNKNOWNCPUID_DEFAULTS:
+            case CPUMUNKNOWNCPUID_LAST_STD_LEAF: /* ASSUME this is executed */
+            case CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX: /** @todo Implement CPUMUNKNOWNCPUID_LAST_STD_LEAF_WITH_ECX */
+                *pEax = pVM->cpum.s.GuestInfo.DefCpuId.uEax;
+                *pEbx = pVM->cpum.s.GuestInfo.DefCpuId.uEbx;
+                *pEcx = pVM->cpum.s.GuestInfo.DefCpuId.uEcx;
+                *pEdx = pVM->cpum.s.GuestInfo.DefCpuId.uEdx;
+                break;
+            case CPUMUNKNOWNCPUID_PASSTHRU:
+                *pEax = uLeaf;
+                *pEbx = 0;
+                *pEcx = uSubLeaf;
+                *pEdx = 0;
+                break;
+        }
+    }
+    Log2(("CPUMGetGuestCpuId: uLeaf=%#010x/%#010x %RX32 %RX32 %RX32 %RX32\n", uLeaf, uSubLeaf, *pEax, *pEbx, *pEcx, *pEdx));
+}
+
+
+/**
+ * Sets the visibility of the X86_CPUID_FEATURE_EDX_APIC and
+ * X86_CPUID_AMD_FEATURE_EDX_APIC CPUID bits.
+ *
+ * @returns Previous value.
+ * @param   pVCpu       The cross context virtual CPU structure to make the
+ *                      change on.  Usually the calling EMT.
+ * @param   fVisible    Whether to make it visible (true) or hide it (false).
+ *
+ * @remarks This is "VMMDECL" so that it still links with
+ *          the old APIC code which is in VBoxDD2 and not in
+ *          the VMM module.
+ */
+VMMDECL(bool) CPUMSetGuestCpuIdPerCpuApicFeature(PVMCPU pVCpu, bool fVisible)
+{
+    bool fOld = pVCpu->cpum.s.fCpuIdApicFeatureVisible;
+    pVCpu->cpum.s.fCpuIdApicFeatureVisible = fVisible;
+    return fOld;
+}
+
+
+/**
+ * Gets the host CPU vendor.
+ *
+ * @returns CPU vendor.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(CPUMCPUVENDOR) CPUMGetHostCpuVendor(PVM pVM)
+{
+    return (CPUMCPUVENDOR)pVM->cpum.s.HostFeatures.enmCpuVendor;
+}
+
+
+/**
+ * Gets the host CPU microarchitecture.
+ *
+ * @returns CPU microarchitecture.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(CPUMMICROARCH) CPUMGetHostMicroarch(PCVM pVM)
+{
+    return pVM->cpum.s.HostFeatures.enmMicroarch;
+}
+
+
+/**
+ * Gets the guest CPU vendor.
+ *
+ * @returns CPU vendor.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(CPUMCPUVENDOR) CPUMGetGuestCpuVendor(PVM pVM)
+{
+    return (CPUMCPUVENDOR)pVM->cpum.s.GuestFeatures.enmCpuVendor;
+}
+
+
+/**
+ * Gets the guest CPU microarchitecture.
+ *
+ * @returns CPU microarchitecture.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(CPUMMICROARCH) CPUMGetGuestMicroarch(PCVM pVM)
+{
+    return pVM->cpum.s.GuestFeatures.enmMicroarch;
+}
+
+
+/**
+ * Gets the maximum number of physical and linear address bits supported by the
+ * guest.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pcPhysAddrWidth     Where to store the physical address width.
+ * @param   pcLinearAddrWidth   Where to store the linear address width.
+ */
+VMMDECL(void) CPUMGetGuestAddrWidths(PCVM pVM, uint8_t *pcPhysAddrWidth, uint8_t *pcLinearAddrWidth)
+{
+    AssertPtr(pVM);
+    AssertReturnVoid(pcPhysAddrWidth);
+    AssertReturnVoid(pcLinearAddrWidth);
+    *pcPhysAddrWidth   = pVM->cpum.s.GuestFeatures.cMaxPhysAddrWidth;
+    *pcLinearAddrWidth = pVM->cpum.s.GuestFeatures.cMaxLinearAddrWidth;
+}
+
+
+VMMDECL(int) CPUMSetGuestDR0(PVMCPUCC pVCpu, uint64_t uDr0)
+{
+    pVCpu->cpum.s.Guest.dr[0] = uDr0;
+    return CPUMRecalcHyperDRx(pVCpu, 0);
+}
+
+
+VMMDECL(int) CPUMSetGuestDR1(PVMCPUCC pVCpu, uint64_t uDr1)
+{
+    pVCpu->cpum.s.Guest.dr[1] = uDr1;
+    return CPUMRecalcHyperDRx(pVCpu, 1);
+}
+
+
+VMMDECL(int) CPUMSetGuestDR2(PVMCPUCC pVCpu, uint64_t uDr2)
+{
+    pVCpu->cpum.s.Guest.dr[2] = uDr2;
+    return CPUMRecalcHyperDRx(pVCpu, 2);
+}
+
+
+VMMDECL(int) CPUMSetGuestDR3(PVMCPUCC pVCpu, uint64_t uDr3)
+{
+    pVCpu->cpum.s.Guest.dr[3] = uDr3;
+    return CPUMRecalcHyperDRx(pVCpu, 3);
+}
+
+
+VMMDECL(int) CPUMSetGuestDR6(PVMCPU pVCpu, uint64_t uDr6)
+{
+    pVCpu->cpum.s.Guest.dr[6] = uDr6;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_DR6;
+    return VINF_SUCCESS; /* No need to recalc. */
+}
+
+
+VMMDECL(int) CPUMSetGuestDR7(PVMCPUCC pVCpu, uint64_t uDr7)
+{
+    pVCpu->cpum.s.Guest.dr[7] = uDr7;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_DR7;
+    return CPUMRecalcHyperDRx(pVCpu, 7);
+}
+
+
+VMMDECL(int) CPUMSetGuestDRx(PVMCPUCC pVCpu, uint32_t iReg, uint64_t Value)
+{
+    AssertReturn(iReg <= DISDREG_DR7, VERR_INVALID_PARAMETER);
+    /* DR4 is an alias for DR6, and DR5 is an alias for DR7. */
+    if (iReg == 4 || iReg == 5)
+        iReg += 2;
+    pVCpu->cpum.s.Guest.dr[iReg] = Value;
+    return CPUMRecalcHyperDRx(pVCpu, iReg);
+}
+
+
+/**
+ * Recalculates the hypervisor DRx register values based on current guest
+ * registers and DBGF breakpoints, updating changed registers depending on the
+ * context.
+ *
+ * This is called whenever a guest DRx register is modified (any context) and
+ * when DBGF sets a hardware breakpoint (ring-3 only, rendezvous).
+ *
+ * In raw-mode context this function will reload any (hyper) DRx registers which
+ * comes out with a different value.  It may also have to save the host debug
+ * registers if that haven't been done already.  In this context though, we'll
+ * be intercepting and emulating all DRx accesses, so the hypervisor DRx values
+ * are only important when breakpoints are actually enabled.
+ *
+ * In ring-0 (HM) context DR0-3 will be relocated by us, while DR7 will be
+ * reloaded by the HM code if it changes.  Further more, we will only use the
+ * combined register set when the VBox debugger is actually using hardware BPs,
+ * when it isn't we'll keep the guest DR0-3 + (maybe) DR6 loaded (DR6 doesn't
+ * concern us here).
+ *
+ * In ring-3 we won't be loading anything, so well calculate hypervisor values
+ * all the time.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iGstReg     The guest debug register number that was modified.
+ *                      UINT8_MAX if not guest register.
+ */
+VMMDECL(int) CPUMRecalcHyperDRx(PVMCPUCC pVCpu, uint8_t iGstReg)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+#ifndef IN_RING0
+    RT_NOREF_PV(iGstReg);
+#endif
+
+    /*
+     * Compare the DR7s first.
+     *
+     * We only care about the enabled flags.  GD is virtualized when we
+     * dispatch the #DB, we never enable it.  The DBGF DR7 value is will
+     * always have the LE and GE bits set, so no need to check and disable
+     * stuff if they're cleared like we have to for the guest DR7.
+     */
+    RTGCUINTREG uGstDr7 = CPUMGetGuestDR7(pVCpu);
+    /** @todo This isn't correct. BPs work without setting LE and GE under AMD-V.  They are also documented as unsupported by P6+. */
+    if (!(uGstDr7 & (X86_DR7_LE | X86_DR7_GE)))
+        uGstDr7 = 0;
+    else if (!(uGstDr7 & X86_DR7_LE))
+        uGstDr7 &= ~X86_DR7_LE_ALL;
+    else if (!(uGstDr7 & X86_DR7_GE))
+        uGstDr7 &= ~X86_DR7_GE_ALL;
+
+    const RTGCUINTREG uDbgfDr7 = DBGFBpGetDR7(pVM);
+    if ((uGstDr7 | uDbgfDr7) & X86_DR7_ENABLED_MASK)
+    {
+        Assert(!CPUMIsGuestDebugStateActive(pVCpu));
+
+        /*
+         * Ok, something is enabled.  Recalc each of the breakpoints, taking
+         * the VM debugger ones of the guest ones.  In raw-mode context we will
+         * not allow breakpoints with values inside the hypervisor area.
+         */
+        RTGCUINTREG uNewDr7 = X86_DR7_GE | X86_DR7_LE | X86_DR7_RA1_MASK;
+
+        /* bp 0 */
+        RTGCUINTREG uNewDr0;
+        if (uDbgfDr7 & (X86_DR7_L0 | X86_DR7_G0))
+        {
+            uNewDr7 |= uDbgfDr7 & (X86_DR7_L0 | X86_DR7_G0 | X86_DR7_RW0_MASK | X86_DR7_LEN0_MASK);
+            uNewDr0 = DBGFBpGetDR0(pVM);
+        }
+        else if (uGstDr7 & (X86_DR7_L0 | X86_DR7_G0))
+        {
+            uNewDr0 = CPUMGetGuestDR0(pVCpu);
+            uNewDr7 |= uGstDr7 & (X86_DR7_L0 | X86_DR7_G0 | X86_DR7_RW0_MASK | X86_DR7_LEN0_MASK);
+        }
+        else
+            uNewDr0 = 0;
+
+        /* bp 1 */
+        RTGCUINTREG uNewDr1;
+        if (uDbgfDr7 & (X86_DR7_L1 | X86_DR7_G1))
+        {
+            uNewDr7 |= uDbgfDr7 & (X86_DR7_L1 | X86_DR7_G1 | X86_DR7_RW1_MASK | X86_DR7_LEN1_MASK);
+            uNewDr1 = DBGFBpGetDR1(pVM);
+        }
+        else if (uGstDr7 & (X86_DR7_L1 | X86_DR7_G1))
+        {
+            uNewDr1 = CPUMGetGuestDR1(pVCpu);
+            uNewDr7 |= uGstDr7 & (X86_DR7_L1 | X86_DR7_G1 | X86_DR7_RW1_MASK | X86_DR7_LEN1_MASK);
+        }
+        else
+            uNewDr1 = 0;
+
+        /* bp 2 */
+        RTGCUINTREG uNewDr2;
+        if (uDbgfDr7 & (X86_DR7_L2 | X86_DR7_G2))
+        {
+            uNewDr7 |= uDbgfDr7 & (X86_DR7_L2 | X86_DR7_G2 | X86_DR7_RW2_MASK | X86_DR7_LEN2_MASK);
+            uNewDr2 = DBGFBpGetDR2(pVM);
+        }
+        else if (uGstDr7 & (X86_DR7_L2 | X86_DR7_G2))
+        {
+            uNewDr2 = CPUMGetGuestDR2(pVCpu);
+            uNewDr7 |= uGstDr7 & (X86_DR7_L2 | X86_DR7_G2 | X86_DR7_RW2_MASK | X86_DR7_LEN2_MASK);
+        }
+        else
+            uNewDr2 = 0;
+
+        /* bp 3 */
+        RTGCUINTREG uNewDr3;
+        if (uDbgfDr7 & (X86_DR7_L3 | X86_DR7_G3))
+        {
+            uNewDr7 |= uDbgfDr7 & (X86_DR7_L3 | X86_DR7_G3 | X86_DR7_RW3_MASK | X86_DR7_LEN3_MASK);
+            uNewDr3 = DBGFBpGetDR3(pVM);
+        }
+        else if (uGstDr7 & (X86_DR7_L3 | X86_DR7_G3))
+        {
+            uNewDr3 = CPUMGetGuestDR3(pVCpu);
+            uNewDr7 |= uGstDr7 & (X86_DR7_L3 | X86_DR7_G3 | X86_DR7_RW3_MASK | X86_DR7_LEN3_MASK);
+        }
+        else
+            uNewDr3 = 0;
+
+        /*
+         * Apply the updates.
+         */
+        pVCpu->cpum.s.fUseFlags |= CPUM_USE_DEBUG_REGS_HYPER;
+        if (uNewDr3 != pVCpu->cpum.s.Hyper.dr[3])
+            CPUMSetHyperDR3(pVCpu, uNewDr3);
+        if (uNewDr2 != pVCpu->cpum.s.Hyper.dr[2])
+            CPUMSetHyperDR2(pVCpu, uNewDr2);
+        if (uNewDr1 != pVCpu->cpum.s.Hyper.dr[1])
+            CPUMSetHyperDR1(pVCpu, uNewDr1);
+        if (uNewDr0 != pVCpu->cpum.s.Hyper.dr[0])
+            CPUMSetHyperDR0(pVCpu, uNewDr0);
+        if (uNewDr7 != pVCpu->cpum.s.Hyper.dr[7])
+            CPUMSetHyperDR7(pVCpu, uNewDr7);
+    }
+#ifdef IN_RING0
+    else if (CPUMIsGuestDebugStateActive(pVCpu))
+    {
+        /*
+         * Reload the register that was modified.  Normally this won't happen
+         * as we won't intercept DRx writes when not having the hyper debug
+         * state loaded, but in case we do for some reason we'll simply deal
+         * with it.
+         */
+        switch (iGstReg)
+        {
+            case 0: ASMSetDR0(CPUMGetGuestDR0(pVCpu)); break;
+            case 1: ASMSetDR1(CPUMGetGuestDR1(pVCpu)); break;
+            case 2: ASMSetDR2(CPUMGetGuestDR2(pVCpu)); break;
+            case 3: ASMSetDR3(CPUMGetGuestDR3(pVCpu)); break;
+            default:
+                AssertReturn(iGstReg != UINT8_MAX, VERR_INTERNAL_ERROR_3);
+        }
+    }
+#endif
+    else
+    {
+        /*
+         * No active debug state any more.  In raw-mode this means we have to
+         * make sure DR7 has everything disabled now, if we armed it already.
+         * In ring-0 we might end up here when just single stepping.
+         */
+#ifdef IN_RING0
+        if (pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_HYPER)
+        {
+            if (pVCpu->cpum.s.Hyper.dr[0])
+                ASMSetDR0(0);
+            if (pVCpu->cpum.s.Hyper.dr[1])
+                ASMSetDR1(0);
+            if (pVCpu->cpum.s.Hyper.dr[2])
+                ASMSetDR2(0);
+            if (pVCpu->cpum.s.Hyper.dr[3])
+                ASMSetDR3(0);
+            pVCpu->cpum.s.fUseFlags &= ~CPUM_USED_DEBUG_REGS_HYPER;
+        }
+#endif
+        pVCpu->cpum.s.fUseFlags &= ~CPUM_USE_DEBUG_REGS_HYPER;
+
+        /* Clear all the registers. */
+        pVCpu->cpum.s.Hyper.dr[7] = X86_DR7_RA1_MASK;
+        pVCpu->cpum.s.Hyper.dr[3] = 0;
+        pVCpu->cpum.s.Hyper.dr[2] = 0;
+        pVCpu->cpum.s.Hyper.dr[1] = 0;
+        pVCpu->cpum.s.Hyper.dr[0] = 0;
+
+    }
+    Log2(("CPUMRecalcHyperDRx: fUseFlags=%#x %RGr %RGr %RGr %RGr  %RGr %RGr\n",
+          pVCpu->cpum.s.fUseFlags, pVCpu->cpum.s.Hyper.dr[0], pVCpu->cpum.s.Hyper.dr[1],
+          pVCpu->cpum.s.Hyper.dr[2], pVCpu->cpum.s.Hyper.dr[3], pVCpu->cpum.s.Hyper.dr[6],
+          pVCpu->cpum.s.Hyper.dr[7]));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Set the guest XCR0 register.
+ *
+ * Will load additional state if the FPU state is already loaded (in ring-0 &
+ * raw-mode context).
+ *
+ * @returns VINF_SUCCESS on success, VERR_CPUM_RAISE_GP_0 on invalid input
+ *          value.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uNewValue   The new value.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(int)   CPUMSetGuestXcr0(PVMCPUCC pVCpu, uint64_t uNewValue)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_XCRx);
+    if (   (uNewValue & ~pVCpu->CTX_SUFF(pVM)->cpum.s.fXStateGuestMask) == 0
+        /* The X87 bit cannot be cleared. */
+        && (uNewValue & XSAVE_C_X87)
+        /* AVX requires SSE. */
+        && (uNewValue & (XSAVE_C_SSE | XSAVE_C_YMM)) != XSAVE_C_YMM
+        /* AVX-512 requires YMM, SSE and all of its three components to be enabled. */
+        && (   (uNewValue & (XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI)) == 0
+            ||    (uNewValue & (XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI))
+               ==              (XSAVE_C_SSE | XSAVE_C_YMM | XSAVE_C_OPMASK | XSAVE_C_ZMM_HI256 | XSAVE_C_ZMM_16HI) )
+       )
+    {
+        pVCpu->cpum.s.Guest.aXcr[0] = uNewValue;
+
+        /* If more state components are enabled, we need to take care to load
+           them if the FPU/SSE state is already loaded.  May otherwise leak
+           host state to the guest. */
+        uint64_t fNewComponents = ~pVCpu->cpum.s.Guest.fXStateMask & uNewValue;
+        if (fNewComponents)
+        {
+#ifdef IN_RING0
+            if (pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST)
+            {
+                if (pVCpu->cpum.s.Guest.fXStateMask != 0)
+                    /* Adding more components. */
+                    ASMXRstor(&pVCpu->cpum.s.Guest.XState, fNewComponents);
+                else
+                {
+                    /* We're switching from FXSAVE/FXRSTOR to XSAVE/XRSTOR. */
+                    pVCpu->cpum.s.Guest.fXStateMask |= XSAVE_C_X87 | XSAVE_C_SSE;
+                    if (uNewValue & ~(XSAVE_C_X87 | XSAVE_C_SSE))
+                        ASMXRstor(&pVCpu->cpum.s.Guest.XState, uNewValue & ~(XSAVE_C_X87 | XSAVE_C_SSE));
+                }
+            }
+#endif
+            pVCpu->cpum.s.Guest.fXStateMask |= uNewValue;
+        }
+        return VINF_SUCCESS;
+    }
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * Tests if the guest has No-Execute Page Protection Enabled (NXE).
+ *
+ * @returns true if in real mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestNXEnabled(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_EFER);
+    return !!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_NXE);
+}
+
+
+/**
+ * Tests if the guest has the Page Size Extension enabled (PSE).
+ *
+ * @returns true if in real mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestPageSizeExtEnabled(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4);
+    /* PAE or AMD64 implies support for big pages regardless of CR4.PSE */
+    return !!(pVCpu->cpum.s.Guest.cr4 & (X86_CR4_PSE | X86_CR4_PAE));
+}
+
+
+/**
+ * Tests if the guest has the paging enabled (PG).
+ *
+ * @returns true if in real mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestPagingEnabled(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return !!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PG);
+}
+
+
+/**
+ * Tests if the guest has the paging enabled (PG).
+ *
+ * @returns true if in real mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestR0WriteProtEnabled(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return !!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_WP);
+}
+
+
+/**
+ * Tests if the guest is running in real mode or not.
+ *
+ * @returns true if in real mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInRealMode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return !(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE);
+}
+
+
+/**
+ * Tests if the guest is running in real or virtual 8086 mode.
+ *
+ * @returns @c true if it is, @c false if not.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInRealOrV86Mode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS);
+    return !(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE)
+        || pVCpu->cpum.s.Guest.eflags.Bits.u1VM; /** @todo verify that this cannot be set in long mode. */
+}
+
+
+/**
+ * Tests if the guest is running in protected or not.
+ *
+ * @returns true if in protected mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInProtectedMode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return !!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE);
+}
+
+
+/**
+ * Tests if the guest is running in paged protected or not.
+ *
+ * @returns true if in paged protected mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInPagedProtectedMode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0);
+    return (pVCpu->cpum.s.Guest.cr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG);
+}
+
+
+/**
+ * Tests if the guest is running in long mode or not.
+ *
+ * @returns true if in long mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInLongMode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_EFER);
+    return (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA) == MSR_K6_EFER_LMA;
+}
+
+
+/**
+ * Tests if the guest is running in PAE mode or not.
+ *
+ * @returns true if in PAE mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestInPAEMode(PCVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER);
+    /* Intel mentions EFER.LMA and EFER.LME in different parts of their spec. We shall use EFER.LMA rather
+       than EFER.LME as it reflects if the CPU has entered paging with EFER.LME set.  */
+    return (pVCpu->cpum.s.Guest.cr4 & X86_CR4_PAE)
+        && (pVCpu->cpum.s.Guest.cr0 & X86_CR0_PG)
+        && !(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA);
+}
+
+
+/**
+ * Tests if the guest is running in 64 bits mode or not.
+ *
+ * @returns true if in 64 bits protected mode, otherwise false.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(bool) CPUMIsGuestIn64BitCode(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
+    if (!CPUMIsGuestInLongMode(pVCpu))
+        return false;
+    CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs);
+    return pVCpu->cpum.s.Guest.cs.Attr.n.u1Long;
+}
+
+
+/**
+ * Helper for CPUMIsGuestIn64BitCodeEx that handles lazy resolving of hidden CS
+ * registers.
+ *
+ * @returns true if in 64 bits protected mode, otherwise false.
+ * @param   pCtx        Pointer to the current guest CPU context.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestIn64BitCodeSlow(PCPUMCTX pCtx)
+{
+    return CPUMIsGuestIn64BitCode(CPUM_GUEST_CTX_TO_VMCPU(pCtx));
+}
+
+
+/**
+ * Sets the specified changed flags (CPUM_CHANGED_*).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   fChangedAdd The changed flags to add.
+ */
+VMMDECL(void) CPUMSetChangedFlags(PVMCPU pVCpu, uint32_t fChangedAdd)
+{
+    pVCpu->cpum.s.fChanged |= fChangedAdd;
+}
+
+
+/**
+ * Checks if the CPU supports the XSAVE and XRSTOR instruction.
+ *
+ * @returns true if supported.
+ * @returns false if not supported.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(bool) CPUMSupportsXSave(PVM pVM)
+{
+    return pVM->cpum.s.HostFeatures.fXSaveRstor != 0;
+}
+
+
+/**
+ * Checks if the host OS uses the SYSENTER / SYSEXIT instructions.
+ * @returns true if used.
+ * @returns false if not used.
+ * @param   pVM       The cross context VM structure.
+ */
+VMMDECL(bool) CPUMIsHostUsingSysEnter(PVM pVM)
+{
+    return RT_BOOL(pVM->cpum.s.fHostUseFlags & CPUM_USE_SYSENTER);
+}
+
+
+/**
+ * Checks if the host OS uses the SYSCALL / SYSRET instructions.
+ * @returns true if used.
+ * @returns false if not used.
+ * @param   pVM       The cross context VM structure.
+ */
+VMMDECL(bool) CPUMIsHostUsingSysCall(PVM pVM)
+{
+    return RT_BOOL(pVM->cpum.s.fHostUseFlags & CPUM_USE_SYSCALL);
+}
+
+
+/**
+ * Checks if we activated the FPU/XMM state of the guest OS.
+ *
+ * Obsolete: This differs from CPUMIsGuestFPUStateLoaded() in that it refers to
+ * the next time we'll be executing guest code, so it may return true for
+ * 64-on-32 when we still haven't actually loaded the FPU status, just scheduled
+ * it to be loaded the next time we go thru the world switcher
+ * (CPUM_SYNC_FPU_STATE).
+ *
+ * @returns true / false.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestFPUStateActive(PVMCPU pVCpu)
+{
+    bool fRet = RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST);
+    AssertMsg(fRet == pVCpu->cpum.s.Guest.fUsedFpuGuest, ("fRet=%d\n", fRet));
+    return fRet;
+}
+
+
+/**
+ * Checks if we've really loaded the FPU/XMM state of the guest OS.
+ *
+ * @returns true / false.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsGuestFPUStateLoaded(PVMCPU pVCpu)
+{
+    bool fRet = RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_GUEST);
+    AssertMsg(fRet == pVCpu->cpum.s.Guest.fUsedFpuGuest, ("fRet=%d\n", fRet));
+    return fRet;
+}
+
+
+/**
+ * Checks if we saved the FPU/XMM state of the host OS.
+ *
+ * @returns true / false.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMDECL(bool) CPUMIsHostFPUStateSaved(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_FPU_HOST);
+}
+
+
+/**
+ * Checks if the guest debug state is active.
+ *
+ * @returns boolean
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(bool) CPUMIsGuestDebugStateActive(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_GUEST);
+}
+
+
+/**
+ * Checks if the hyper debug state is active.
+ *
+ * @returns boolean
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(bool) CPUMIsHyperDebugStateActive(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->cpum.s.fUseFlags & CPUM_USED_DEBUG_REGS_HYPER);
+}
+
+
+/**
+ * Mark the guest's debug state as inactive.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @todo    This API doesn't make sense any more.
+ */
+VMMDECL(void) CPUMDeactivateGuestDebugState(PVMCPU pVCpu)
+{
+    Assert(!(pVCpu->cpum.s.fUseFlags & (CPUM_USED_DEBUG_REGS_GUEST | CPUM_USED_DEBUG_REGS_HYPER | CPUM_USED_DEBUG_REGS_HOST)));
+    NOREF(pVCpu);
+}
+
+
+/**
+ * Get the current privilege level of the guest.
+ *
+ * @returns CPL
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(uint32_t) CPUMGetGuestCPL(PVMCPU pVCpu)
+{
+    /*
+     * CPL can reliably be found in SS.DPL (hidden regs valid) or SS if not.
+     *
+     * Note! We used to check CS.DPL here, assuming it was always equal to
+     * CPL even if a conforming segment was loaded.  But this turned out to
+     * only apply to older AMD-V.  With VT-x we had an ACP2 regression
+     * during install after a far call to ring 2 with VT-x.  Then on newer
+     * AMD-V CPUs we have to move the VMCB.guest.u8CPL into cs.Attr.n.u2Dpl
+     * as well as ss.Attr.n.u2Dpl to make this (and other) code work right.
+     *
+     * So, forget CS.DPL, always use SS.DPL.
+     *
+     * Note! The SS RPL is always equal to the CPL, while the CS RPL
+     * isn't necessarily equal if the segment is conforming.
+     * See section 4.11.1 in the AMD manual.
+     *
+     * Update: Where the heck does it say CS.RPL can differ from CPL other than
+     *         right after real->prot mode switch and when in V8086 mode?  That
+     *         section says the RPL specified in a direct transfere (call, jmp,
+     *         ret) is not the one loaded into CS. Besides, if CS.RPL != CPL
+     *         it would be impossible for an exception handle or the iret
+     *         instruction to figure out whether SS:ESP are part of the frame
+     *         or not.  VBox or qemu bug must've lead to this misconception.
+     *
+     * Update2: On an AMD bulldozer system here, I've no trouble loading a null
+     *         selector into SS with an RPL other than the CPL when CPL != 3 and
+     *         we're in 64-bit mode.  The intel dev box doesn't allow this, on
+     *         RPL = CPL.  Weird.
+     */
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
+    uint32_t uCpl;
+    if (pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE)
+    {
+        if (!pVCpu->cpum.s.Guest.eflags.Bits.u1VM)
+        {
+            if (CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.s.Guest.ss))
+                uCpl = pVCpu->cpum.s.Guest.ss.Attr.n.u2Dpl;
+            else
+                uCpl = (pVCpu->cpum.s.Guest.ss.Sel & X86_SEL_RPL);
+        }
+        else
+            uCpl = 3; /* V86 has CPL=3; REM doesn't set DPL=3 in V8086 mode. See @bugref{5130}. */
+    }
+    else
+        uCpl = 0;     /* Real mode is zero; CPL set to 3 for VT-x real-mode emulation. */
+    return uCpl;
+}
+
+
+/**
+ * Gets the current guest CPU mode.
+ *
+ * If paging mode is what you need, check out PGMGetGuestMode().
+ *
+ * @returns The CPU mode.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(CPUMMODE) CPUMGetGuestMode(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER);
+    CPUMMODE enmMode;
+    if (!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE))
+        enmMode = CPUMMODE_REAL;
+    else if (!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA))
+        enmMode = CPUMMODE_PROTECTED;
+    else
+        enmMode = CPUMMODE_LONG;
+
+    return enmMode;
+}
+
+
+/**
+ * Figure whether the CPU is currently executing 16, 32 or 64 bit code.
+ *
+ * @returns 16, 32 or 64.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(uint32_t)       CPUMGetGuestCodeBits(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS);
+
+    if (!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE))
+        return 16;
+
+    if (pVCpu->cpum.s.Guest.eflags.Bits.u1VM)
+    {
+        Assert(!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA));
+        return 16;
+    }
+
+    CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs);
+    if (   pVCpu->cpum.s.Guest.cs.Attr.n.u1Long
+        && (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA))
+        return 64;
+
+    if (pVCpu->cpum.s.Guest.cs.Attr.n.u1DefBig)
+        return 32;
+
+    return 16;
+}
+
+
+VMMDECL(DISCPUMODE)     CPUMGetGuestDisMode(PVMCPU pVCpu)
+{
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS);
+
+    if (!(pVCpu->cpum.s.Guest.cr0 & X86_CR0_PE))
+        return DISCPUMODE_16BIT;
+
+    if (pVCpu->cpum.s.Guest.eflags.Bits.u1VM)
+    {
+        Assert(!(pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA));
+        return DISCPUMODE_16BIT;
+    }
+
+    CPUMSELREG_LAZY_LOAD_HIDDEN_PARTS(pVCpu, &pVCpu->cpum.s.Guest.cs);
+    if (   pVCpu->cpum.s.Guest.cs.Attr.n.u1Long
+        && (pVCpu->cpum.s.Guest.msrEFER & MSR_K6_EFER_LMA))
+        return DISCPUMODE_64BIT;
+
+    if (pVCpu->cpum.s.Guest.cs.Attr.n.u1DefBig)
+        return DISCPUMODE_32BIT;
+
+    return DISCPUMODE_16BIT;
+}
+
+
+/**
+ * Gets the guest MXCSR_MASK value.
+ *
+ * This does not access the x87 state, but the value we determined at VM
+ * initialization.
+ *
+ * @returns MXCSR mask.
+ * @param   pVM                 The cross context VM structure.
+ */
+VMMDECL(uint32_t) CPUMGetGuestMxCsrMask(PVM pVM)
+{
+    return pVM->cpum.s.GuestInfo.fMxCsrMask;
+}
+
+
+/**
+ * Returns whether the guest has physical interrupts enabled.
+ *
+ * @returns @c true if interrupts are enabled, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks Warning! This function does -not- take into account the global-interrupt
+ *          flag (GIF).
+ */
+VMM_INT_DECL(bool) CPUMIsGuestPhysIntrEnabled(PVMCPU pVCpu)
+{
+    switch (CPUMGetGuestInNestedHwvirtMode(&pVCpu->cpum.s.Guest))
+    {
+        case CPUMHWVIRT_NONE:
+        default:
+            return pVCpu->cpum.s.Guest.eflags.Bits.u1IF;
+        case CPUMHWVIRT_VMX:
+            return CPUMIsGuestVmxPhysIntrEnabled(&pVCpu->cpum.s.Guest);
+        case CPUMHWVIRT_SVM:
+            return CPUMIsGuestSvmPhysIntrEnabled(pVCpu, &pVCpu->cpum.s.Guest);
+    }
+}
+
+
+/**
+ * Returns whether the nested-guest has virtual interrupts enabled.
+ *
+ * @returns @c true if interrupts are enabled, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remarks Warning! This function does -not- take into account the global-interrupt
+ *          flag (GIF).
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVirtIntrEnabled(PVMCPU pVCpu)
+{
+    PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    Assert(CPUMIsGuestInNestedHwvirtMode(pCtx));
+
+    if (CPUMIsGuestInVmxNonRootMode(pCtx))
+        return CPUMIsGuestVmxVirtIntrEnabled(pCtx);
+
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
+    return CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
+}
+
+
+/**
+ * Calculates the interruptiblity of the guest.
+ *
+ * @returns Interruptibility level.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(CPUMINTERRUPTIBILITY) CPUMGetGuestInterruptibility(PVMCPU pVCpu)
+{
+#if 1
+    /* Global-interrupt flag blocks pretty much everything we care about here. */
+    if (CPUMGetGuestGif(&pVCpu->cpum.s.Guest))
+    {
+        /*
+         * Physical interrupts are primarily blocked using EFLAGS. However, we cannot access
+         * it directly here. If and how EFLAGS are used depends on the context (nested-guest
+         * or raw-mode). Hence we use the function below which handles the details.
+         */
+        if (   !(pVCpu->cpum.s.Guest.eflags.uBoth & CPUMCTX_INHIBIT_ALL_MASK)
+            || (   !(pVCpu->cpum.s.Guest.eflags.uBoth & CPUMCTX_INHIBIT_NMI)
+                && pVCpu->cpum.s.Guest.uRipInhibitInt != pVCpu->cpum.s.Guest.rip))
+        {
+            /** @todo OPT: this next call should be inlined! */
+            if (CPUMIsGuestPhysIntrEnabled(pVCpu))
+            {
+                /** @todo OPT: type this out as it repeats tests. */
+                if (   !CPUMIsGuestInNestedHwvirtMode(&pVCpu->cpum.s.Guest)
+                    || CPUMIsGuestVirtIntrEnabled(pVCpu))
+                    return CPUMINTERRUPTIBILITY_UNRESTRAINED;
+
+                /* Physical interrupts are enabled, but nested-guest virtual interrupts are disabled. */
+                return CPUMINTERRUPTIBILITY_VIRT_INT_DISABLED;
+            }
+            return CPUMINTERRUPTIBILITY_INT_DISABLED;
+        }
+
+        /*
+         * Blocking the delivery of NMIs during an interrupt shadow is CPU implementation
+         * specific. Therefore, in practice, we can't deliver an NMI in an interrupt shadow.
+         * However, there is some uncertainity regarding the converse, i.e. whether
+         * NMI-blocking until IRET blocks delivery of physical interrupts.
+         *
+         * See Intel spec. 25.4.1 "Event Blocking".
+         */
+        /** @todo r=bird: The above comment mixes up VMX root-mode and non-root. Section
+         *        25.4.1 is only applicable to VMX non-root mode.  In root mode /
+         *        non-VMX mode, I have not see any evidence in the intel manuals that
+         *        NMIs are not blocked when in an interrupt shadow. Section "6.7
+         *        NONMASKABLE INTERRUPT (NMI)" in SDM 3A seems pretty clear to me.
+         */
+        if (!(pVCpu->cpum.s.Guest.eflags.uBoth & CPUMCTX_INHIBIT_NMI))
+            return CPUMINTERRUPTIBILITY_INT_INHIBITED;
+        return CPUMINTERRUPTIBILITY_NMI_INHIBIT;
+    }
+    return CPUMINTERRUPTIBILITY_GLOBAL_INHIBIT;
+#else
+    if (pVCpu->cpum.s.Guest.rflags.Bits.u1IF)
+    {
+        if (pVCpu->cpum.s.Guest.hwvirt.fGif)
+        {
+            if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_BLOCK_NMIS | VMCPU_FF_INHIBIT_INTERRUPTS))
+                return CPUMINTERRUPTIBILITY_UNRESTRAINED;
+
+            /** @todo does blocking NMIs mean interrupts are also inhibited? */
+            if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
+            {
+                if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+                    return CPUMINTERRUPTIBILITY_INT_INHIBITED;
+                return CPUMINTERRUPTIBILITY_NMI_INHIBIT;
+            }
+            AssertFailed();
+            return CPUMINTERRUPTIBILITY_NMI_INHIBIT;
+        }
+        return CPUMINTERRUPTIBILITY_GLOBAL_INHIBIT;
+    }
+    else
+    {
+        if (pVCpu->cpum.s.Guest.hwvirt.fGif)
+        {
+            if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+                return CPUMINTERRUPTIBILITY_NMI_INHIBIT;
+            return CPUMINTERRUPTIBILITY_INT_DISABLED;
+        }
+        return CPUMINTERRUPTIBILITY_GLOBAL_INHIBIT;
+    }
+#endif
+}
+
+
+/**
+ * Checks whether the SVM nested-guest has physical interrupts enabled.
+ *
+ * @returns true if interrupts are enabled, false otherwise.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtx    The guest-CPU context.
+ *
+ * @remarks This does -not- take into account the global-interrupt flag.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestSvmPhysIntrEnabled(PCVMCPU pVCpu, PCCPUMCTX pCtx)
+{
+    /** @todo Optimization: Avoid this function call and use a pointer to the
+     *        relevant eflags instead (setup during VMRUN instruction emulation). */
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
+
+    X86EFLAGS fEFlags;
+    if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, pCtx))
+        fEFlags.u = pCtx->hwvirt.svm.HostState.rflags.u;
+    else
+        fEFlags.u = pCtx->eflags.u;
+
+    return fEFlags.Bits.u1IF;
+}
+
+
+/**
+ * Checks whether the SVM nested-guest is in a state to receive virtual (setup
+ * for injection by VMRUN instruction) interrupts.
+ *
+ * @returns VBox status code.
+ * @retval  true if it's ready, false otherwise.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtx    The guest-CPU context.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestSvmVirtIntrEnabled(PCVMCPU pVCpu, PCCPUMCTX pCtx)
+{
+    RT_NOREF(pVCpu);
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
+
+    PCSVMVMCBCTRL pVmcbCtrl    = &pCtx->hwvirt.svm.Vmcb.ctrl;
+    PCSVMINTCTRL  pVmcbIntCtrl = &pVmcbCtrl->IntCtrl;
+    Assert(!pVmcbIntCtrl->n.u1VGifEnable);      /* We don't support passing virtual-GIF feature to the guest yet. */
+    if (   !pVmcbIntCtrl->n.u1IgnoreTPR
+        &&  pVmcbIntCtrl->n.u4VIntrPrio <= pVmcbIntCtrl->n.u8VTPR)
+        return false;
+
+    return RT_BOOL(pCtx->eflags.u & X86_EFL_IF);
+}
+
+
+/**
+ * Gets the pending SVM nested-guest interruptvector.
+ *
+ * @returns The nested-guest interrupt to inject.
+ * @param   pCtx            The guest-CPU context.
+ */
+VMM_INT_DECL(uint8_t) CPUMGetGuestSvmVirtIntrVector(PCCPUMCTX pCtx)
+{
+    return pCtx->hwvirt.svm.Vmcb.ctrl.IntCtrl.n.u8VIntrVector;
+}
+
+
+/**
+ * Restores the host-state from the host-state save area as part of a \#VMEXIT.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtx        The guest-CPU context.
+ */
+VMM_INT_DECL(void) CPUMSvmVmExitRestoreHostState(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    /*
+     * Reload the guest's "host state".
+     */
+    PSVMHOSTSTATE pHostState = &pCtx->hwvirt.svm.HostState;
+    pCtx->es         = pHostState->es;
+    pCtx->cs         = pHostState->cs;
+    pCtx->ss         = pHostState->ss;
+    pCtx->ds         = pHostState->ds;
+    pCtx->gdtr       = pHostState->gdtr;
+    pCtx->idtr       = pHostState->idtr;
+    CPUMSetGuestEferMsrNoChecks(pVCpu, pCtx->msrEFER, pHostState->uEferMsr);
+    CPUMSetGuestCR0(pVCpu, pHostState->uCr0 | X86_CR0_PE);
+    pCtx->cr3        = pHostState->uCr3;
+    CPUMSetGuestCR4(pVCpu, pHostState->uCr4);
+    pCtx->rflags.u   = pHostState->rflags.u;
+    pCtx->rflags.Bits.u1VM = 0;
+    pCtx->rip        = pHostState->uRip;
+    pCtx->rsp        = pHostState->uRsp;
+    pCtx->rax        = pHostState->uRax;
+    pCtx->dr[7]     &= ~(X86_DR7_ENABLED_MASK | X86_DR7_RAZ_MASK | X86_DR7_MBZ_MASK);
+    pCtx->dr[7]     |= X86_DR7_RA1_MASK;
+    Assert(pCtx->ss.Attr.n.u2Dpl == 0);
+
+    /** @todo if RIP is not canonical or outside the CS segment limit, we need to
+     *        raise \#GP(0) in the guest. */
+
+    /** @todo check the loaded host-state for consistency. Figure out what
+     *        exactly this involves? */
+}
+
+
+/**
+ * Saves the host-state to the host-state save area as part of a VMRUN.
+ *
+ * @param   pCtx        The guest-CPU context.
+ * @param   cbInstr     The length of the VMRUN instruction in bytes.
+ */
+VMM_INT_DECL(void) CPUMSvmVmRunSaveHostState(PCPUMCTX pCtx, uint8_t cbInstr)
+{
+    PSVMHOSTSTATE pHostState = &pCtx->hwvirt.svm.HostState;
+    pHostState->es       = pCtx->es;
+    pHostState->cs       = pCtx->cs;
+    pHostState->ss       = pCtx->ss;
+    pHostState->ds       = pCtx->ds;
+    pHostState->gdtr     = pCtx->gdtr;
+    pHostState->idtr     = pCtx->idtr;
+    pHostState->uEferMsr = pCtx->msrEFER;
+    pHostState->uCr0     = pCtx->cr0;
+    pHostState->uCr3     = pCtx->cr3;
+    pHostState->uCr4     = pCtx->cr4;
+    pHostState->rflags.u = pCtx->rflags.u;
+    pHostState->uRip     = pCtx->rip + cbInstr;
+    pHostState->uRsp     = pCtx->rsp;
+    pHostState->uRax     = pCtx->rax;
+}
+
+
+/**
+ * Applies the TSC offset of a nested-guest if any and returns the TSC value for the
+ * nested-guest.
+ *
+ * @returns The TSC offset after applying any nested-guest TSC offset.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uTscValue   The guest TSC.
+ *
+ * @sa      CPUMRemoveNestedGuestTscOffset.
+ */
+VMM_INT_DECL(uint64_t) CPUMApplyNestedGuestTscOffset(PCVMCPU pVCpu, uint64_t uTscValue)
+{
+    PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    if (CPUMIsGuestInVmxNonRootMode(pCtx))
+    {
+        if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_USE_TSC_OFFSETTING))
+            return uTscValue + pCtx->hwvirt.vmx.Vmcs.u64TscOffset.u;
+        return uTscValue;
+    }
+
+    if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+    {
+        uint64_t offTsc;
+        if (!HMGetGuestSvmTscOffset(pVCpu, &offTsc))
+            offTsc = pCtx->hwvirt.svm.Vmcb.ctrl.u64TSCOffset;
+        return uTscValue + offTsc;
+    }
+    return uTscValue;
+}
+
+
+/**
+ * Removes the TSC offset of a nested-guest if any and returns the TSC value for the
+ * guest.
+ *
+ * @returns The TSC offset after removing any nested-guest TSC offset.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uTscValue   The nested-guest TSC.
+ *
+ * @sa      CPUMApplyNestedGuestTscOffset.
+ */
+VMM_INT_DECL(uint64_t) CPUMRemoveNestedGuestTscOffset(PCVMCPU pVCpu, uint64_t uTscValue)
+{
+    PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    if (CPUMIsGuestInVmxNonRootMode(pCtx))
+    {
+        if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_USE_TSC_OFFSETTING))
+            return uTscValue - pCtx->hwvirt.vmx.Vmcs.u64TscOffset.u;
+        return uTscValue;
+    }
+
+    if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
+    {
+        uint64_t offTsc;
+        if (!HMGetGuestSvmTscOffset(pVCpu, &offTsc))
+            offTsc = pCtx->hwvirt.svm.Vmcb.ctrl.u64TSCOffset;
+        return uTscValue - offTsc;
+    }
+    return uTscValue;
+}
+
+
+/**
+ * Used to dynamically imports state residing in NEM or HM.
+ *
+ * This is a worker for the CPUM_IMPORT_EXTRN_RET() macro and various IEM ones.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   fExtrnImport    The fields to import.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(int) CPUMImportGuestStateOnDemand(PVMCPUCC pVCpu, uint64_t fExtrnImport)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    if (pVCpu->cpum.s.Guest.fExtrn & fExtrnImport)
+    {
+        switch (pVCpu->cpum.s.Guest.fExtrn & CPUMCTX_EXTRN_KEEPER_MASK)
+        {
+            case CPUMCTX_EXTRN_KEEPER_NEM:
+            {
+                int rc = NEMImportStateOnDemand(pVCpu, fExtrnImport);
+                Assert(rc == VINF_SUCCESS || RT_FAILURE_NP(rc));
+                return rc;
+            }
+
+            case CPUMCTX_EXTRN_KEEPER_HM:
+            {
+#ifdef IN_RING0
+                int rc = HMR0ImportStateOnDemand(pVCpu, fExtrnImport);
+                Assert(rc == VINF_SUCCESS || RT_FAILURE_NP(rc));
+                return rc;
+#else
+                AssertLogRelMsgFailed(("TODO Fetch HM state: %#RX64 vs %#RX64\n", pVCpu->cpum.s.Guest.fExtrn, fExtrnImport));
+                return VINF_SUCCESS;
+#endif
+            }
+            default:
+                AssertLogRelMsgFailedReturn(("%#RX64 vs %#RX64\n", pVCpu->cpum.s.Guest.fExtrn, fExtrnImport), VERR_CPUM_IPE_2);
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets valid CR4 bits for the guest.
+ *
+ * @returns Valid CR4 bits.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestCR4ValidMask(PVM pVM)
+{
+    PCCPUMFEATURES pGuestFeatures = &pVM->cpum.s.GuestFeatures;
+    uint64_t fMask = X86_CR4_VME | X86_CR4_PVI
+                   | X86_CR4_TSD | X86_CR4_DE
+                   | X86_CR4_MCE | X86_CR4_PCE;
+    if (pGuestFeatures->fPae)
+        fMask |= X86_CR4_PAE;
+    if (pGuestFeatures->fPge)
+        fMask |= X86_CR4_PGE;
+    if (pGuestFeatures->fPse)
+        fMask |= X86_CR4_PSE;
+    if (pGuestFeatures->fFxSaveRstor)
+        fMask |= X86_CR4_OSFXSR;
+    if (pGuestFeatures->fVmx)
+        fMask |= X86_CR4_VMXE;
+    if (pGuestFeatures->fXSaveRstor)
+        fMask |= X86_CR4_OSXSAVE;
+    if (pGuestFeatures->fPcid)
+        fMask |= X86_CR4_PCIDE;
+    if (pGuestFeatures->fFsGsBase)
+        fMask |= X86_CR4_FSGSBASE;
+    if (pGuestFeatures->fSse)
+        fMask |= X86_CR4_OSXMMEEXCPT;
+    return fMask;
+}
+
+
+/**
+ * Sets the PAE PDPEs for the guest.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ * @param   paPaePdpes  The PAE PDPEs to set.
+ */
+VMM_INT_DECL(void) CPUMSetGuestPaePdpes(PVMCPU pVCpu, PCX86PDPE paPaePdpes)
+{
+    Assert(paPaePdpes);
+    for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->cpum.s.Guest.aPaePdpes); i++)
+        pVCpu->cpum.s.Guest.aPaePdpes[i].u = paPaePdpes[i].u;
+    pVCpu->cpum.s.Guest.fExtrn &= ~CPUMCTX_EXTRN_CR3;
+}
+
+
+/**
+ * Gets the PAE PDPTEs for the guest.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ * @param   paPaePdpes  Where to store the PAE PDPEs.
+ */
+VMM_INT_DECL(void) CPUMGetGuestPaePdpes(PVMCPU pVCpu, PX86PDPE paPaePdpes)
+{
+    Assert(paPaePdpes);
+    CPUM_INT_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR3);
+    for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->cpum.s.Guest.aPaePdpes); i++)
+        paPaePdpes[i].u = pVCpu->cpum.s.Guest.aPaePdpes[i].u;
+}
+
+
+/**
+ * Starts a VMX-preemption timer to expire as specified by the nested hypervisor.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   uTimer          The VMCS preemption timer value.
+ * @param   cShift          The VMX-preemption timer shift (usually based on guest
+ *                          VMX MSR rate).
+ * @param   pu64EntryTick   Where to store the current tick when the timer is
+ *                          programmed.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(int) CPUMStartGuestVmxPremptTimer(PVMCPUCC pVCpu, uint32_t uTimer, uint8_t cShift, uint64_t *pu64EntryTick)
+{
+    Assert(uTimer);
+    Assert(cShift <= 31);
+    Assert(pu64EntryTick);
+    VMCPU_ASSERT_EMT(pVCpu);
+    uint64_t const cTicksToNext = uTimer << cShift;
+    return TMTimerSetRelative(pVCpu->CTX_SUFF(pVM), pVCpu->cpum.s.hNestedVmxPreemptTimer, cTicksToNext, pu64EntryTick);
+}
+
+
+/**
+ * Stops the VMX-preemption timer from firing.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling thread.
+ * @thread  EMT.
+ *
+ * @remarks This can be called during VM reset, so we cannot assume it will be on
+ *          the EMT corresponding to @c pVCpu.
+ */
+VMM_INT_DECL(int) CPUMStopGuestVmxPremptTimer(PVMCPUCC pVCpu)
+{
+    /*
+     * CPUM gets initialized before TM, so we defer creation of timers till CPUMR3InitCompleted().
+     * However, we still get called during CPUMR3Init() and hence we need to check if we have
+     * a valid timer object before trying to stop it.
+     */
+    int rc;
+    TMTIMERHANDLE hTimer = pVCpu->cpum.s.hNestedVmxPreemptTimer;
+    if (hTimer != NIL_TMTIMERHANDLE)
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        rc = TMTimerLock(pVM, hTimer, VERR_IGNORED);
+        if (rc == VINF_SUCCESS)
+        {
+            if (TMTimerIsActive(pVM, hTimer))
+                TMTimerStop(pVM, hTimer);
+            TMTimerUnlock(pVM, hTimer);
+        }
+    }
+    else
+        rc = VERR_NOT_FOUND;
+    return rc;
+}
+
+
+/**
+ * Gets the read and write permission bits for an MSR in an MSR bitmap.
+ *
+ * @returns VMXMSRPM_XXX - the MSR permission.
+ * @param   pvMsrBitmap     Pointer to the MSR bitmap.
+ * @param   idMsr           The MSR to get permissions for.
+ *
+ * @sa      hmR0VmxSetMsrPermission.
+ */
+VMM_INT_DECL(uint32_t) CPUMGetVmxMsrPermission(void const *pvMsrBitmap, uint32_t idMsr)
+{
+    AssertPtrReturn(pvMsrBitmap, VMXMSRPM_EXIT_RD | VMXMSRPM_EXIT_WR);
+
+    uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
+
+    /*
+     * MSR Layout:
+     *   Byte index            MSR range            Interpreted as
+     * 0x000 - 0x3ff    0x00000000 - 0x00001fff    Low MSR read bits.
+     * 0x400 - 0x7ff    0xc0000000 - 0xc0001fff    High MSR read bits.
+     * 0x800 - 0xbff    0x00000000 - 0x00001fff    Low MSR write bits.
+     * 0xc00 - 0xfff    0xc0000000 - 0xc0001fff    High MSR write bits.
+     *
+     * A bit corresponding to an MSR within the above range causes a VM-exit
+     * if the bit is 1 on executions of RDMSR/WRMSR.  If an MSR falls out of
+     * the MSR range, it always cause a VM-exit.
+     *
+     * See Intel spec. 24.6.9 "MSR-Bitmap Address".
+     */
+    uint32_t const offBitmapRead  = 0;
+    uint32_t const offBitmapWrite = 0x800;
+    uint32_t       offMsr;
+    uint32_t       iBit;
+    if (idMsr <= UINT32_C(0x00001fff))
+    {
+        offMsr = 0;
+        iBit   = idMsr;
+    }
+    else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
+    {
+        offMsr = 0x400;
+        iBit   = idMsr - UINT32_C(0xc0000000);
+    }
+    else
+    {
+        LogFunc(("Warning! Out of range MSR %#RX32\n", idMsr));
+        return VMXMSRPM_EXIT_RD | VMXMSRPM_EXIT_WR;
+    }
+
+    /*
+     * Get the MSR read permissions.
+     */
+    uint32_t fRet;
+    uint32_t const offMsrRead = offBitmapRead + offMsr;
+    Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
+    if (ASMBitTest(pbMsrBitmap, (offMsrRead << 3) + iBit))
+        fRet = VMXMSRPM_EXIT_RD;
+    else
+        fRet = VMXMSRPM_ALLOW_RD;
+
+    /*
+     * Get the MSR write permissions.
+     */
+    uint32_t const offMsrWrite = offBitmapWrite + offMsr;
+    Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
+    if (ASMBitTest(pbMsrBitmap, (offMsrWrite << 3) + iBit))
+        fRet |= VMXMSRPM_EXIT_WR;
+    else
+        fRet |= VMXMSRPM_ALLOW_WR;
+
+    Assert(VMXMSRPM_IS_FLAG_VALID(fRet));
+    return fRet;
+}
+
+
+/**
+ * Checks the permission bits for the specified I/O port from the given I/O bitmap
+ * to see if causes a VM-exit.
+ *
+ * @returns @c true if the I/O port access must cause a VM-exit, @c false otherwise.
+ * @param   pbIoBitmap  Pointer to I/O bitmap.
+ * @param   uPort       The I/O port being accessed.
+ * @param   cbAccess    e size of the I/O access in bytes (1, 2 or 4 bytes).
+ */
+static bool cpumGetVmxIoBitmapPermission(uint8_t const *pbIoBitmap, uint16_t uPort, uint8_t cbAccess)
+{
+    Assert(cbAccess == 1 || cbAccess == 2 || cbAccess == 4);
+
+    /*
+     * If the I/O port access wraps around the 16-bit port I/O space, we must cause a
+     * VM-exit.
+     *
+     * Reading 1, 2, 4 bytes at ports 0xffff, 0xfffe and 0xfffc are valid and do not
+     * constitute a wrap around. However, reading 2 bytes at port 0xffff or 4 bytes
+     * from port 0xffff/0xfffe/0xfffd constitute a wrap around. In other words, any
+     * access to -both- ports 0xffff and port 0 is a wrap around.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    uint32_t const uPortLast = uPort + cbAccess;
+    if (uPortLast > 0x10000)
+        return true;
+
+    /*
+     * If any bit corresponding to the I/O access is set, we must cause a VM-exit.
+     */
+    uint16_t const offPerm    = uPort >> 3;                         /* Byte offset of the port. */
+    uint16_t const idxPermBit = uPort - (offPerm << 3);             /* Bit offset within byte. */
+    Assert(idxPermBit < 8);
+    static const uint8_t s_afMask[] = { 0x0, 0x1, 0x3, 0x7, 0xf };  /* Bit-mask for all access sizes. */
+    uint16_t const fMask = s_afMask[cbAccess] << idxPermBit;        /* Bit-mask of the access. */
+
+    /* Fetch 8 or 16-bits depending on whether the access spans 8-bit boundary. */
+    RTUINT16U uPerm;
+    uPerm.s.Lo = pbIoBitmap[offPerm];
+    if (idxPermBit + cbAccess > 8)
+        uPerm.s.Hi = pbIoBitmap[offPerm + 1];
+    else
+        uPerm.s.Hi = 0;
+
+    /* If any bit for the access is 1, we must cause a VM-exit. */
+    if (uPerm.u & fMask)
+        return true;
+
+    return false;
+}
+
+
+/**
+ * Returns whether the given VMCS field is valid and supported for the guest.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   u64VmcsField    The VMCS field.
+ *
+ * @remarks This takes into account the CPU features exposed to the guest.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVmxVmcsFieldValid(PVMCC pVM, uint64_t u64VmcsField)
+{
+    uint32_t const uFieldEncHi = RT_HI_U32(u64VmcsField);
+    uint32_t const uFieldEncLo = RT_LO_U32(u64VmcsField);
+    if (!uFieldEncHi)
+    { /* likely */ }
+    else
+        return false;
+
+    PCCPUMFEATURES pFeat = &pVM->cpum.s.GuestFeatures;
+    switch (uFieldEncLo)
+    {
+        /*
+         * 16-bit fields.
+         */
+        /* Control fields. */
+        case VMX_VMCS16_VPID:                             return pFeat->fVmxVpid;
+        case VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR:         return pFeat->fVmxPostedInt;
+        case VMX_VMCS16_EPTP_INDEX:                       return pFeat->fVmxEptXcptVe;
+
+        /* Guest-state fields. */
+        case VMX_VMCS16_GUEST_ES_SEL:
+        case VMX_VMCS16_GUEST_CS_SEL:
+        case VMX_VMCS16_GUEST_SS_SEL:
+        case VMX_VMCS16_GUEST_DS_SEL:
+        case VMX_VMCS16_GUEST_FS_SEL:
+        case VMX_VMCS16_GUEST_GS_SEL:
+        case VMX_VMCS16_GUEST_LDTR_SEL:
+        case VMX_VMCS16_GUEST_TR_SEL:                     return true;
+        case VMX_VMCS16_GUEST_INTR_STATUS:                return pFeat->fVmxVirtIntDelivery;
+        case VMX_VMCS16_GUEST_PML_INDEX:                  return pFeat->fVmxPml;
+
+        /* Host-state fields. */
+        case VMX_VMCS16_HOST_ES_SEL:
+        case VMX_VMCS16_HOST_CS_SEL:
+        case VMX_VMCS16_HOST_SS_SEL:
+        case VMX_VMCS16_HOST_DS_SEL:
+        case VMX_VMCS16_HOST_FS_SEL:
+        case VMX_VMCS16_HOST_GS_SEL:
+        case VMX_VMCS16_HOST_TR_SEL:                      return true;
+
+        /*
+         * 64-bit fields.
+         */
+        /* Control fields. */
+        case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
+        case VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH:
+        case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
+        case VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH:            return pFeat->fVmxUseIoBitmaps;
+        case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
+        case VMX_VMCS64_CTRL_MSR_BITMAP_HIGH:             return pFeat->fVmxUseMsrBitmaps;
+        case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
+        case VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH:
+        case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
+        case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH:
+        case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
+        case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH:
+        case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
+        case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH:          return true;
+        case VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL:
+        case VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH:          return pFeat->fVmxPml;
+        case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
+        case VMX_VMCS64_CTRL_TSC_OFFSET_HIGH:             return true;
+        case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
+        case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH:     return pFeat->fVmxUseTprShadow;
+        case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
+        case VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH:        return pFeat->fVmxVirtApicAccess;
+        case VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL:
+        case VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH:       return pFeat->fVmxPostedInt;
+        case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
+        case VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH:           return pFeat->fVmxVmFunc;
+        case VMX_VMCS64_CTRL_EPTP_FULL:
+        case VMX_VMCS64_CTRL_EPTP_HIGH:                   return pFeat->fVmxEpt;
+        case VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL:
+        case VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH:           return pFeat->fVmxVirtIntDelivery;
+        case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
+        case VMX_VMCS64_CTRL_EPTP_LIST_HIGH:
+        {
+            PCVMCPU pVCpu = pVM->CTX_SUFF(apCpus)[0];
+            uint64_t const uVmFuncMsr = pVCpu->cpum.s.Guest.hwvirt.vmx.Msrs.u64VmFunc;
+            return RT_BOOL(RT_BF_GET(uVmFuncMsr, VMX_BF_VMFUNC_EPTP_SWITCHING));
+        }
+        case VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL:
+        case VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH:
+        case VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL:
+        case VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH:         return pFeat->fVmxVmcsShadowing;
+        case VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL:
+        case VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH:      return pFeat->fVmxEptXcptVe;
+        case VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL:
+        case VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH:     return pFeat->fVmxXsavesXrstors;
+        case VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL:
+        case VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH:         return pFeat->fVmxUseTscScaling;
+        case VMX_VMCS64_CTRL_PROC_EXEC3_FULL:
+        case VMX_VMCS64_CTRL_PROC_EXEC3_HIGH:             return pFeat->fVmxTertiaryExecCtls;
+
+        /* Read-only data fields. */
+        case VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL:
+        case VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH:          return pFeat->fVmxEpt;
+
+        /* Guest-state fields. */
+        case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
+        case VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH:
+        case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
+        case VMX_VMCS64_GUEST_DEBUGCTL_HIGH:              return true;
+        case VMX_VMCS64_GUEST_PAT_FULL:
+        case VMX_VMCS64_GUEST_PAT_HIGH:                   return pFeat->fVmxEntryLoadPatMsr || pFeat->fVmxExitSavePatMsr;
+        case VMX_VMCS64_GUEST_EFER_FULL:
+        case VMX_VMCS64_GUEST_EFER_HIGH:                  return pFeat->fVmxEntryLoadEferMsr || pFeat->fVmxExitSaveEferMsr;
+        case VMX_VMCS64_GUEST_PDPTE0_FULL:
+        case VMX_VMCS64_GUEST_PDPTE0_HIGH:
+        case VMX_VMCS64_GUEST_PDPTE1_FULL:
+        case VMX_VMCS64_GUEST_PDPTE1_HIGH:
+        case VMX_VMCS64_GUEST_PDPTE2_FULL:
+        case VMX_VMCS64_GUEST_PDPTE2_HIGH:
+        case VMX_VMCS64_GUEST_PDPTE3_FULL:
+        case VMX_VMCS64_GUEST_PDPTE3_HIGH:                return pFeat->fVmxEpt;
+
+        /* Host-state fields. */
+        case VMX_VMCS64_HOST_PAT_FULL:
+        case VMX_VMCS64_HOST_PAT_HIGH:                    return pFeat->fVmxExitLoadPatMsr;
+        case VMX_VMCS64_HOST_EFER_FULL:
+        case VMX_VMCS64_HOST_EFER_HIGH:                   return pFeat->fVmxExitLoadEferMsr;
+
+        /*
+         * 32-bit fields.
+         */
+        /* Control fields. */
+        case VMX_VMCS32_CTRL_PIN_EXEC:
+        case VMX_VMCS32_CTRL_PROC_EXEC:
+        case VMX_VMCS32_CTRL_EXCEPTION_BITMAP:
+        case VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK:
+        case VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH:
+        case VMX_VMCS32_CTRL_CR3_TARGET_COUNT:
+        case VMX_VMCS32_CTRL_EXIT:
+        case VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT:
+        case VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT:
+        case VMX_VMCS32_CTRL_ENTRY:
+        case VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT:
+        case VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO:
+        case VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE:
+        case VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH:          return true;
+        case VMX_VMCS32_CTRL_TPR_THRESHOLD:               return pFeat->fVmxUseTprShadow;
+        case VMX_VMCS32_CTRL_PROC_EXEC2:                  return pFeat->fVmxSecondaryExecCtls;
+        case VMX_VMCS32_CTRL_PLE_GAP:
+        case VMX_VMCS32_CTRL_PLE_WINDOW:                  return pFeat->fVmxPauseLoopExit;
+
+        /* Read-only data fields. */
+        case VMX_VMCS32_RO_VM_INSTR_ERROR:
+        case VMX_VMCS32_RO_EXIT_REASON:
+        case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
+        case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
+        case VMX_VMCS32_RO_IDT_VECTORING_INFO:
+        case VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE:
+        case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
+        case VMX_VMCS32_RO_EXIT_INSTR_INFO:               return true;
+
+        /* Guest-state fields. */
+        case VMX_VMCS32_GUEST_ES_LIMIT:
+        case VMX_VMCS32_GUEST_CS_LIMIT:
+        case VMX_VMCS32_GUEST_SS_LIMIT:
+        case VMX_VMCS32_GUEST_DS_LIMIT:
+        case VMX_VMCS32_GUEST_FS_LIMIT:
+        case VMX_VMCS32_GUEST_GS_LIMIT:
+        case VMX_VMCS32_GUEST_LDTR_LIMIT:
+        case VMX_VMCS32_GUEST_TR_LIMIT:
+        case VMX_VMCS32_GUEST_GDTR_LIMIT:
+        case VMX_VMCS32_GUEST_IDTR_LIMIT:
+        case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
+        case VMX_VMCS32_GUEST_INT_STATE:
+        case VMX_VMCS32_GUEST_ACTIVITY_STATE:
+        case VMX_VMCS32_GUEST_SMBASE:
+        case VMX_VMCS32_GUEST_SYSENTER_CS:                return true;
+        case VMX_VMCS32_PREEMPT_TIMER_VALUE:              return pFeat->fVmxPreemptTimer;
+
+        /* Host-state fields. */
+        case VMX_VMCS32_HOST_SYSENTER_CS:                 return true;
+
+        /*
+         * Natural-width fields.
+         */
+        /* Control fields. */
+        case VMX_VMCS_CTRL_CR0_MASK:
+        case VMX_VMCS_CTRL_CR4_MASK:
+        case VMX_VMCS_CTRL_CR0_READ_SHADOW:
+        case VMX_VMCS_CTRL_CR4_READ_SHADOW:
+        case VMX_VMCS_CTRL_CR3_TARGET_VAL0:
+        case VMX_VMCS_CTRL_CR3_TARGET_VAL1:
+        case VMX_VMCS_CTRL_CR3_TARGET_VAL2:
+        case VMX_VMCS_CTRL_CR3_TARGET_VAL3:               return true;
+
+        /* Read-only data fields. */
+        case VMX_VMCS_RO_EXIT_QUALIFICATION:
+        case VMX_VMCS_RO_IO_RCX:
+        case VMX_VMCS_RO_IO_RSI:
+        case VMX_VMCS_RO_IO_RDI:
+        case VMX_VMCS_RO_IO_RIP:
+        case VMX_VMCS_RO_GUEST_LINEAR_ADDR:               return true;
+
+        /* Guest-state fields. */
+        case VMX_VMCS_GUEST_CR0:
+        case VMX_VMCS_GUEST_CR3:
+        case VMX_VMCS_GUEST_CR4:
+        case VMX_VMCS_GUEST_ES_BASE:
+        case VMX_VMCS_GUEST_CS_BASE:
+        case VMX_VMCS_GUEST_SS_BASE:
+        case VMX_VMCS_GUEST_DS_BASE:
+        case VMX_VMCS_GUEST_FS_BASE:
+        case VMX_VMCS_GUEST_GS_BASE:
+        case VMX_VMCS_GUEST_LDTR_BASE:
+        case VMX_VMCS_GUEST_TR_BASE:
+        case VMX_VMCS_GUEST_GDTR_BASE:
+        case VMX_VMCS_GUEST_IDTR_BASE:
+        case VMX_VMCS_GUEST_DR7:
+        case VMX_VMCS_GUEST_RSP:
+        case VMX_VMCS_GUEST_RIP:
+        case VMX_VMCS_GUEST_RFLAGS:
+        case VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS:
+        case VMX_VMCS_GUEST_SYSENTER_ESP:
+        case VMX_VMCS_GUEST_SYSENTER_EIP:                 return true;
+
+        /* Host-state fields. */
+        case VMX_VMCS_HOST_CR0:
+        case VMX_VMCS_HOST_CR3:
+        case VMX_VMCS_HOST_CR4:
+        case VMX_VMCS_HOST_FS_BASE:
+        case VMX_VMCS_HOST_GS_BASE:
+        case VMX_VMCS_HOST_TR_BASE:
+        case VMX_VMCS_HOST_GDTR_BASE:
+        case VMX_VMCS_HOST_IDTR_BASE:
+        case VMX_VMCS_HOST_SYSENTER_ESP:
+        case VMX_VMCS_HOST_SYSENTER_EIP:
+        case VMX_VMCS_HOST_RSP:
+        case VMX_VMCS_HOST_RIP:                           return true;
+    }
+
+    return false;
+}
+
+
+/**
+ * Checks whether the given I/O access should cause a nested-guest VM-exit.
+ *
+ * @returns @c true if it causes a VM-exit, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   u16Port     The I/O port being accessed.
+ * @param   cbAccess    The size of the I/O access in bytes (1, 2 or 4 bytes).
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVmxIoInterceptSet(PCVMCPU pVCpu, uint16_t u16Port, uint8_t cbAccess)
+{
+    PCCPUMCTX pCtx = &pVCpu->cpum.s.Guest;
+    if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_UNCOND_IO_EXIT))
+        return true;
+
+    if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_USE_IO_BITMAPS))
+        return cpumGetVmxIoBitmapPermission(pCtx->hwvirt.vmx.abIoBitmap, u16Port, cbAccess);
+
+    return false;
+}
+
+
+/**
+ * Checks whether the Mov-to-CR3 instruction causes a nested-guest VM-exit.
+ *
+ * @returns @c true if it causes a VM-exit, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uNewCr3     The CR3 value being written.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVmxMovToCr3InterceptSet(PVMCPU pVCpu, uint64_t uNewCr3)
+{
+    /*
+     * If the CR3-load exiting control is set and the new CR3 value does not
+     * match any of the CR3-target values in the VMCS, we must cause a VM-exit.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    PCCPUMCTX const pCtx = &pVCpu->cpum.s.Guest;
+    if (CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_CR3_LOAD_EXIT))
+    {
+        uint32_t const uCr3TargetCount = pCtx->hwvirt.vmx.Vmcs.u32Cr3TargetCount;
+        Assert(uCr3TargetCount <= VMX_V_CR3_TARGET_COUNT);
+
+        /* If the CR3-target count is 0, cause a VM-exit. */
+        if (uCr3TargetCount == 0)
+            return true;
+
+        /* If the CR3 being written doesn't match any of the target values, cause a VM-exit. */
+        AssertCompile(VMX_V_CR3_TARGET_COUNT == 4);
+        if (   uNewCr3 != pCtx->hwvirt.vmx.Vmcs.u64Cr3Target0.u
+            && uNewCr3 != pCtx->hwvirt.vmx.Vmcs.u64Cr3Target1.u
+            && uNewCr3 != pCtx->hwvirt.vmx.Vmcs.u64Cr3Target2.u
+            && uNewCr3 != pCtx->hwvirt.vmx.Vmcs.u64Cr3Target3.u)
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * Checks whether a VMREAD or VMWRITE instruction for the given VMCS field causes a
+ * VM-exit or not.
+ *
+ * @returns @c true if the VMREAD/VMWRITE is intercepted, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason (VMX_EXIT_VMREAD or
+ *                          VMX_EXIT_VMREAD).
+ * @param   u64VmcsField    The VMCS field.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVmxVmreadVmwriteInterceptSet(PCVMCPU pVCpu, uint32_t uExitReason, uint64_t u64VmcsField)
+{
+    Assert(CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.s.Guest));
+    Assert(   uExitReason == VMX_EXIT_VMREAD
+           || uExitReason == VMX_EXIT_VMWRITE);
+
+    /*
+     * Without VMCS shadowing, all VMREAD and VMWRITE instructions are intercepted.
+     */
+    if (!CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.s.Guest, VMX_PROC_CTLS2_VMCS_SHADOWING))
+        return true;
+
+    /*
+     * If any reserved bit in the 64-bit VMCS field encoding is set, the VMREAD/VMWRITE
+     * is intercepted. This excludes any reserved bits in the valid parts of the field
+     * encoding (i.e. bit 12).
+     */
+    if (u64VmcsField & VMX_VMCSFIELD_RSVD_MASK)
+        return true;
+
+    /*
+     * Finally, consult the VMREAD/VMWRITE bitmap whether to intercept the instruction or not.
+     */
+    uint32_t const        u32VmcsField = RT_LO_U32(u64VmcsField);
+    uint8_t const * const pbBitmap     = uExitReason == VMX_EXIT_VMREAD
+                                       ? &pVCpu->cpum.s.Guest.hwvirt.vmx.abVmreadBitmap[0]
+                                       : &pVCpu->cpum.s.Guest.hwvirt.vmx.abVmwriteBitmap[0];
+    Assert(pbBitmap);
+    Assert(u32VmcsField >> 3 < VMX_V_VMREAD_VMWRITE_BITMAP_SIZE);
+    return ASMBitTest(pbBitmap, (u32VmcsField << 3) + (u32VmcsField & 7));
+}
+
+
+
+/**
+ * Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
+ *
+ * @param   pvIoBitmap      Pointer to the nested-guest IO bitmap.
+ * @param   u16Port         The IO port being accessed.
+ * @param   enmIoType       The type of IO access.
+ * @param   cbReg           The IO operand size in bytes.
+ * @param   cAddrSizeBits   The address size bits (for 16, 32 or 64).
+ * @param   iEffSeg         The effective segment number.
+ * @param   fRep            Whether this is a repeating IO instruction (REP prefix).
+ * @param   fStrIo          Whether this is a string IO instruction.
+ * @param   pIoExitInfo     Pointer to the SVMIOIOEXITINFO struct to be filled.
+ *                          Optional, can be NULL.
+ */
+VMM_INT_DECL(bool) CPUMIsSvmIoInterceptSet(void *pvIoBitmap, uint16_t u16Port, SVMIOIOTYPE enmIoType, uint8_t cbReg,
+                                           uint8_t cAddrSizeBits, uint8_t iEffSeg, bool fRep, bool fStrIo,
+                                           PSVMIOIOEXITINFO pIoExitInfo)
+{
+    Assert(cAddrSizeBits == 16 || cAddrSizeBits == 32 || cAddrSizeBits == 64);
+    Assert(cbReg == 1 || cbReg == 2 || cbReg == 4 || cbReg == 8);
+
+    /*
+     * The IOPM layout:
+     * Each bit represents one 8-bit port. That makes a total of 0..65535 bits or
+     * two 4K pages.
+     *
+     * For IO instructions that access more than a single byte, the permission bits
+     * for all bytes are checked; if any bit is set to 1, the IO access is intercepted.
+     *
+     * Since it's possible to do a 32-bit IO access at port 65534 (accessing 4 bytes),
+     * we need 3 extra bits beyond the second 4K page.
+     */
+    static const uint16_t s_auSizeMasks[] = { 0, 1, 3, 0, 0xf, 0, 0, 0 };
+
+    uint16_t const offIopm   = u16Port >> 3;
+    uint16_t const fSizeMask = s_auSizeMasks[(cAddrSizeBits >> SVM_IOIO_OP_SIZE_SHIFT) & 7];
+    uint8_t  const cShift    = u16Port - (offIopm << 3);
+    uint16_t const fIopmMask = (1 << cShift) | (fSizeMask << cShift);
+
+    uint8_t const *pbIopm = (uint8_t *)pvIoBitmap;
+    Assert(pbIopm);
+    pbIopm += offIopm;
+    uint16_t const u16Iopm = *(uint16_t *)pbIopm;
+    if (u16Iopm & fIopmMask)
+    {
+        if (pIoExitInfo)
+        {
+            static const uint32_t s_auIoOpSize[] =
+            { SVM_IOIO_32_BIT_OP, SVM_IOIO_8_BIT_OP, SVM_IOIO_16_BIT_OP, 0, SVM_IOIO_32_BIT_OP, 0, 0, 0 };
+
+            static const uint32_t s_auIoAddrSize[] =
+            { 0, SVM_IOIO_16_BIT_ADDR, SVM_IOIO_32_BIT_ADDR, 0, SVM_IOIO_64_BIT_ADDR, 0, 0, 0 };
+
+            pIoExitInfo->u         = s_auIoOpSize[cbReg & 7];
+            pIoExitInfo->u        |= s_auIoAddrSize[(cAddrSizeBits >> 4) & 7];
+            pIoExitInfo->n.u1Str   = fStrIo;
+            pIoExitInfo->n.u1Rep   = fRep;
+            pIoExitInfo->n.u3Seg   = iEffSeg & 7;
+            pIoExitInfo->n.u1Type  = enmIoType;
+            pIoExitInfo->n.u16Port = u16Port;
+        }
+        return true;
+    }
+
+    /** @todo remove later (for debugging as VirtualBox always traps all IO
+     *        intercepts). */
+    AssertMsgFailed(("CPUMSvmIsIOInterceptActive: We expect an IO intercept here!\n"));
+    return false;
+}
+
+
+/**
+ * Gets the MSR permission bitmap byte and bit offset for the specified MSR.
+ *
+ * @returns VBox status code.
+ * @param   idMsr       The MSR being requested.
+ * @param   pbOffMsrpm  Where to store the byte offset in the MSR permission
+ *                      bitmap for @a idMsr.
+ * @param   puMsrpmBit  Where to store the bit offset starting at the byte
+ *                      returned in @a pbOffMsrpm.
+ */
+VMM_INT_DECL(int) CPUMGetSvmMsrpmOffsetAndBit(uint32_t idMsr, uint16_t *pbOffMsrpm, uint8_t *puMsrpmBit)
+{
+    Assert(pbOffMsrpm);
+    Assert(puMsrpmBit);
+
+    /*
+     * MSRPM Layout:
+     * Byte offset          MSR range
+     * 0x000  - 0x7ff       0x00000000 - 0x00001fff
+     * 0x800  - 0xfff       0xc0000000 - 0xc0001fff
+     * 0x1000 - 0x17ff      0xc0010000 - 0xc0011fff
+     * 0x1800 - 0x1fff              Reserved
+     *
+     * Each MSR is represented by 2 permission bits (read and write).
+     */
+    if (idMsr <= 0x00001fff)
+    {
+        /* Pentium-compatible MSRs. */
+        uint32_t const bitoffMsr = idMsr << 1;
+        *pbOffMsrpm = bitoffMsr >> 3;
+        *puMsrpmBit = bitoffMsr & 7;
+        return VINF_SUCCESS;
+    }
+
+    if (   idMsr >= 0xc0000000
+        && idMsr <= 0xc0001fff)
+    {
+        /* AMD Sixth Generation x86 Processor MSRs. */
+        uint32_t const bitoffMsr = (idMsr - 0xc0000000) << 1;
+        *pbOffMsrpm = 0x800 + (bitoffMsr >> 3);
+        *puMsrpmBit = bitoffMsr & 7;
+        return VINF_SUCCESS;
+    }
+
+    if (   idMsr >= 0xc0010000
+        && idMsr <= 0xc0011fff)
+    {
+        /* AMD Seventh and Eighth Generation Processor MSRs. */
+        uint32_t const bitoffMsr = (idMsr - 0xc0010000) << 1;
+        *pbOffMsrpm = 0x1000 + (bitoffMsr >> 3);
+        *puMsrpmBit = bitoffMsr & 7;
+        return VINF_SUCCESS;
+    }
+
+    *pbOffMsrpm = 0;
+    *puMsrpmBit = 0;
+    return VERR_OUT_OF_RANGE;
+}
+
+
+/**
+ * Checks whether the guest is in VMX non-root mode and using EPT paging.
+ *
+ * @returns @c true if in VMX non-root operation with EPT, @c false otherwise.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVmxEptPagingEnabled(PCVMCPUCC pVCpu)
+{
+    return CPUMIsGuestVmxEptPagingEnabledEx(&pVCpu->cpum.s.Guest);
+}
+
+
+/**
+ * Checks whether the guest is in VMX non-root mode and using EPT paging and the
+ * nested-guest is in PAE mode.
+ *
+ * @returns @c true if in VMX non-root operation with EPT, @c false otherwise.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) CPUMIsGuestVmxEptPaePagingEnabled(PCVMCPUCC pVCpu)
+{
+    return    CPUMIsGuestVmxEptPagingEnabledEx(&pVCpu->cpum.s.Guest)
+           && CPUMIsGuestInPAEModeEx(&pVCpu->cpum.s.Guest);
+}
+
+
+/**
+ * Returns the guest-physical address of the APIC-access page when executing a
+ * nested-guest.
+ *
+ * @returns The APIC-access page guest-physical address.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(uint64_t) CPUMGetGuestVmxApicAccessPageAddr(PCVMCPUCC pVCpu)
+{
+    return CPUMGetGuestVmxApicAccessPageAddrEx(&pVCpu->cpum.s.Guest);
+}
+
diff --git a/src/VBox/VMM/VMMAll/DBGFAll.cpp b/src/VBox/VMM/VMMAll/DBGFAll.cpp
new file mode 100644
index 00000000..dc1cb1d5
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/DBGFAll.cpp
@@ -0,0 +1,592 @@
+/* $Id: DBGFAll.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, All Context Code.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/dbgf.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/stdarg.h>
+
+
+/*
+ * Check the read-only VM members.
+ */
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.bmSoftIntBreakpoints,  VM, dbgf.ro.bmSoftIntBreakpoints);
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.bmHardIntBreakpoints,  VM, dbgf.ro.bmHardIntBreakpoints);
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.bmSelectedEvents,      VM, dbgf.ro.bmSelectedEvents);
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.cHardIntBreakpoints,   VM, dbgf.ro.cHardIntBreakpoints);
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.cSoftIntBreakpoints,   VM, dbgf.ro.cSoftIntBreakpoints);
+AssertCompileMembersSameSizeAndOffset(VM, dbgf.s.cSelectedEvents,       VM, dbgf.ro.cSelectedEvents);
+
+
+/**
+ * Gets the hardware breakpoint configuration as DR7.
+ *
+ * @returns DR7 from the DBGF point of view.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR7(PVM pVM)
+{
+    RTGCUINTREG uDr7 = X86_DR7_GD | X86_DR7_GE | X86_DR7_LE | X86_DR7_RA1_MASK;
+    for (uint32_t i = 0; i < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); i++)
+    {
+        if (   pVM->dbgf.s.aHwBreakpoints[i].fEnabled
+            && pVM->dbgf.s.aHwBreakpoints[i].hBp != NIL_DBGFBP)
+        {
+            static const uint8_t s_au8Sizes[8] =
+            {
+                X86_DR7_LEN_BYTE, X86_DR7_LEN_BYTE, X86_DR7_LEN_WORD, X86_DR7_LEN_BYTE,
+                X86_DR7_LEN_DWORD,X86_DR7_LEN_BYTE, X86_DR7_LEN_BYTE, X86_DR7_LEN_QWORD
+            };
+            uDr7 |= X86_DR7_G(i)
+                 |  X86_DR7_RW(i, pVM->dbgf.s.aHwBreakpoints[i].fType)
+                 |  X86_DR7_LEN(i, s_au8Sizes[pVM->dbgf.s.aHwBreakpoints[i].cb]);
+        }
+    }
+    return uDr7;
+}
+
+
+/**
+ * Gets the address of the hardware breakpoint number 0.
+ *
+ * @returns DR0 from the DBGF point of view.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR0(PVM pVM)
+{
+    return pVM->dbgf.s.aHwBreakpoints[0].GCPtr;
+}
+
+
+/**
+ * Gets the address of the hardware breakpoint number 1.
+ *
+ * @returns DR1 from the DBGF point of view.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR1(PVM pVM)
+{
+    return pVM->dbgf.s.aHwBreakpoints[1].GCPtr;
+}
+
+
+/**
+ * Gets the address of the hardware breakpoint number 2.
+ *
+ * @returns DR2 from the DBGF point of view.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR2(PVM pVM)
+{
+    return pVM->dbgf.s.aHwBreakpoints[2].GCPtr;
+}
+
+
+/**
+ * Gets the address of the hardware breakpoint number 3.
+ *
+ * @returns DR3 from the DBGF point of view.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(RTGCUINTREG) DBGFBpGetDR3(PVM pVM)
+{
+    return pVM->dbgf.s.aHwBreakpoints[3].GCPtr;
+}
+
+
+/**
+ * Checks if any of the hardware breakpoints are armed.
+ *
+ * @returns true if armed, false if not.
+ * @param   pVM        The cross context VM structure.
+ * @remarks Don't call this from CPUMRecalcHyperDRx!
+ */
+VMM_INT_DECL(bool) DBGFBpIsHwArmed(PVM pVM)
+{
+    return pVM->dbgf.s.cEnabledHwBreakpoints > 0;
+}
+
+
+/**
+ * Checks if any of the hardware I/O breakpoints are armed.
+ *
+ * @returns true if armed, false if not.
+ * @param   pVM        The cross context VM structure.
+ * @remarks Don't call this from CPUMRecalcHyperDRx!
+ */
+VMM_INT_DECL(bool) DBGFBpIsHwIoArmed(PVM pVM)
+{
+    return pVM->dbgf.s.cEnabledHwIoBreakpoints > 0;
+}
+
+
+/**
+ * Checks if any INT3 breakpoints are armed.
+ *
+ * @returns true if armed, false if not.
+ * @param   pVM        The cross context VM structure.
+ * @remarks Don't call this from CPUMRecalcHyperDRx!
+ */
+VMM_INT_DECL(bool) DBGFBpIsInt3Armed(PVM pVM)
+{
+    /** @todo There was a todo here and returning false when I (bird) removed
+     *        VBOX_WITH_LOTS_OF_DBGF_BPS, so this might not be correct. */
+    return pVM->dbgf.s.cEnabledInt3Breakpoints > 0;
+}
+
+
+/**
+ * Checks instruction boundrary for guest or hypervisor hardware breakpoints.
+ *
+ * @returns Strict VBox status code.  May return DRx register import errors in
+ *          addition to the ones detailed.
+ * @retval  VINF_SUCCESS no breakpoint.
+ * @retval  VINF_EM_DBG_BREAKPOINT hypervisor breakpoint triggered.
+ * @retval  VINF_EM_RAW_GUEST_TRAP caller must trigger \#DB trap, DR6 and DR7
+ *          have been updated appropriately.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtrPC     The unsegmented PC address.
+ */
+VMM_INT_DECL(VBOXSTRICTRC)  DBGFBpCheckInstruction(PVMCC pVM, PVMCPUCC pVCpu, RTGCPTR GCPtrPC)
+{
+    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_DR7);
+
+    /*
+     * Check hyper breakpoints first as the VMM debugger has priority over
+     * the guest.
+     */
+    /** @todo we need some kind of resume flag for these. */
+    if (pVM->dbgf.s.cEnabledHwBreakpoints > 0)
+        for (unsigned iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++)
+        {
+            if (   pVM->dbgf.s.aHwBreakpoints[iBp].GCPtr != GCPtrPC
+                || pVM->dbgf.s.aHwBreakpoints[iBp].fType != X86_DR7_RW_EO
+                || pVM->dbgf.s.aHwBreakpoints[iBp].cb    != 1
+                || !pVM->dbgf.s.aHwBreakpoints[iBp].fEnabled
+                || pVM->dbgf.s.aHwBreakpoints[iBp].hBp   == NIL_DBGFBP)
+            { /*likely*/ }
+            else
+            {
+                /* (See also DBGFRZTrap01Handler.) */
+                pVCpu->dbgf.s.hBpActive = pVM->dbgf.s.aHwBreakpoints[iBp].hBp;
+                pVCpu->dbgf.s.fSingleSteppingRaw = false;
+
+                LogFlow(("DBGFBpCheckInstruction: hit hw breakpoint %u at %04x:%RGv (%RGv)\n",
+                         iBp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPtrPC));
+                return VINF_EM_DBG_BREAKPOINT;
+            }
+        }
+
+    /*
+     * Check the guest.
+     */
+    uint32_t const fDr7 = (uint32_t)pVCpu->cpum.GstCtx.dr[7];
+    if (X86_DR7_ANY_EO_ENABLED(fDr7) && !pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
+    {
+        /*
+         * The CPU (10980XE & 6700K at least) will set the DR6.BPx bits for any
+         * DRx that matches the current PC and is configured as an execution
+         * breakpoint (RWx=EO, LENx=1byte).  They don't have to be enabled,
+         * however one that is enabled must match for the #DB to be raised and
+         * DR6 to be modified, of course.
+         */
+        CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+        uint32_t fMatched = 0;
+        uint32_t fEnabled = 0;
+        for (unsigned iBp = 0, uBpMask = 1; iBp < 4; iBp++, uBpMask <<= 1)
+            if (X86_DR7_IS_EO_CFG(fDr7, iBp))
+            {
+                if (fDr7 & X86_DR7_L_G(iBp))
+                    fEnabled |= uBpMask;
+                if (pVCpu->cpum.GstCtx.dr[iBp] == GCPtrPC)
+                    fMatched |= uBpMask;
+            }
+        if (!(fEnabled & fMatched))
+        { /*likely*/ }
+        else if (fEnabled & fMatched)
+        {
+            /*
+             * Update DR6 and DR7.
+             *
+             * See "AMD64 Architecture Programmer's Manual Volume 2", chapter
+             * 13.1.1.3 for details on DR6 bits.  The basics is that the B0..B3
+             * bits are always cleared while the others must be cleared by software.
+             *
+             * The following sub chapters says the GD bit is always cleared when
+             * generating a #DB so the handler can safely access the debug registers.
+             */
+            CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_DR6);
+            pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
+            if (pVM->cpum.ro.GuestFeatures.enmCpuVendor != CPUMCPUVENDOR_INTEL)
+                pVCpu->cpum.GstCtx.dr[6] |= fMatched & fEnabled;
+            else
+                pVCpu->cpum.GstCtx.dr[6] |= fMatched;    /* Intel: All matched, regardless of whether they're enabled or not  */
+            pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
+            LogFlow(("DBGFBpCheckInstruction: hit hw breakpoints %#x at %04x:%RGv (%RGv)\n",
+                     fMatched, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPtrPC));
+            return VINF_EM_RAW_GUEST_TRAP;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks I/O access for guest or hypervisor hardware breakpoints.
+ *
+ * @returns Strict VBox status code
+ * @retval  VINF_SUCCESS no breakpoint.
+ * @retval  VINF_EM_DBG_BREAKPOINT hypervisor breakpoint triggered.
+ * @retval  VINF_EM_RAW_GUEST_TRAP guest breakpoint triggered, DR6 and DR7 have
+ *          been updated appropriately.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pCtx        The CPU context for the calling EMT.
+ * @param   uIoPort     The I/O port being accessed.
+ * @param   cbValue     The size/width of the access, in bytes.
+ */
+VMM_INT_DECL(VBOXSTRICTRC)  DBGFBpCheckIo(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTIOPORT uIoPort, uint8_t cbValue)
+{
+    uint32_t const uIoPortFirst = uIoPort;
+    uint32_t const uIoPortLast  = uIoPortFirst + cbValue - 1;
+
+    /*
+     * Check hyper breakpoints first as the VMM debugger has priority over
+     * the guest.
+     */
+    if (pVM->dbgf.s.cEnabledHwIoBreakpoints > 0)
+    {
+        for (unsigned iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++)
+        {
+            if (   pVM->dbgf.s.aHwBreakpoints[iBp].fType == X86_DR7_RW_IO
+                && pVM->dbgf.s.aHwBreakpoints[iBp].fEnabled
+                && pVM->dbgf.s.aHwBreakpoints[iBp].hBp   != NIL_DBGFBP)
+            {
+                uint8_t  cbReg      = pVM->dbgf.s.aHwBreakpoints[iBp].cb; Assert(RT_IS_POWER_OF_TWO(cbReg));
+                uint64_t uDrXFirst  = pVM->dbgf.s.aHwBreakpoints[iBp].GCPtr & ~(uint64_t)(cbReg - 1);
+                uint64_t uDrXLast   = uDrXFirst + cbReg - 1;
+                if (uDrXFirst <= uIoPortLast && uDrXLast >= uIoPortFirst)
+                {
+                    /* (See also DBGFRZTrap01Handler.) */
+                    pVCpu->dbgf.s.hBpActive = pVM->dbgf.s.aHwBreakpoints[iBp].hBp;
+                    pVCpu->dbgf.s.fSingleSteppingRaw = false;
+
+                    LogFlow(("DBGFBpCheckIo: hit hw breakpoint %d at %04x:%RGv (iop %#x)\n",
+                             iBp, pCtx->cs.Sel, pCtx->rip, uIoPort));
+                    return VINF_EM_DBG_BREAKPOINT;
+                }
+            }
+        }
+    }
+
+    /*
+     * Check the guest.
+     */
+    uint32_t const uDr7 = pCtx->dr[7];
+    if (   (uDr7 & X86_DR7_ENABLED_MASK)
+        && X86_DR7_ANY_RW_IO(uDr7)
+        && (pCtx->cr4 & X86_CR4_DE) )
+    {
+        for (unsigned iBp = 0; iBp < 4; iBp++)
+        {
+            if (   (uDr7 & X86_DR7_L_G(iBp))
+                && X86_DR7_GET_RW(uDr7, iBp) == X86_DR7_RW_IO)
+            {
+                /* ASSUME the breakpoint and the I/O width qualifier uses the same encoding (1 2 x 4). */
+                static uint8_t const s_abInvAlign[4] = { 0, 1, 7, 3 };
+                uint8_t  cbInvAlign = s_abInvAlign[X86_DR7_GET_LEN(uDr7, iBp)];
+                uint64_t uDrXFirst  = pCtx->dr[iBp] & ~(uint64_t)cbInvAlign;
+                uint64_t uDrXLast   = uDrXFirst + cbInvAlign;
+
+                if (uDrXFirst <= uIoPortLast && uDrXLast >= uIoPortFirst)
+                {
+                    /*
+                     * Update DR6 and DR7.
+                     *
+                     * See "AMD64 Architecture Programmer's Manual Volume 2",
+                     * chapter 13.1.1.3 for details on DR6 bits.  The basics is
+                     * that the B0..B3 bits are always cleared while the others
+                     * must be cleared by software.
+                     *
+                     * The following sub chapters says the GD bit is always
+                     * cleared when generating a #DB so the handler can safely
+                     * access the debug registers.
+                     */
+                    pCtx->dr[6] &= ~X86_DR6_B_MASK;
+                    pCtx->dr[6] |= X86_DR6_B(iBp);
+                    pCtx->dr[7] &= ~X86_DR7_GD;
+                    LogFlow(("DBGFBpCheckIo: hit hw breakpoint %d at %04x:%RGv (iop %#x)\n",
+                             iBp, pCtx->cs.Sel, pCtx->rip, uIoPort));
+                    return VINF_EM_RAW_GUEST_TRAP;
+                }
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks I/O access for guest or hypervisor hardware breakpoints.
+ *
+ * Caller must make sure DR0-3 and DR7 are present in the CPU context before
+ * calling this function.
+ *
+ * @returns CPUMCTX_DBG_DBGF_BP, CPUMCTX_DBG_HIT_DRX_MASK, or 0 (no match).
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uIoPort     The I/O port being accessed.
+ * @param   cbValue     The size/width of the access, in bytes.
+ */
+VMM_INT_DECL(uint32_t) DBGFBpCheckIo2(PVMCC pVM, PVMCPUCC pVCpu, RTIOPORT uIoPort, uint8_t cbValue)
+{
+    uint32_t const uIoPortFirst = uIoPort;
+    uint32_t const uIoPortLast  = uIoPortFirst + cbValue - 1;
+
+    /*
+     * Check hyper breakpoints first as the VMM debugger has priority over
+     * the guest.
+     */
+    if (pVM->dbgf.s.cEnabledHwIoBreakpoints > 0)
+        for (unsigned iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++)
+        {
+            if (   pVM->dbgf.s.aHwBreakpoints[iBp].fType == X86_DR7_RW_IO
+                && pVM->dbgf.s.aHwBreakpoints[iBp].fEnabled
+                && pVM->dbgf.s.aHwBreakpoints[iBp].hBp   != NIL_DBGFBP)
+            {
+                uint8_t  cbReg      = pVM->dbgf.s.aHwBreakpoints[iBp].cb; Assert(RT_IS_POWER_OF_TWO(cbReg));
+                uint64_t uDrXFirst  = pVM->dbgf.s.aHwBreakpoints[iBp].GCPtr & ~(uint64_t)(cbReg - 1);
+                uint64_t uDrXLast   = uDrXFirst + cbReg - 1;
+                if (uDrXFirst <= uIoPortLast && uDrXLast >= uIoPortFirst)
+                {
+                    /* (See also DBGFRZTrap01Handler.) */
+                    pVCpu->dbgf.s.hBpActive = pVM->dbgf.s.aHwBreakpoints[iBp].hBp;
+                    pVCpu->dbgf.s.fSingleSteppingRaw = false;
+
+                    LogFlow(("DBGFBpCheckIo2: hit hw breakpoint %d at %04x:%RGv (iop %#x L %u)\n",
+                             iBp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uIoPort, cbValue));
+                    return CPUMCTX_DBG_DBGF_BP;
+                }
+            }
+        }
+
+    /*
+     * Check the guest.
+     */
+    uint32_t const fDr7 = pVCpu->cpum.GstCtx.dr[7];
+    if (   (fDr7 & X86_DR7_ENABLED_MASK)
+        && X86_DR7_ANY_RW_IO(fDr7)
+        && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE) )
+    {
+        uint32_t fEnabled = 0;
+        uint32_t fMatched = 0;
+        for (unsigned iBp = 0, uBpMask = 1; iBp < 4; iBp++, uBpMask <<= 1)
+        {
+            if (fDr7 & X86_DR7_L_G(iBp))
+                fEnabled |= uBpMask;
+            if (X86_DR7_GET_RW(fDr7, iBp) == X86_DR7_RW_IO)
+            {
+                /* ASSUME the breakpoint and the I/O width qualifier uses the same encoding (1 2 x 4). */
+                static uint8_t const s_abInvAlign[4] = { 0, 1, 7, 3 };
+                uint8_t  const cbInvAlign = s_abInvAlign[X86_DR7_GET_LEN(fDr7, iBp)];
+                uint64_t const uDrXFirst  = pVCpu->cpum.GstCtx.dr[iBp] & ~(uint64_t)cbInvAlign;
+                uint64_t const uDrXLast   = uDrXFirst + cbInvAlign;
+                if (uDrXFirst <= uIoPortLast && uDrXLast >= uIoPortFirst)
+                    fMatched |= uBpMask;
+            }
+        }
+        if (fEnabled & fMatched)
+        {
+            LogFlow(("DBGFBpCheckIo2: hit hw breakpoint %#x at %04x:%RGv (iop %#x L %u)\n",
+                     fMatched, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uIoPort, cbValue));
+            return fMatched << CPUMCTX_DBG_HIT_DRX_SHIFT;
+        }
+    }
+
+    return 0;
+}
+
+
+/**
+ * Returns the single stepping state for a virtual CPU.
+ *
+ * @returns stepping (true) or not (false).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) DBGFIsStepping(PVMCPU pVCpu)
+{
+    return pVCpu->dbgf.s.fSingleSteppingRaw;
+}
+
+
+/**
+ * Checks if the specified generic event is enabled or not.
+ *
+ * @returns true / false.
+ * @param   pVM                 The cross context VM structure.
+ * @param   enmEvent            The generic event being raised.
+ * @param   uEventArg           The argument of that event.
+ */
+DECLINLINE(bool) dbgfEventIsGenericWithArgEnabled(PVM pVM, DBGFEVENTTYPE enmEvent, uint64_t uEventArg)
+{
+    if (DBGF_IS_EVENT_ENABLED(pVM, enmEvent))
+    {
+        switch (enmEvent)
+        {
+            case DBGFEVENT_INTERRUPT_HARDWARE:
+                AssertReturn(uEventArg < 256, false);
+                return ASMBitTest(pVM->dbgf.s.bmHardIntBreakpoints, (uint32_t)uEventArg);
+
+            case DBGFEVENT_INTERRUPT_SOFTWARE:
+                AssertReturn(uEventArg < 256, false);
+                return ASMBitTest(pVM->dbgf.s.bmSoftIntBreakpoints, (uint32_t)uEventArg);
+
+            default:
+                return true;
+
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Raises a generic debug event if enabled and not being ignored.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_EM_DBG_EVENT if the event was raised and the caller should
+ *          return ASAP to the debugger (via EM).  We set VMCPU_FF_DBGF so, it
+ *          is okay not to pass this along in some situations.
+ * @retval  VINF_SUCCESS if the event was disabled or ignored.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   enmEvent            The generic event being raised.
+ * @param   enmCtx              The context in which this event is being raised.
+ * @param   cArgs               Number of arguments (0 - 6).
+ * @param   ...                 Event arguments.
+ *
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) DBGFEventGenericWithArgs(PVM pVM, PVMCPU pVCpu, DBGFEVENTTYPE enmEvent, DBGFEVENTCTX enmCtx,
+                                                    unsigned cArgs, ...)
+{
+    Assert(cArgs < RT_ELEMENTS(pVCpu->dbgf.s.aEvents[0].Event.u.Generic.auArgs));
+
+    /*
+     * Is it enabled.
+     */
+    va_list va;
+    va_start(va, cArgs);
+    uint64_t uEventArg0 = cArgs ? va_arg(va, uint64_t) : 0;
+    if (dbgfEventIsGenericWithArgEnabled(pVM, enmEvent, uEventArg0))
+    {
+        /*
+         * Any events on the stack. Should the incoming event be ignored?
+         */
+        uint64_t const rip = CPUMGetGuestRIP(pVCpu);
+        uint32_t i = pVCpu->dbgf.s.cEvents;
+        if (i > 0)
+        {
+            while (i-- > 0)
+            {
+                if (   pVCpu->dbgf.s.aEvents[i].Event.enmType   == enmEvent
+                    && pVCpu->dbgf.s.aEvents[i].enmState        == DBGFEVENTSTATE_IGNORE
+                    && pVCpu->dbgf.s.aEvents[i].rip             == rip)
+                {
+                    pVCpu->dbgf.s.aEvents[i].enmState = DBGFEVENTSTATE_RESTORABLE;
+                    va_end(va);
+                    return VINF_SUCCESS;
+                }
+                Assert(pVCpu->dbgf.s.aEvents[i].enmState != DBGFEVENTSTATE_CURRENT);
+            }
+
+            /*
+             * Trim the event stack.
+             */
+            i = pVCpu->dbgf.s.cEvents;
+            while (i-- > 0)
+            {
+                if (   pVCpu->dbgf.s.aEvents[i].rip == rip
+                    && (   pVCpu->dbgf.s.aEvents[i].enmState == DBGFEVENTSTATE_RESTORABLE
+                        || pVCpu->dbgf.s.aEvents[i].enmState == DBGFEVENTSTATE_IGNORE) )
+                    pVCpu->dbgf.s.aEvents[i].enmState = DBGFEVENTSTATE_IGNORE;
+                else
+                {
+                    if (i + 1 != pVCpu->dbgf.s.cEvents)
+                        memmove(&pVCpu->dbgf.s.aEvents[i], &pVCpu->dbgf.s.aEvents[i + 1],
+                                (pVCpu->dbgf.s.cEvents - i) * sizeof(pVCpu->dbgf.s.aEvents));
+                    pVCpu->dbgf.s.cEvents--;
+                }
+            }
+
+            i = pVCpu->dbgf.s.cEvents;
+            AssertStmt(i < RT_ELEMENTS(pVCpu->dbgf.s.aEvents), i = RT_ELEMENTS(pVCpu->dbgf.s.aEvents) - 1);
+        }
+
+        /*
+         * Push the event.
+         */
+        pVCpu->dbgf.s.aEvents[i].enmState               = DBGFEVENTSTATE_CURRENT;
+        pVCpu->dbgf.s.aEvents[i].rip                    = rip;
+        pVCpu->dbgf.s.aEvents[i].Event.enmType          = enmEvent;
+        pVCpu->dbgf.s.aEvents[i].Event.enmCtx           = enmCtx;
+        pVCpu->dbgf.s.aEvents[i].Event.u.Generic.cArgs  = cArgs;
+        pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs[0] = uEventArg0;
+        if (cArgs > 1)
+        {
+            AssertStmt(cArgs < RT_ELEMENTS(pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs),
+                       cArgs = RT_ELEMENTS(pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs));
+            for (unsigned iArg = 1; iArg < cArgs; iArg++)
+                pVCpu->dbgf.s.aEvents[i].Event.u.Generic.auArgs[iArg] = va_arg(va, uint64_t);
+        }
+        pVCpu->dbgf.s.cEvents = i + 1;
+
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_DBGF);
+        va_end(va);
+        return VINF_EM_DBG_EVENT;
+    }
+
+    va_end(va);
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMAll/DBGFAllBp.cpp b/src/VBox/VMM/VMMAll/DBGFAllBp.cpp
new file mode 100644
index 00000000..b6bb2669
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/DBGFAllBp.cpp
@@ -0,0 +1,604 @@
+/* $Id: DBGFAllBp.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, All Context breakpoint management part.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/log.h>
+#include "DBGFInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+
+#include "DBGFInline.h"
+
+
+#ifdef IN_RC
+# error "You lucky person have the pleasure to implement the raw mode part for this!"
+#endif
+
+
+/**
+ * Returns the internal breakpoint state for the given handle.
+ *
+ * @returns Pointer to the internal breakpoint state or NULL if the handle is invalid.
+ * @param   pVM                 The ring-0 VM structure pointer.
+ * @param   hBp                 The breakpoint handle to resolve.
+ * @param   ppBpR0              Where to store the pointer to the ring-0 only part of the breakpoint
+ *                              on success, optional.
+ */
+#ifdef IN_RING0
+DECLINLINE(PDBGFBPINT) dbgfBpGetByHnd(PVMCC pVM, DBGFBP hBp, PDBGFBPINTR0 *ppBpR0)
+#else
+DECLINLINE(PDBGFBPINT) dbgfBpGetByHnd(PVMCC pVM, DBGFBP hBp)
+#endif
+{
+    uint32_t idChunk  = DBGF_BP_HND_GET_CHUNK_ID(hBp);
+    uint32_t idxEntry = DBGF_BP_HND_GET_ENTRY(hBp);
+
+    AssertReturn(idChunk < DBGF_BP_CHUNK_COUNT, NULL);
+    AssertReturn(idxEntry < DBGF_BP_COUNT_PER_CHUNK, NULL);
+
+#ifdef IN_RING0
+    PDBGFBPCHUNKR0 pBpChunk = &pVM->dbgfr0.s.aBpChunks[idChunk];
+    AssertPtrReturn(pBpChunk->CTX_SUFF(paBpBaseShared), NULL);
+
+    if (ppBpR0)
+        *ppBpR0 = &pBpChunk->paBpBaseR0Only[idxEntry];
+    return &pBpChunk->CTX_SUFF(paBpBaseShared)[idxEntry];
+
+#elif defined(IN_RING3)
+    PUVM pUVM = pVM->pUVM;
+    PDBGFBPCHUNKR3 pBpChunk = &pUVM->dbgf.s.aBpChunks[idChunk];
+    AssertPtrReturn(pBpChunk->CTX_SUFF(pBpBase), NULL);
+
+    return &pBpChunk->CTX_SUFF(pBpBase)[idxEntry];
+
+#else
+# error "Unsupported context"
+#endif
+}
+
+
+/**
+ * Returns the pointer to the L2 table entry from the given index.
+ *
+ * @returns Current context pointer to the L2 table entry or NULL if the provided index value is invalid.
+ * @param   pVM         The cross context VM structure.
+ * @param   idxL2       The L2 table index to resolve.
+ *
+ * @note The content of the resolved L2 table entry is not validated!.
+ */
+DECLINLINE(PCDBGFBPL2ENTRY) dbgfBpL2GetByIdx(PVMCC pVM, uint32_t idxL2)
+{
+    uint32_t idChunk  = DBGF_BP_L2_IDX_GET_CHUNK_ID(idxL2);
+    uint32_t idxEntry = DBGF_BP_L2_IDX_GET_ENTRY(idxL2);
+
+    AssertReturn(idChunk < DBGF_BP_L2_TBL_CHUNK_COUNT, NULL);
+    AssertReturn(idxEntry < DBGF_BP_L2_TBL_ENTRIES_PER_CHUNK, NULL);
+
+#ifdef IN_RING0
+    PDBGFBPL2TBLCHUNKR0 pL2Chunk = &pVM->dbgfr0.s.aBpL2TblChunks[idChunk];
+    AssertPtrReturn(pL2Chunk->CTX_SUFF(paBpL2TblBaseShared), NULL);
+
+    return &pL2Chunk->CTX_SUFF(paBpL2TblBaseShared)[idxEntry];
+#elif defined(IN_RING3)
+    PUVM pUVM = pVM->pUVM;
+    PDBGFBPL2TBLCHUNKR3 pL2Chunk = &pUVM->dbgf.s.aBpL2TblChunks[idChunk];
+    AssertPtrReturn(pL2Chunk->pbmAlloc, NULL);
+    AssertReturn(ASMBitTest(pL2Chunk->pbmAlloc, idxEntry), NULL);
+
+    return &pL2Chunk->CTX_SUFF(pL2Base)[idxEntry];
+#endif
+}
+
+
+#ifdef IN_RING0
+/**
+ * Returns the internal breakpoint owner state for the given handle.
+ *
+ * @returns Pointer to the internal ring-0 breakpoint owner state or NULL if the handle is invalid.
+ * @param   pVM                 The cross context VM structure.
+ * @param   hBpOwner            The breakpoint owner handle to resolve.
+ */
+DECLINLINE(PCDBGFBPOWNERINTR0) dbgfR0BpOwnerGetByHnd(PVMCC pVM, DBGFBPOWNER hBpOwner)
+{
+    if (hBpOwner == NIL_DBGFBPOWNER)
+        return NULL;
+
+    AssertReturn(hBpOwner < DBGF_BP_OWNER_COUNT_MAX, NULL);
+
+    PCDBGFBPOWNERINTR0 pBpOwnerR0 = &pVM->dbgfr0.s.paBpOwnersR0[hBpOwner];
+    AssertReturn(pBpOwnerR0->cRefs > 1, NULL);
+
+    return pBpOwnerR0;
+}
+#endif
+
+
+/**
+ * Executes the actions associated with the given breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the register context for the CPU.
+ * @param   hBp         The breakpoint handle which hit.
+ * @param   pBp         The shared breakpoint state.
+ * @param   pBpR0       The ring-0 only breakpoint state.
+ */
+#ifdef IN_RING0
+DECLINLINE(int) dbgfBpHit(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, DBGFBP hBp, PDBGFBPINT pBp, PDBGFBPINTR0 pBpR0)
+#else
+DECLINLINE(int) dbgfBpHit(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, DBGFBP hBp, PDBGFBPINT pBp)
+#endif
+{
+    uint64_t cHits = ASMAtomicIncU64(&pBp->Pub.cHits); RT_NOREF(cHits);
+
+    RT_NOREF(pCtx);
+    LogFlow(("dbgfBpHit: hit breakpoint %u at %04x:%RGv cHits=0x%RX64\n", hBp, pCtx->cs.Sel, pCtx->rip, cHits));
+
+    int rc = VINF_EM_DBG_BREAKPOINT;
+#ifdef IN_RING0
+    PCDBGFBPOWNERINTR0 pBpOwnerR0 = dbgfR0BpOwnerGetByHnd(pVM,
+                                                            pBpR0->fInUse
+                                                          ? pBpR0->hOwner
+                                                          : NIL_DBGFBPOWNER);
+    if (pBpOwnerR0)
+    {
+        AssertReturn(pBpOwnerR0->pfnBpIoHitR0, VERR_DBGF_BP_IPE_1);
+
+        VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+
+        if (DBGF_BP_PUB_IS_EXEC_BEFORE(&pBp->Pub))
+            rcStrict = pBpOwnerR0->pfnBpHitR0(pVM, pVCpu->idCpu, pBpR0->pvUserR0, hBp, &pBp->Pub, DBGF_BP_F_HIT_EXEC_BEFORE);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            uint8_t abInstr[DBGF_BP_INSN_MAX];
+            RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
+            rc = PGMPhysSimpleReadGCPtr(pVCpu, &abInstr[0], GCPtrInstr, sizeof(abInstr));
+            AssertRC(rc);
+            if (RT_SUCCESS(rc))
+            {
+                /* Replace the int3 with the original instruction byte. */
+                abInstr[0] = pBp->Pub.u.Int3.bOrg;
+                rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, GCPtrInstr, &abInstr[0], sizeof(abInstr));
+                if (   rcStrict == VINF_SUCCESS
+                    && DBGF_BP_PUB_IS_EXEC_AFTER(&pBp->Pub))
+                {
+                    rcStrict = pBpOwnerR0->pfnBpHitR0(pVM, pVCpu->idCpu, pBpR0->pvUserR0, hBp, &pBp->Pub, DBGF_BP_F_HIT_EXEC_AFTER);
+                    if (rcStrict == VINF_SUCCESS)
+                        rc = VINF_SUCCESS;
+                    else if (   rcStrict == VINF_DBGF_BP_HALT
+                             || rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER)
+                    {
+                        pVCpu->dbgf.s.hBpActive = hBp;
+                        if (rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER)
+                            pVCpu->dbgf.s.fBpInvokeOwnerCallback = true;
+                        else
+                            pVCpu->dbgf.s.fBpInvokeOwnerCallback = false;
+                    }
+                    else /* Guru meditation. */
+                        rc = VERR_DBGF_BP_OWNER_CALLBACK_WRONG_STATUS;
+                }
+                else
+                    rc = VBOXSTRICTRC_VAL(rcStrict);
+            }
+        }
+        else if (   rcStrict == VINF_DBGF_BP_HALT
+                 || rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER)
+        {
+            pVCpu->dbgf.s.hBpActive = hBp;
+            if (rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER)
+                pVCpu->dbgf.s.fBpInvokeOwnerCallback = true;
+            else
+                pVCpu->dbgf.s.fBpInvokeOwnerCallback = false;
+        }
+        else /* Guru meditation. */
+            rc = VERR_DBGF_BP_OWNER_CALLBACK_WRONG_STATUS;
+    }
+    else
+    {
+        pVCpu->dbgf.s.fBpInvokeOwnerCallback = true; /* Need to check this for ring-3 only owners. */
+        pVCpu->dbgf.s.hBpActive              = hBp;
+    }
+#else
+    RT_NOREF(pVM);
+    pVCpu->dbgf.s.fBpInvokeOwnerCallback = true;
+    pVCpu->dbgf.s.hBpActive = hBp;
+#endif
+
+    return rc;
+}
+
+
+/**
+ * Executes the actions associated with the given port I/O breakpoint.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   fBefore     Flag whether the check is done before the access is carried out,
+ *                      false if it is done after the access.
+ * @param   fAccess     Access flags, see DBGFBPIOACCESS_XXX.
+ * @param   uAddr       The address of the access, for port I/O this will hold the port number.
+ * @param   uValue      The value read or written (the value for reads is only valid when DBGF_BP_F_HIT_EXEC_AFTER is set).
+ * @param   hBp         The breakpoint handle which hit.
+ * @param   pBp         The shared breakpoint state.
+ * @param   pBpR0       The ring-0 only breakpoint state.
+ */
+#ifdef IN_RING0
+DECLINLINE(VBOXSTRICTRC) dbgfBpPortIoHit(PVMCC pVM, PVMCPU pVCpu, bool fBefore, uint32_t fAccess, uint64_t uAddr, uint64_t uValue,
+                                         DBGFBP hBp, PDBGFBPINT pBp, PDBGFBPINTR0 pBpR0)
+#else
+DECLINLINE(VBOXSTRICTRC) dbgfBpPortIoHit(PVMCC pVM, PVMCPU pVCpu, bool fBefore, uint32_t fAccess, uint64_t uAddr, uint64_t uValue,
+                                         DBGFBP hBp, PDBGFBPINT pBp)
+#endif
+{
+    ASMAtomicIncU64(&pBp->Pub.cHits);
+
+    VBOXSTRICTRC rcStrict = VINF_EM_DBG_BREAKPOINT;
+#ifdef IN_RING0
+    PCDBGFBPOWNERINTR0 pBpOwnerR0 = dbgfR0BpOwnerGetByHnd(pVM,
+                                                            pBpR0->fInUse
+                                                          ? pBpR0->hOwner
+                                                          : NIL_DBGFBPOWNER);
+    if (pBpOwnerR0)
+    {
+        AssertReturn(pBpOwnerR0->pfnBpIoHitR0, VERR_DBGF_BP_IPE_1);
+        rcStrict = pBpOwnerR0->pfnBpIoHitR0(pVM, pVCpu->idCpuUnsafe, pBpR0->pvUserR0, hBp, &pBp->Pub,
+                                              fBefore
+                                            ? DBGF_BP_F_HIT_EXEC_BEFORE
+                                            : DBGF_BP_F_HIT_EXEC_AFTER,
+                                            fAccess, uAddr, uValue);
+    }
+    else
+    {
+        pVCpu->dbgf.s.fBpInvokeOwnerCallback = true; /* Need to check this for ring-3 only owners. */
+        pVCpu->dbgf.s.hBpActive              = hBp;
+        pVCpu->dbgf.s.fBpIoActive            = true;
+        pVCpu->dbgf.s.fBpIoBefore            = fBefore;
+        pVCpu->dbgf.s.uBpIoAddress           = uAddr;
+        pVCpu->dbgf.s.fBpIoAccess            = fAccess;
+        pVCpu->dbgf.s.uBpIoValue             = uValue;
+    }
+#else
+    /* Resolve owner (can be NIL_DBGFBPOWNER) and invoke callback if there is one. */
+    if (pBp->Pub.hOwner != NIL_DBGFBPOWNER)
+    {
+        PCDBGFBPOWNERINT pBpOwner = dbgfR3BpOwnerGetByHnd(pVM->pUVM, pBp->Pub.hOwner);
+        if (pBpOwner)
+        {
+            AssertReturn(pBpOwner->pfnBpIoHitR3, VERR_DBGF_BP_IPE_1);
+            rcStrict = pBpOwner->pfnBpIoHitR3(pVM, pVCpu->idCpu, pBp->pvUserR3, hBp, &pBp->Pub,
+                                                fBefore
+                                              ? DBGF_BP_F_HIT_EXEC_BEFORE
+                                              : DBGF_BP_F_HIT_EXEC_AFTER,
+                                              fAccess, uAddr, uValue);
+        }
+    }
+#endif
+    if (   rcStrict == VINF_DBGF_BP_HALT
+        || rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER)
+    {
+        pVCpu->dbgf.s.hBpActive = hBp;
+        if (rcStrict == VINF_DBGF_R3_BP_OWNER_DEFER)
+            pVCpu->dbgf.s.fBpInvokeOwnerCallback = true;
+        else
+            pVCpu->dbgf.s.fBpInvokeOwnerCallback = false;
+        rcStrict = VINF_EM_DBG_BREAKPOINT;
+    }
+    else if (   rcStrict != VINF_SUCCESS
+             && rcStrict != VINF_EM_DBG_BREAKPOINT)
+        rcStrict = VERR_DBGF_BP_OWNER_CALLBACK_WRONG_STATUS; /* Guru meditation. */
+
+    return rcStrict;
+}
+
+
+/**
+ * Walks the L2 table starting at the given root index searching for the given key.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the register context for the CPU.
+ * @param   idxL2Root   L2 table index of the table root.
+ * @param   GCPtrKey    The key to search for.
+ */
+static int dbgfBpL2Walk(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, uint32_t idxL2Root, RTGCUINTPTR GCPtrKey)
+{
+    /** @todo We don't use the depth right now but abort the walking after a fixed amount of levels. */
+    uint8_t iDepth = 32;
+    PCDBGFBPL2ENTRY pL2Entry = dbgfBpL2GetByIdx(pVM, idxL2Root);
+
+    while (RT_LIKELY(   iDepth-- > 0
+                     && pL2Entry))
+    {
+        /* Make a copy of the entry before verification. */
+        DBGFBPL2ENTRY L2Entry;
+        L2Entry.u64GCPtrKeyAndBpHnd1       = ASMAtomicReadU64((volatile uint64_t *)&pL2Entry->u64GCPtrKeyAndBpHnd1);
+        L2Entry.u64LeftRightIdxDepthBpHnd2 = ASMAtomicReadU64((volatile uint64_t *)&pL2Entry->u64LeftRightIdxDepthBpHnd2);
+
+        RTGCUINTPTR GCPtrL2Entry = DBGF_BP_L2_ENTRY_GET_GCPTR(L2Entry.u64GCPtrKeyAndBpHnd1);
+        if (GCPtrKey == GCPtrL2Entry)
+        {
+            DBGFBP hBp = DBGF_BP_L2_ENTRY_GET_BP_HND(L2Entry.u64GCPtrKeyAndBpHnd1, L2Entry.u64LeftRightIdxDepthBpHnd2);
+
+            /* Query the internal breakpoint state from the handle. */
+#ifdef IN_RING3
+            PDBGFBPINT   pBp = dbgfBpGetByHnd(pVM, hBp);
+#else
+            PDBGFBPINTR0 pBpR0 = NULL;
+            PDBGFBPINT   pBp = dbgfBpGetByHnd(pVM, hBp, &pBpR0);
+#endif
+            if (   pBp
+                && DBGF_BP_PUB_GET_TYPE(&pBp->Pub) == DBGFBPTYPE_INT3)
+#ifdef IN_RING3
+                return dbgfBpHit(pVM, pVCpu, pCtx, hBp, pBp);
+#else
+                return dbgfBpHit(pVM, pVCpu, pCtx, hBp, pBp, pBpR0);
+#endif
+
+            /* The entry got corrupted, just abort. */
+            return VERR_DBGF_BP_L2_LOOKUP_FAILED;
+        }
+
+        /* Not found, get to the next level. */
+        uint32_t idxL2Next =   (GCPtrKey < GCPtrL2Entry)
+                             ? DBGF_BP_L2_ENTRY_GET_IDX_LEFT(L2Entry.u64LeftRightIdxDepthBpHnd2)
+                             : DBGF_BP_L2_ENTRY_GET_IDX_RIGHT(L2Entry.u64LeftRightIdxDepthBpHnd2);
+        /* It is genuine guest trap or we hit some assertion if we are at the end. */
+        if (idxL2Next == DBGF_BP_L2_ENTRY_IDX_END)
+            return VINF_EM_RAW_GUEST_TRAP;
+
+        pL2Entry = dbgfBpL2GetByIdx(pVM, idxL2Next);
+    }
+
+    return VERR_DBGF_BP_L2_LOOKUP_FAILED;
+}
+
+
+/**
+ * Checks whether there is a port I/O breakpoint for the given range and access size.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_EM_DBG_BREAKPOINT means there is a breakpoint pending.
+ * @retval  VINF_SUCCESS means everything is fine to continue.
+ * @retval  anything else means a fatal error causing a guru meditation.
+ *
+ * @param   pVM             The current context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uIoPort         The I/O port being accessed.
+ * @param   fAccess         Appropriate DBGFBPIOACCESS_XXX.
+ * @param   uValue          The value being written to or read from the device
+ *                          (The value is only valid for a read when the
+ *                          call is made after the access, writes are always valid).
+ * @param   fBefore         Flag whether the check is done before the access is carried out,
+ *                          false if it is done after the access.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) DBGFBpCheckPortIo(PVMCC pVM, PVMCPU pVCpu, RTIOPORT uIoPort,
+                                             uint32_t fAccess, uint32_t uValue, bool fBefore)
+{
+    RT_NOREF(uValue); /** @todo Trigger only on specific values. */
+
+    /* Don't trigger in single stepping mode. */
+    if (pVCpu->dbgf.s.fSingleSteppingRaw)
+        return VINF_SUCCESS;
+
+#if defined(IN_RING0)
+    uint32_t volatile *paBpLocPortIo = pVM->dbgfr0.s.CTX_SUFF(paBpLocPortIo);
+#elif defined(IN_RING3)
+    PUVM pUVM = pVM->pUVM;
+    uint32_t volatile *paBpLocPortIo = pUVM->dbgf.s.CTX_SUFF(paBpLocPortIo);
+#else
+# error "Unsupported host context"
+#endif
+    if (paBpLocPortIo)
+    {
+        const uint32_t u32Entry = ASMAtomicReadU32(&paBpLocPortIo[uIoPort]);
+        if (u32Entry != DBGF_BP_INT3_L1_ENTRY_TYPE_NULL)
+        {
+            uint8_t u8Type = DBGF_BP_INT3_L1_ENTRY_GET_TYPE(u32Entry);
+            if (RT_LIKELY(u8Type == DBGF_BP_INT3_L1_ENTRY_TYPE_BP_HND))
+            {
+                DBGFBP hBp = DBGF_BP_INT3_L1_ENTRY_GET_BP_HND(u32Entry);
+
+                /* Query the internal breakpoint state from the handle. */
+#ifdef IN_RING3
+                PDBGFBPINT   pBp = dbgfBpGetByHnd(pVM, hBp);
+#else
+                PDBGFBPINTR0 pBpR0 = NULL;
+                PDBGFBPINT   pBp = dbgfBpGetByHnd(pVM, hBp, &pBpR0);
+#endif
+                if (   pBp
+                    && DBGF_BP_PUB_GET_TYPE(&pBp->Pub) == DBGFBPTYPE_PORT_IO)
+                {
+                    if (   uIoPort >= pBp->Pub.u.PortIo.uPort
+                        && uIoPort < pBp->Pub.u.PortIo.uPort + pBp->Pub.u.PortIo.cPorts
+                        && pBp->Pub.u.PortIo.fAccess & fAccess
+                        && (   (   fBefore
+                                && DBGF_BP_PUB_IS_EXEC_BEFORE(&pBp->Pub))
+                            || (   !fBefore
+                                && DBGF_BP_PUB_IS_EXEC_AFTER(&pBp->Pub))))
+#ifdef IN_RING3
+                        return dbgfBpPortIoHit(pVM, pVCpu, fBefore, fAccess, uIoPort, uValue, hBp, pBp);
+#else
+                        return dbgfBpPortIoHit(pVM, pVCpu, fBefore, fAccess, uIoPort, uValue, hBp, pBp, pBpR0);
+#endif
+                    /* else: No matching port I/O breakpoint. */
+                }
+                else /* Invalid breakpoint handle or not an port I/O breakpoint. */
+                    return VERR_DBGF_BP_L1_LOOKUP_FAILED;
+            }
+            else /* Some invalid type. */
+                return VERR_DBGF_BP_L1_LOOKUP_FAILED;
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * \#DB (Debug event) handler.
+ *
+ * @returns VBox status code.
+ *          VINF_SUCCESS means we completely handled this trap,
+ *          other codes are passed execution to host context.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Pointer to the register context for the CPU.
+ * @param   uDr6            The DR6 hypervisor register value.
+ * @param   fAltStepping    Alternative stepping indicator.
+ */
+VMM_INT_DECL(int) DBGFTrap01Handler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTGCUINTREG uDr6, bool fAltStepping)
+{
+    /** @todo Intel docs say that X86_DR6_BS has the highest priority... */
+    RT_NOREF(pCtx);
+
+    /*
+     * A breakpoint?
+     */
+    AssertCompile(X86_DR6_B0 == 1 && X86_DR6_B1 == 2 && X86_DR6_B2 == 4 && X86_DR6_B3 == 8);
+    if (   (uDr6 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3))
+        && pVM->dbgf.s.cEnabledHwBreakpoints > 0)
+    {
+        for (unsigned iBp = 0; iBp < RT_ELEMENTS(pVM->dbgf.s.aHwBreakpoints); iBp++)
+            if (    ((uint32_t)uDr6 & RT_BIT_32(iBp))
+                &&  pVM->dbgf.s.aHwBreakpoints[iBp].hBp != NIL_DBGFBP)
+            {
+                pVCpu->dbgf.s.hBpActive = pVM->dbgf.s.aHwBreakpoints[iBp].hBp;
+                pVCpu->dbgf.s.fSingleSteppingRaw = false;
+                LogFlow(("DBGFRZTrap03Handler: hit hw breakpoint %x at %04x:%RGv\n",
+                         pVM->dbgf.s.aHwBreakpoints[iBp].hBp, pCtx->cs.Sel, pCtx->rip));
+
+                return VINF_EM_DBG_BREAKPOINT;
+            }
+    }
+
+    /*
+     * Single step?
+     * Are we single stepping or is it the guest?
+     */
+    if (   (uDr6 & X86_DR6_BS)
+        && (pVCpu->dbgf.s.fSingleSteppingRaw || fAltStepping))
+    {
+        pVCpu->dbgf.s.fSingleSteppingRaw = false;
+        LogFlow(("DBGFRZTrap01Handler: single step at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
+        return VINF_EM_DBG_STEPPED;
+    }
+
+    LogFlow(("DBGFRZTrap01Handler: guest debug event %#x at %04x:%RGv!\n", (uint32_t)uDr6, pCtx->cs.Sel, pCtx->rip));
+    return VINF_EM_RAW_GUEST_TRAP;
+}
+
+
+/**
+ * \#BP (Breakpoint) handler.
+ *
+ * @returns VBox status code.
+ *          VINF_SUCCESS means we completely handled this trap,
+ *          other codes are passed execution to host context.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the register context for the CPU.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) DBGFTrap03Handler(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+#if defined(IN_RING0)
+    uint32_t volatile *paBpLocL1 = pVM->dbgfr0.s.CTX_SUFF(paBpLocL1);
+#elif defined(IN_RING3)
+    PUVM pUVM = pVM->pUVM;
+    uint32_t volatile *paBpLocL1 = pUVM->dbgf.s.CTX_SUFF(paBpLocL1);
+#else
+# error "Unsupported host context"
+#endif
+    if (paBpLocL1)
+    {
+        RTGCPTR GCPtrBp;
+        int rc = SELMValidateAndConvertCSAddr(pVCpu, pCtx->eflags.u, pCtx->ss.Sel, pCtx->cs.Sel, &pCtx->cs,
+                                              pCtx->rip /* no -1 outside non-rawmode */, &GCPtrBp);
+        AssertRCReturn(rc, rc);
+
+        const uint16_t idxL1      = DBGF_BP_INT3_L1_IDX_EXTRACT_FROM_ADDR(GCPtrBp);
+        const uint32_t u32L1Entry = ASMAtomicReadU32(&paBpLocL1[idxL1]);
+
+        LogFlowFunc(("GCPtrBp=%RGv idxL1=%u u32L1Entry=%#x\n", GCPtrBp, idxL1, u32L1Entry));
+        if (u32L1Entry != DBGF_BP_INT3_L1_ENTRY_TYPE_NULL)
+        {
+            uint8_t u8Type = DBGF_BP_INT3_L1_ENTRY_GET_TYPE(u32L1Entry);
+            if (u8Type == DBGF_BP_INT3_L1_ENTRY_TYPE_BP_HND)
+            {
+                DBGFBP hBp = DBGF_BP_INT3_L1_ENTRY_GET_BP_HND(u32L1Entry);
+
+                /* Query the internal breakpoint state from the handle. */
+#ifdef IN_RING3
+                PDBGFBPINT   pBp = dbgfBpGetByHnd(pVM, hBp);
+#else
+                PDBGFBPINTR0 pBpR0 = NULL;
+                PDBGFBPINT   pBp = dbgfBpGetByHnd(pVM, hBp, &pBpR0);
+#endif
+                if (   pBp
+                    && DBGF_BP_PUB_GET_TYPE(&pBp->Pub) == DBGFBPTYPE_INT3)
+                {
+                    if (pBp->Pub.u.Int3.GCPtr == (RTGCUINTPTR)GCPtrBp)
+#ifdef IN_RING3
+                        rc = dbgfBpHit(pVM, pVCpu, pCtx, hBp, pBp);
+#else
+                        rc = dbgfBpHit(pVM, pVCpu, pCtx, hBp, pBp, pBpR0);
+#endif
+                    else
+                        rc = VINF_EM_RAW_GUEST_TRAP; /* Genuine guest trap. */
+                }
+                else /* Invalid breakpoint handle or not an int3 breakpoint. */
+                    rc = VERR_DBGF_BP_L1_LOOKUP_FAILED;
+            }
+            else if (u8Type == DBGF_BP_INT3_L1_ENTRY_TYPE_L2_IDX)
+                rc = dbgfBpL2Walk(pVM, pVCpu, pCtx, DBGF_BP_INT3_L1_ENTRY_GET_L2_IDX(u32L1Entry),
+                                  DBGF_BP_INT3_L2_KEY_EXTRACT_FROM_ADDR((RTGCUINTPTR)GCPtrBp));
+            else /* Some invalid type. */
+                rc = VERR_DBGF_BP_L1_LOOKUP_FAILED;
+        }
+        else
+            rc = VINF_EM_RAW_GUEST_TRAP; /* Genuine guest trap. */
+
+        return rc;
+    }
+
+    return VINF_EM_RAW_GUEST_TRAP;
+}
+
diff --git a/src/VBox/VMM/VMMAll/DBGFAllTracer.cpp b/src/VBox/VMM/VMMAll/DBGFAllTracer.cpp
new file mode 100644
index 00000000..8e79602b
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/DBGFAllTracer.cpp
@@ -0,0 +1,715 @@
+/* $Id: DBGFAllTracer.cpp $ */
+/** @file
+ * DBGF - Debugger Facility, All Context Code tracing part.
+ */
+
+/*
+ * Copyright (C) 2020-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_DBGF
+#include "DBGFInternal.h"
+#include <VBox/types.h>
+#include <VBox/err.h>
+#include <VBox/vmm/dbgf.h>
+#if defined(IN_RING3)
+# include <VBox/vmm/uvm.h>
+# include <VBox/vmm/vm.h>
+#elif defined(IN_RING0)
+# include <VBox/vmm/gvm.h>
+#else
+# error "Invalid environment"
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+
+/**
+ * Returns the current context tracer instance of the given VM instance.
+ *
+ * @returns Current context pointer to the DBGF tracer instance.
+ */
+DECLINLINE(PDBGFTRACERINSCC) dbgfTracerGetInstance(PVMCC pVM)
+{
+#if defined(IN_RING0)
+     return pVM->dbgfr0.s.pTracerR0;
+#elif defined(IN_RING3)
+     PUVM pUVM = pVM->pUVM;
+     return pUVM->dbgf.s.pTracerR3;
+#elif defined(IN_RC)
+# error "Not implemented"
+#else
+# error "No/Invalid context specified"
+#endif
+}
+
+
+/**
+ * Returns the size of the tracing ring buffer.
+ *
+ * @returns Size of the ring buffer in bytes.
+ * @param   pThisCC                 The event tracer instance current context data.
+ */
+DECLINLINE(size_t) dbgfTracerGetRingBufSz(PDBGFTRACERINSCC pThisCC)
+{
+#if defined(IN_RING0) /* For R0 we are extra cautious and use the ring buffer size stored in R0 memory so R3 can't corrupt it. */
+    return pThisCC->cbRingBuf;
+#else
+    return pThisCC->CTX_SUFF(pShared)->cbRingBuf;
+#endif
+}
+
+
+/**
+ * Posts a single event descriptor to the ring buffer of the given tracer instance - extended version.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   pThisCC                 The event tracer instance current context data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   enmTraceEvt             The trace event type posted.
+ * @param   idEvtPrev               The previous event ID the posted event links to.
+ * @param   pvEvtDesc               The event descriptor to copy after the header.
+ * @param   cbEvtDesc               Size of the event descriptor.
+ * @param   pidEvt                  Where to store the assigned event ID, optional.
+ */
+static int dbgfTracerEvtPostEx(PVMCC pVM, PDBGFTRACERINSCC pThisCC, DBGFTRACEREVTSRC hEvtSrc,
+                               DBGFTRACEREVT enmTraceEvt, uint64_t idEvtPrev, const void *pvEvtDesc,
+                               size_t cbEvtDesc, uint64_t *pidEvt)
+{
+    LogFlowFunc(("pVM=%p pThisCC=%p hEvtSrc=%llu enmTraceEvt=%u idEvtPrev=%llu pvEvtDesc=%p cbEvtDesc=%zu pidEvt=%p\n",
+                 pVM, pThisCC, hEvtSrc, enmTraceEvt, idEvtPrev, pvEvtDesc, cbEvtDesc, pidEvt));
+
+    PDBGFTRACERSHARED pSharedCC = pThisCC->CTX_SUFF(pShared);
+    size_t cRingBufEvts = dbgfTracerGetRingBufSz(pThisCC) / DBGF_TRACER_EVT_SZ;
+    AssertReturn(cRingBufEvts, VERR_DBGF_TRACER_IPE_1);
+    AssertReturn(cbEvtDesc <= DBGF_TRACER_EVT_PAYLOAD_SZ, VERR_DBGF_TRACER_IPE_1);
+
+    /* Grab a new event ID first. */
+    uint64_t idEvt = ASMAtomicIncU64(&pSharedCC->idEvt) - 1;
+    uint64_t idxRingBuf = idEvt % cRingBufEvts; /* This gives the index in the ring buffer for the event. */
+    PDBGFTRACEREVTHDR pEvtHdr = (PDBGFTRACEREVTHDR)(pThisCC->CTX_SUFF(pbRingBuf) + idxRingBuf * DBGF_TRACER_EVT_SZ);
+
+    if (RT_UNLIKELY(ASMAtomicReadU64(&pEvtHdr->idEvt) != DBGF_TRACER_EVT_HDR_ID_INVALID))
+    {
+        /** @todo The event ring buffer is full and we need to go back (from R0 to R3) and wait for the flusher thread to
+         * get its act together.
+         */
+        AssertMsgFailed(("Flush thread can't keep up with event amount!\n"));
+    }
+
+    /* Write the event and kick the flush thread if necessary. */
+    if (cbEvtDesc)
+        memcpy(pEvtHdr + 1, pvEvtDesc, cbEvtDesc);
+    pEvtHdr->idEvtPrev   = idEvtPrev;
+    pEvtHdr->hEvtSrc     = hEvtSrc;
+    pEvtHdr->enmEvt      = enmTraceEvt;
+    pEvtHdr->fFlags      = DBGF_TRACER_EVT_HDR_F_DEFAULT;
+    ASMAtomicWriteU64(&pEvtHdr->idEvt, idEvt);
+
+    int rc = VINF_SUCCESS;
+    if (!ASMAtomicXchgBool(&pSharedCC->fEvtsWaiting, true))
+    {
+        if (!ASMAtomicXchgBool(&pSharedCC->fFlushThrdActive, true))
+            rc = SUPSemEventSignal(pVM->pSession, pSharedCC->hSupSemEvtFlush);
+    }
+
+    if (pidEvt)
+        *pidEvt = idEvt;
+    return rc;
+}
+
+
+/**
+ * Posts a single event descriptor to the ring buffer of the given tracer instance.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   pThisCC                 The event tracer instance current context data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   enmTraceEvt             The trace event type posted.
+ * @param   pvEvtDesc               The event descriptor to copy after the header.
+ * @param   pidEvt                  Where to store the assigned event ID, optional.
+ */
+DECLINLINE(int) dbgfTracerEvtPostSingle(PVMCC pVM, PDBGFTRACERINSCC pThisCC, DBGFTRACEREVTSRC hEvtSrc,
+                                        DBGFTRACEREVT enmTraceEvt, const void *pvEvtDesc, uint64_t *pidEvt)
+{
+    return dbgfTracerEvtPostEx(pVM, pThisCC, hEvtSrc, enmTraceEvt, DBGF_TRACER_EVT_HDR_ID_INVALID,
+                               pvEvtDesc, DBGF_TRACER_EVT_PAYLOAD_SZ, pidEvt);
+}
+
+
+#ifdef IN_RING3
+/**
+ * Posts a single event descriptor to the ring buffer of the given tracer instance - R3 only variant
+ * (used for the register/deregister event source events currently).
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   pThisCC                 The event tracer instance current context data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   enmTraceEvt             The trace event type posted.
+ * @param   pvEvtDesc               The event descriptor to copy after the header.
+ * @param   cbEvtDesc               Event descriptor size in bytes.
+ * @param   pidEvt                  Where to store the assigned event ID, optional.
+ */
+DECLHIDDEN(int) dbgfTracerR3EvtPostSingle(PVMCC pVM, PDBGFTRACERINSCC pThisCC, DBGFTRACEREVTSRC hEvtSrc,
+                                          DBGFTRACEREVT enmTraceEvt, const void *pvEvtDesc, size_t cbEvtDesc,
+                                          uint64_t *pidEvt)
+{
+    return dbgfTracerEvtPostEx(pVM, pThisCC, hEvtSrc, enmTraceEvt, DBGF_TRACER_EVT_HDR_ID_INVALID,
+                               pvEvtDesc, cbEvtDesc, pidEvt);
+}
+#endif
+
+/**
+ * Copies the given MMIO value into the event descriptor based on the given size.
+ *
+ * @param   pEvtMmio                Pointer to the MMIO event descriptor to fill.
+ * @param   pvVal                   The value to copy.
+ * @param   cbVal                   Size of the value in bytes.
+ */
+static void dbgfTracerEvtMmioCopyVal(PDBGFTRACEREVTMMIO pEvtMmio, const void *pvVal, size_t cbVal)
+{
+    switch (cbVal)
+    {
+        case 1:
+            pEvtMmio->u64Val = *(uint8_t *)pvVal;
+            break;
+        case 2:
+            pEvtMmio->u64Val = *(uint16_t *)pvVal;
+            break;
+        case 4:
+            pEvtMmio->u64Val = *(uint32_t *)pvVal;
+            break;
+        case 8:
+            pEvtMmio->u64Val = *(uint64_t *)pvVal;
+            break;
+        default:
+            AssertMsgFailed(("The value size %zu is not supported!\n", cbVal));
+    }
+}
+
+
+/**
+ * Copies the given I/O port value into the event descriptor based on the given size.
+ *
+ * @param   pEvtIoPort              Pointer to the I/O port read/write event descriptor to fill.
+ * @param   pvVal                   The value to copy.
+ * @param   cbVal                   Size of the value in bytes.
+ */
+static void dbgfTracerEvtIoPortCopyVal(PDBGFTRACEREVTIOPORT pEvtIoPort, const void *pvVal, size_t cbVal)
+{
+    switch (cbVal)
+    {
+        case 1:
+            pEvtIoPort->u32Val = *(uint8_t *)pvVal;
+            break;
+        case 2:
+            pEvtIoPort->u32Val = *(uint16_t *)pvVal;
+            break;
+        case 4:
+            pEvtIoPort->u32Val = *(uint32_t *)pvVal;
+            break;
+        default:
+            AssertMsgFailed(("The value size %zu is not supported!\n", cbVal));
+    }
+}
+
+
+/**
+ * Handles a guest memory transfer event.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   pThisCC                 The event tracer instance current context data.
+ * @param   enmTraceEvt             The trace event type posted.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   GCPhys                  The guest physical address the transfer starts at.
+ * @param   pvBuf                   The data being transfered.
+ * @param   cbXfer                  Number of bytes being transfered.
+ */
+static int dbgfTracerEvtGCPhys(PVMCC pVM, PDBGFTRACERINSCC pThisCC, DBGFTRACEREVT enmTraceEvt, DBGFTRACEREVTSRC hEvtSrc,
+                               RTGCPHYS GCPhys, const void *pvBuf, size_t cbXfer)
+{
+    /* Fast path for really small transfers where everything fits into the descriptor. */
+    DBGFTRACEREVTGCPHYS EvtGCPhys;
+    EvtGCPhys.GCPhys = GCPhys;
+    EvtGCPhys.cbXfer = cbXfer;
+    if (cbXfer <= sizeof(EvtGCPhys.abData))
+    {
+        memcpy(&EvtGCPhys.abData[0], pvBuf, cbXfer);
+        return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, enmTraceEvt, &EvtGCPhys, NULL /*pidEvt*/);
+    }
+
+    /*
+     * Slow path where we have to split the data into multiple entries.
+     * Each one is linked to the previous one by the previous event ID.
+     */
+    const uint8_t *pbBuf = (const uint8_t *)pvBuf;
+    size_t cbLeft = cbXfer;
+    uint64_t idEvtPrev = 0;
+    memcpy(&EvtGCPhys.abData[0], pbBuf, sizeof(EvtGCPhys.abData));
+    pbBuf  += sizeof(EvtGCPhys.abData);
+    cbLeft -= sizeof(EvtGCPhys.abData);
+
+    int rc = dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, enmTraceEvt, &EvtGCPhys, &idEvtPrev);
+    while (   RT_SUCCESS(rc)
+           && cbLeft)
+    {
+        size_t cbThisXfer = RT_MIN(cbLeft, DBGF_TRACER_EVT_PAYLOAD_SZ);
+        rc = dbgfTracerEvtPostEx(pVM, pThisCC, hEvtSrc, enmTraceEvt, idEvtPrev,
+                                 pbBuf, cbThisXfer, &idEvtPrev);
+
+        pbBuf  += cbThisXfer;
+        cbLeft -= cbThisXfer;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Handles a I/O port string transfer event.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   pThisCC                 The event tracer instance current context data.
+ * @param   enmTraceEvt             The trace event type posted.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hIoPorts                The I/O port region handle for the transfer.
+ * @param   offPort                 The offset into the region where the transfer happened.
+ * @param   pv                      The data being transfered.
+ * @param   cb                      Number of bytes of valid data in the buffer.
+ * @param   cbItem                  Item size in bytes.
+ * @param   cTransfersReq           Number of transfers requested.
+ * @param   cTransfersRet           Number of transfers done.
+ */
+static int dbgfTracerEvtIoPortStr(PVMCC pVM, PDBGFTRACERINSCC pThisCC, DBGFTRACEREVT enmTraceEvt, DBGFTRACEREVTSRC hEvtSrc,
+                                  uint64_t hIoPorts, RTIOPORT offPort, const void *pv, size_t cb, size_t cbItem, uint32_t cTransfersReq,
+                                  uint32_t cTransfersRet)
+{
+    /* Fast path for really small transfers where everything fits into the descriptor. */
+    DBGFTRACEREVTIOPORTSTR EvtIoPortStr;
+    EvtIoPortStr.hIoPorts      = hIoPorts;
+    EvtIoPortStr.cbItem        = (uint32_t)cbItem;
+    EvtIoPortStr.cTransfersReq = cTransfersReq;
+    EvtIoPortStr.cTransfersRet = cTransfersRet;
+    EvtIoPortStr.offPort       = offPort;
+    if (cb <= sizeof(EvtIoPortStr.abData))
+    {
+        memcpy(&EvtIoPortStr.abData[0], pv, cb);
+        return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, enmTraceEvt, &EvtIoPortStr, NULL /*pidEvt*/);
+    }
+
+    /*
+     * Slow path where we have to split the data into multiple entries.
+     * Each one is linked to the previous one by the previous event ID.
+     */
+    const uint8_t *pbBuf = (const uint8_t *)pv;
+    size_t cbLeft = cb;
+    uint64_t idEvtPrev = 0;
+    memcpy(&EvtIoPortStr.abData[0], pbBuf, sizeof(EvtIoPortStr.abData));
+    pbBuf  += sizeof(EvtIoPortStr.abData);
+    cbLeft -= sizeof(EvtIoPortStr.abData);
+
+    int rc = dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, enmTraceEvt, &EvtIoPortStr, &idEvtPrev);
+    while (   RT_SUCCESS(rc)
+           && cbLeft)
+    {
+        size_t cbThisXfer = RT_MIN(cbLeft, DBGF_TRACER_EVT_PAYLOAD_SZ);
+        rc = dbgfTracerEvtPostEx(pVM, pThisCC, hEvtSrc, enmTraceEvt, idEvtPrev,
+                                 pbBuf, cbThisXfer, &idEvtPrev);
+
+        pbBuf  += cbThisXfer;
+        cbLeft -= cbThisXfer;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Registers an MMIO region mapping event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hRegion                 The MMIO region handle being mapped.
+ * @param   GCPhysMmio              The guest physical address where the region is mapped.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtMmioMap(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hRegion, RTGCPHYS GCPhysMmio)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTMMIOMAP EvtMmioMap;
+    EvtMmioMap.hMmioRegion     = hRegion;
+    EvtMmioMap.GCPhysMmioBase  = GCPhysMmio;
+    EvtMmioMap.au64Pad0[0]     = 0;
+    EvtMmioMap.au64Pad0[1]     = 0;
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_MMIO_MAP, &EvtMmioMap, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an MMIO region unmap event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hRegion                 The MMIO region handle being unmapped.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtMmioUnmap(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hRegion)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTMMIOUNMAP EvtMmioUnmap;
+    EvtMmioUnmap.hMmioRegion = hRegion;
+    EvtMmioUnmap.au64Pad0[0] = 0;
+    EvtMmioUnmap.au64Pad0[1] = 0;
+    EvtMmioUnmap.au64Pad0[2] = 0;
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_MMIO_UNMAP, &EvtMmioUnmap, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an MMIO region read event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hRegion                 The MMIO region handle being read.
+ * @param   offMmio                 The MMIO offset into the region where the read happened.
+ * @param   pvVal                   The value being read.
+ * @param   cbVal                   Value size in bytes.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtMmioRead(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hRegion, RTGCPHYS offMmio, const void *pvVal, size_t cbVal)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTMMIO EvtMmio;
+    EvtMmio.hMmioRegion = hRegion;
+    EvtMmio.offMmio     = offMmio;
+    EvtMmio.cbXfer      = cbVal;
+    dbgfTracerEvtMmioCopyVal(&EvtMmio, pvVal, cbVal);
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_MMIO_READ, &EvtMmio, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an MMIO region write event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hRegion                 The MMIO region handle being written to.
+ * @param   offMmio                 The MMIO offset into the region where the write happened.
+ * @param   pvVal                   The value being written.
+ * @param   cbVal                   Value size in bytes.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtMmioWrite(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hRegion, RTGCPHYS offMmio, const void *pvVal, size_t cbVal)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTMMIO EvtMmio;
+    EvtMmio.hMmioRegion = hRegion;
+    EvtMmio.offMmio     = offMmio;
+    EvtMmio.cbXfer      = cbVal;
+    dbgfTracerEvtMmioCopyVal(&EvtMmio, pvVal, cbVal);
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_MMIO_WRITE, &EvtMmio, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an MMIO region fill event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hRegion                 The MMIO region handle being filled.
+ * @param   offMmio                 The MMIO offset into the region where the fill starts.
+ * @param   u32Item                 The value being used for filling.
+ * @param   cbItem                  Item size in bytes.
+ * @param   cItems                  Number of items being written.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtMmioFill(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hRegion, RTGCPHYS offMmio,
+                                        uint32_t u32Item, uint32_t cbItem, uint32_t cItems)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTMMIOFILL EvtMmioFill;
+    EvtMmioFill.hMmioRegion = hRegion;
+    EvtMmioFill.offMmio     = offMmio;
+    EvtMmioFill.cbItem      = cbItem;
+    EvtMmioFill.cItems      = cItems;
+    EvtMmioFill.u32Item     = u32Item;
+    EvtMmioFill.u32Pad0     = 0;
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_MMIO_FILL, &EvtMmioFill, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an I/O region mapping event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hIoPorts                The I/O port region handle being mapped.
+ * @param   IoPortBase              The I/O port base where the region is mapped.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtIoPortMap(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts, RTIOPORT IoPortBase)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTIOPORTMAP EvtIoPortMap;
+    RT_ZERO(EvtIoPortMap);
+    EvtIoPortMap.hIoPorts    = hIoPorts;
+    EvtIoPortMap.IoPortBase  = IoPortBase;
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IOPORT_MAP, &EvtIoPortMap, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an I/O region unmap event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hIoPorts                The I/O port region handle being unmapped.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtIoPortUnmap(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTIOPORTUNMAP EvtIoPortUnmap;
+    EvtIoPortUnmap.hIoPorts    = hIoPorts;
+    EvtIoPortUnmap.au64Pad0[0] = 0;
+    EvtIoPortUnmap.au64Pad0[1] = 0;
+    EvtIoPortUnmap.au64Pad0[2] = 0;
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IOPORT_UNMAP, &EvtIoPortUnmap, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an I/O region read event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hIoPorts                The I/O port region handle being read from.
+ * @param   offPort                 The offset into the region where the read happened.
+ * @param   pvVal                   The value being read.
+ * @param   cbVal                   Value size in bytes.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtIoPortRead(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts, RTIOPORT offPort, const void *pvVal, size_t cbVal)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTIOPORT EvtIoPort;
+    RT_ZERO(EvtIoPort);
+    EvtIoPort.hIoPorts = hIoPorts;
+    EvtIoPort.offPort  = offPort;
+    EvtIoPort.cbXfer   = cbVal;
+    dbgfTracerEvtIoPortCopyVal(&EvtIoPort, pvVal, cbVal);
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IOPORT_READ, &EvtIoPort, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an I/O region string read event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hIoPorts                The I/O port region handle being read from.
+ * @param   offPort                 The offset into the region where the read happened.
+ * @param   pv                      The data being read.
+ * @param   cb                      Item size in bytes.
+ * @param   cTransfersReq           Number of transfers requested.
+ * @param   cTransfersRet           Number of transfers done.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtIoPortReadStr(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts, RTIOPORT offPort, const void *pv, size_t cb,
+                                             uint32_t cTransfersReq, uint32_t cTransfersRet)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    return dbgfTracerEvtIoPortStr(pVM, pThisCC, DBGFTRACEREVT_IOPORT_READ_STR, hEvtSrc, hIoPorts, offPort, pv, cTransfersRet * cb,
+                                  cb, cTransfersReq, cTransfersRet);
+}
+
+
+/**
+ * Registers an I/O region write event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hIoPorts                The I/O port region handle being written to.
+ * @param   offPort                 The offset into the region where the write happened.
+ * @param   pvVal                   The value being written.
+ * @param   cbVal                   Value size in bytes.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtIoPortWrite(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts, RTIOPORT offPort, const void *pvVal, size_t cbVal)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTIOPORT EvtIoPort;
+    RT_ZERO(EvtIoPort);
+    EvtIoPort.hIoPorts = hIoPorts;
+    EvtIoPort.offPort  = offPort;
+    EvtIoPort.cbXfer   = cbVal;
+    dbgfTracerEvtIoPortCopyVal(&EvtIoPort, pvVal, cbVal);
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IOPORT_WRITE, &EvtIoPort, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an I/O region string write event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   hIoPorts                The I/O port region handle being written to.
+ * @param   offPort                 The offset into the region where the write happened.
+ * @param   pv                      The data being written.
+ * @param   cb                      Item size in bytes.
+ * @param   cTransfersReq           Number of transfers requested.
+ * @param   cTransfersRet           Number of transfers done.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtIoPortWriteStr(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, uint64_t hIoPorts, RTIOPORT offPort, const void *pv, size_t cb,
+                                              uint32_t cTransfersReq, uint32_t cTransfersRet)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    return dbgfTracerEvtIoPortStr(pVM, pThisCC, DBGFTRACEREVT_IOPORT_WRITE_STR, hEvtSrc, hIoPorts, offPort, pv, cTransfersReq * cb,
+                                  cb, cTransfersReq, cTransfersRet);
+}
+
+
+/**
+ * Registers an IRQ change event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   iIrq                    The IRQ line changed.
+ * @param   fIrqLvl                 The new IRQ level mask.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtIrq(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, int32_t iIrq, int32_t fIrqLvl)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTIRQ EvtIrq;
+    RT_ZERO(EvtIrq);
+    EvtIrq.iIrq    = iIrq;
+    EvtIrq.fIrqLvl = fIrqLvl;
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IRQ, &EvtIrq, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an I/O APIC MSI event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   GCPhys                  Guest physical address where the value is written to.
+ * @param   u32Val                  The MSI event value being written.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtIoApicMsi(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, RTGCPHYS GCPhys, uint32_t u32Val)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    DBGFTRACEREVTIOAPICMSI EvtMsi;
+    RT_ZERO(EvtMsi);
+    EvtMsi.GCPhys = GCPhys;
+    EvtMsi.u32Val = u32Val;
+
+    return dbgfTracerEvtPostSingle(pVM, pThisCC, hEvtSrc, DBGFTRACEREVT_IOAPIC_MSI, &EvtMsi, NULL /*pidEvt*/);
+}
+
+
+/**
+ * Registers an guest physical memory read event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   GCPhys                  Guest physical address the read started at.
+ * @param   pvBuf                   The read data.
+ * @param   cbRead                  Number of bytes read.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtGCPhysRead(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, RTGCPHYS GCPhys, const void *pvBuf, size_t cbRead)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    return dbgfTracerEvtGCPhys(pVM, pThisCC, DBGFTRACEREVT_GCPHYS_READ, hEvtSrc, GCPhys, pvBuf, cbRead);
+}
+
+
+/**
+ * Registers an guest physical memory write event for the given event source.
+ *
+ * @returns VBox status code.
+ * @param   pVM                     The current context VM instance data.
+ * @param   hEvtSrc                 The event source for the posted event.
+ * @param   GCPhys                  Guest physical address the write started at.
+ * @param   pvBuf                   The written data.
+ * @param   cbWrite                 Number of bytes written.
+ */
+VMM_INT_DECL(int) DBGFTracerEvtGCPhysWrite(PVMCC pVM, DBGFTRACEREVTSRC hEvtSrc, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite)
+{
+    PDBGFTRACERINSCC pThisCC = dbgfTracerGetInstance(pVM);
+    AssertReturn(pThisCC, VERR_DBGF_TRACER_IPE_1);
+
+    return dbgfTracerEvtGCPhys(pVM, pThisCC, DBGFTRACEREVT_GCPHYS_WRITE, hEvtSrc, GCPhys, pvBuf, cbWrite);
+}
+
diff --git a/src/VBox/VMM/VMMAll/EMAll.cpp b/src/VBox/VMM/VMMAll/EMAll.cpp
new file mode 100644
index 00000000..bc5106d3
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/EMAll.cpp
@@ -0,0 +1,1018 @@
+/* $Id: EMAll.cpp $ */
+/** @file
+ * EM - Execution Monitor(/Manager) - All contexts
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_EM
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/stam.h>
+#include "EMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+
+
+/**
+ * Get the current execution manager status.
+ *
+ * @returns Current status.
+ * @param   pVCpu         The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(EMSTATE) EMGetState(PVMCPU pVCpu)
+{
+    return pVCpu->em.s.enmState;
+}
+
+
+/**
+ * Sets the current execution manager status. (use only when you know what you're doing!)
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   enmNewState The new state, EMSTATE_WAIT_SIPI or EMSTATE_HALTED.
+ */
+VMM_INT_DECL(void)    EMSetState(PVMCPU pVCpu, EMSTATE enmNewState)
+{
+    /* Only allowed combination: */
+    Assert(pVCpu->em.s.enmState == EMSTATE_WAIT_SIPI && enmNewState == EMSTATE_HALTED);
+    pVCpu->em.s.enmState = enmNewState;
+}
+
+
+/**
+ * Enables / disable hypercall instructions.
+ *
+ * This interface is used by GIM to tell the execution monitors whether the
+ * hypercall instruction (VMMCALL & VMCALL) are allowed or should \#UD.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure this applies to.
+ * @param   fEnabled    Whether hypercall instructions are enabled (true) or not.
+ */
+VMMDECL(void) EMSetHypercallInstructionsEnabled(PVMCPU pVCpu, bool fEnabled)
+{
+    pVCpu->em.s.fHypercallEnabled = fEnabled;
+}
+
+
+/**
+ * Checks if hypercall instructions (VMMCALL & VMCALL) are enabled or not.
+ *
+ * @returns true if enabled, false if not.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @note    If this call becomes a performance factor, we can make the data
+ *          field available thru a read-only view in VMCPU.  See VM::cpum.ro.
+ */
+VMMDECL(bool) EMAreHypercallInstructionsEnabled(PVMCPU pVCpu)
+{
+    return pVCpu->em.s.fHypercallEnabled;
+}
+
+
+/**
+ * Prepare an MWAIT - essentials of the MONITOR instruction.
+ *
+ * @returns VINF_SUCCESS
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   rax                 The content of RAX.
+ * @param   rcx                 The content of RCX.
+ * @param   rdx                 The content of RDX.
+ * @param   GCPhys              The physical address corresponding to rax.
+ */
+VMM_INT_DECL(int) EMMonitorWaitPrepare(PVMCPU pVCpu, uint64_t rax, uint64_t rcx, uint64_t rdx, RTGCPHYS GCPhys)
+{
+    pVCpu->em.s.MWait.uMonitorRAX = rax;
+    pVCpu->em.s.MWait.uMonitorRCX = rcx;
+    pVCpu->em.s.MWait.uMonitorRDX = rdx;
+    pVCpu->em.s.MWait.fWait |= EMMWAIT_FLAG_MONITOR_ACTIVE;
+    /** @todo Make use of GCPhys. */
+    NOREF(GCPhys);
+    /** @todo Complete MONITOR implementation.  */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the monitor hardware is armed / active.
+ *
+ * @returns true if armed, false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ */
+VMM_INT_DECL(bool) EMMonitorIsArmed(PVMCPU pVCpu)
+{
+    return RT_BOOL(pVCpu->em.s.MWait.fWait & EMMWAIT_FLAG_MONITOR_ACTIVE);
+}
+
+
+/**
+ * Checks if we're in a MWAIT.
+ *
+ * @retval  1 if regular,
+ * @retval  > 1 if MWAIT with EMMWAIT_FLAG_BREAKIRQIF0
+ * @retval  0 if not armed
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ */
+VMM_INT_DECL(unsigned) EMMonitorWaitIsActive(PVMCPU pVCpu)
+{
+    uint32_t fWait = pVCpu->em.s.MWait.fWait;
+    AssertCompile(EMMWAIT_FLAG_ACTIVE == 1);
+    AssertCompile(EMMWAIT_FLAG_BREAKIRQIF0 == 2);
+    AssertCompile((EMMWAIT_FLAG_ACTIVE << 1) == EMMWAIT_FLAG_BREAKIRQIF0);
+    return fWait & (EMMWAIT_FLAG_ACTIVE | ((fWait & EMMWAIT_FLAG_ACTIVE) << 1));
+}
+
+
+/**
+ * Performs an MWAIT.
+ *
+ * @returns VINF_SUCCESS
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   rax                 The content of RAX.
+ * @param   rcx                 The content of RCX.
+ */
+VMM_INT_DECL(int) EMMonitorWaitPerform(PVMCPU pVCpu, uint64_t rax, uint64_t rcx)
+{
+    pVCpu->em.s.MWait.uMWaitRAX = rax;
+    pVCpu->em.s.MWait.uMWaitRCX = rcx;
+    pVCpu->em.s.MWait.fWait |= EMMWAIT_FLAG_ACTIVE;
+    if (rcx)
+        pVCpu->em.s.MWait.fWait |= EMMWAIT_FLAG_BREAKIRQIF0;
+    else
+        pVCpu->em.s.MWait.fWait &= ~EMMWAIT_FLAG_BREAKIRQIF0;
+    /** @todo not completely correct?? */
+    return VINF_EM_HALT;
+}
+
+
+/**
+ * Clears any address-range monitoring that is active.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ */
+VMM_INT_DECL(void) EMMonitorWaitClear(PVMCPU pVCpu)
+{
+    LogFlowFunc(("Clearing MWAIT\n"));
+    pVCpu->em.s.MWait.fWait &= ~(EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0);
+}
+
+
+/**
+ * Determine if we should continue execution in HM after encountering an mwait
+ * instruction.
+ *
+ * Clears MWAIT flags if returning @c true.
+ *
+ * @returns true if we should continue, false if we should halt.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Current CPU context.
+ */
+VMM_INT_DECL(bool) EMMonitorWaitShouldContinue(PVMCPU pVCpu, PCPUMCTX pCtx)
+{
+    if (CPUMGetGuestGif(pCtx))
+    {
+        if (   CPUMIsGuestPhysIntrEnabled(pVCpu)
+            || (   CPUMIsGuestInNestedHwvirtMode(pCtx)
+                && CPUMIsGuestVirtIntrEnabled(pVCpu))
+            || (   (pVCpu->em.s.MWait.fWait & (EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0))
+                ==                            (EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0)) )
+        {
+            if (VMCPU_FF_IS_ANY_SET(pVCpu, (  VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC
+                                            | VMCPU_FF_INTERRUPT_NESTED_GUEST)))
+            {
+                pVCpu->em.s.MWait.fWait &= ~(EMMWAIT_FLAG_ACTIVE | EMMWAIT_FLAG_BREAKIRQIF0);
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
+
+
+/**
+ * Determine if we should continue execution in HM after encountering a hlt
+ * instruction.
+ *
+ * @returns true if we should continue, false if we should halt.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Current CPU context.
+ */
+VMM_INT_DECL(bool) EMShouldContinueAfterHalt(PVMCPU pVCpu, PCPUMCTX pCtx)
+{
+    if (CPUMGetGuestGif(pCtx))
+    {
+        if (CPUMIsGuestPhysIntrEnabled(pVCpu))
+            return VMCPU_FF_IS_ANY_SET(pVCpu, (VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC));
+
+        if (   CPUMIsGuestInNestedHwvirtMode(pCtx)
+            && CPUMIsGuestVirtIntrEnabled(pVCpu))
+            return VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
+    }
+    return false;
+}
+
+
+/**
+ * Unhalts and wakes up the given CPU.
+ *
+ * This is an API for assisting the KVM hypercall API in implementing KICK_CPU.
+ * It sets VMCPU_FF_UNHALT for @a pVCpuDst and makes sure it is woken up.   If
+ * the CPU isn't currently in a halt, the next HLT instruction it executes will
+ * be affected.
+ *
+ * @returns GVMMR0SchedWakeUpEx result or VINF_SUCCESS depending on context.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpuDst        The cross context virtual CPU structure of the
+ *                          CPU to unhalt and wake up.  This is usually not the
+ *                          same as the caller.
+ * @thread  EMT
+ */
+VMM_INT_DECL(int) EMUnhaltAndWakeUp(PVMCC pVM, PVMCPUCC pVCpuDst)
+{
+    /*
+     * Flag the current(/next) HLT to unhalt immediately.
+     */
+    VMCPU_FF_SET(pVCpuDst, VMCPU_FF_UNHALT);
+
+    /*
+     * Wake up the EMT (technically should be abstracted by VMM/VMEmt, but
+     * just do it here for now).
+     */
+#ifdef IN_RING0
+    /* We might be here with preemption disabled or enabled (i.e. depending on
+       thread-context hooks being used), so don't try obtaining the GVMMR0 used
+       lock here. See @bugref{7270#c148}. */
+    int rc = GVMMR0SchedWakeUpNoGVMNoLock(pVM, pVCpuDst->idCpu);
+    AssertRC(rc);
+
+#elif defined(IN_RING3)
+    VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, 0 /*fFlags*/);
+    int rc = VINF_SUCCESS;
+    RT_NOREF(pVM);
+
+#else
+    /* Nothing to do for raw-mode, shouldn't really be used by raw-mode guests anyway. */
+    Assert(pVM->cCpus == 1); NOREF(pVM);
+    int rc = VINF_SUCCESS;
+#endif
+    return rc;
+}
+
+#ifndef IN_RING3
+
+/**
+ * Makes an I/O port write pending for ring-3 processing.
+ *
+ * @returns VINF_EM_PENDING_R3_IOPORT_READ
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uPort           The I/O port.
+ * @param   cbInstr         The instruction length (for RIP updating).
+ * @param   cbValue         The write size.
+ * @param   uValue          The value being written.
+ * @sa      emR3ExecutePendingIoPortWrite
+ *
+ * @note    Must not be used when I/O port breakpoints are pending or when single stepping.
+ */
+VMMRZ_INT_DECL(VBOXSTRICTRC)
+EMRZSetPendingIoPortWrite(PVMCPU pVCpu, RTIOPORT uPort, uint8_t cbInstr, uint8_t cbValue, uint32_t uValue)
+{
+    Assert(pVCpu->em.s.PendingIoPortAccess.cbValue == 0);
+    pVCpu->em.s.PendingIoPortAccess.uPort     = uPort;
+    pVCpu->em.s.PendingIoPortAccess.cbValue   = cbValue;
+    pVCpu->em.s.PendingIoPortAccess.cbInstr   = cbInstr;
+    pVCpu->em.s.PendingIoPortAccess.uValue    = uValue;
+    return VINF_EM_PENDING_R3_IOPORT_WRITE;
+}
+
+
+/**
+ * Makes an I/O port read pending for ring-3 processing.
+ *
+ * @returns VINF_EM_PENDING_R3_IOPORT_READ
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uPort           The I/O port.
+ * @param   cbInstr         The instruction length (for RIP updating).
+ * @param   cbValue         The read size.
+ * @sa      emR3ExecutePendingIoPortRead
+ *
+ * @note    Must not be used when I/O port breakpoints are pending or when single stepping.
+ */
+VMMRZ_INT_DECL(VBOXSTRICTRC)
+EMRZSetPendingIoPortRead(PVMCPU pVCpu, RTIOPORT uPort, uint8_t cbInstr, uint8_t cbValue)
+{
+    Assert(pVCpu->em.s.PendingIoPortAccess.cbValue == 0);
+    pVCpu->em.s.PendingIoPortAccess.uPort     = uPort;
+    pVCpu->em.s.PendingIoPortAccess.cbValue   = cbValue;
+    pVCpu->em.s.PendingIoPortAccess.cbInstr   = cbInstr;
+    pVCpu->em.s.PendingIoPortAccess.uValue    = UINT32_C(0x52454144); /* 'READ' */
+    return VINF_EM_PENDING_R3_IOPORT_READ;
+}
+
+#endif /* IN_RING3 */
+
+
+/**
+ * Worker for EMHistoryExec that checks for ring-3 returns and flags
+ * continuation of the EMHistoryExec run there.
+ */
+DECL_FORCE_INLINE(void) emHistoryExecSetContinueExitRecIdx(PVMCPU pVCpu, VBOXSTRICTRC rcStrict, PCEMEXITREC pExitRec)
+{
+    pVCpu->em.s.idxContinueExitRec = UINT16_MAX;
+#ifdef IN_RING3
+    RT_NOREF_PV(rcStrict); RT_NOREF_PV(pExitRec);
+#else
+    switch (VBOXSTRICTRC_VAL(rcStrict))
+    {
+        case VINF_SUCCESS:
+        default:
+            break;
+
+        /*
+         * Only status codes that EMHandleRCTmpl.h will resume EMHistoryExec with.
+         */
+        case VINF_IOM_R3_IOPORT_READ:           /* -> emR3ExecuteIOInstruction */
+        case VINF_IOM_R3_IOPORT_WRITE:          /* -> emR3ExecuteIOInstruction */
+        case VINF_IOM_R3_IOPORT_COMMIT_WRITE:   /* -> VMCPU_FF_IOM -> VINF_EM_RESUME_R3_HISTORY_EXEC -> emR3ExecuteIOInstruction */
+        case VINF_IOM_R3_MMIO_READ:             /* -> emR3ExecuteInstruction */
+        case VINF_IOM_R3_MMIO_WRITE:            /* -> emR3ExecuteInstruction */
+        case VINF_IOM_R3_MMIO_READ_WRITE:       /* -> emR3ExecuteInstruction */
+        case VINF_IOM_R3_MMIO_COMMIT_WRITE:     /* -> VMCPU_FF_IOM -> VINF_EM_RESUME_R3_HISTORY_EXEC -> emR3ExecuteIOInstruction */
+        case VINF_CPUM_R3_MSR_READ:             /* -> emR3ExecuteInstruction */
+        case VINF_CPUM_R3_MSR_WRITE:            /* -> emR3ExecuteInstruction */
+        case VINF_GIM_R3_HYPERCALL:             /* -> emR3ExecuteInstruction */
+            pVCpu->em.s.idxContinueExitRec = (uint16_t)(pExitRec - &pVCpu->em.s.aExitRecords[0]);
+            break;
+    }
+#endif /* !IN_RING3 */
+}
+
+
+/**
+ * Execute using history.
+ *
+ * This function will be called when EMHistoryAddExit() and friends returns a
+ * non-NULL result.  This happens in response to probing or when probing has
+ * uncovered adjacent exits which can more effectively be reached by using IEM
+ * than restarting execution using the main execution engine and fielding an
+ * regular exit.
+ *
+ * @returns VBox strict status code, see IEMExecForExits.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitRec        The exit record return by a previous history add
+ *                          or update call.
+ * @param   fWillExit       Flags indicating to IEM what will cause exits, TBD.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) EMHistoryExec(PVMCPUCC pVCpu, PCEMEXITREC pExitRec, uint32_t fWillExit)
+{
+    Assert(pExitRec);
+    VMCPU_ASSERT_EMT(pVCpu);
+    IEMEXECFOREXITSTATS ExecStats;
+    switch (pExitRec->enmAction)
+    {
+        /*
+         * Executes multiple instruction stopping only when we've gone a given
+         * number without perceived exits.
+         */
+        case EMEXITACTION_EXEC_WITH_MAX:
+        {
+            STAM_REL_PROFILE_START(&pVCpu->em.s.StatHistoryExec, a);
+            LogFlow(("EMHistoryExec/EXEC_WITH_MAX: %RX64, max %u\n", pExitRec->uFlatPC, pExitRec->cMaxInstructionsWithoutExit));
+            VBOXSTRICTRC rcStrict = IEMExecForExits(pVCpu, fWillExit,
+                                                    pExitRec->cMaxInstructionsWithoutExit /* cMinInstructions*/,
+                                                    pVCpu->em.s.cHistoryExecMaxInstructions,
+                                                    pExitRec->cMaxInstructionsWithoutExit,
+                                                    &ExecStats);
+            LogFlow(("EMHistoryExec/EXEC_WITH_MAX: %Rrc cExits=%u cMaxExitDistance=%u cInstructions=%u\n",
+                     VBOXSTRICTRC_VAL(rcStrict), ExecStats.cExits, ExecStats.cMaxExitDistance, ExecStats.cInstructions));
+            emHistoryExecSetContinueExitRecIdx(pVCpu, rcStrict, pExitRec);
+
+            /* Ignore instructions IEM doesn't know about. */
+            if (   (   rcStrict != VERR_IEM_INSTR_NOT_IMPLEMENTED
+                    && rcStrict != VERR_IEM_ASPECT_NOT_IMPLEMENTED)
+                || ExecStats.cInstructions == 0)
+            { /* likely */ }
+            else
+                rcStrict = VINF_SUCCESS;
+
+            if (ExecStats.cExits > 1)
+                STAM_REL_COUNTER_ADD(&pVCpu->em.s.StatHistoryExecSavedExits, ExecStats.cExits - 1);
+            STAM_REL_COUNTER_ADD(&pVCpu->em.s.StatHistoryExecInstructions, ExecStats.cInstructions);
+            STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHistoryExec, a);
+            return rcStrict;
+        }
+
+        /*
+         * Probe a exit for close by exits.
+         */
+        case EMEXITACTION_EXEC_PROBE:
+        {
+            STAM_REL_PROFILE_START(&pVCpu->em.s.StatHistoryProbe, b);
+            LogFlow(("EMHistoryExec/EXEC_PROBE: %RX64\n", pExitRec->uFlatPC));
+            PEMEXITREC   pExitRecUnconst = (PEMEXITREC)pExitRec;
+            VBOXSTRICTRC rcStrict = IEMExecForExits(pVCpu, fWillExit,
+                                                    pVCpu->em.s.cHistoryProbeMinInstructions,
+                                                    pVCpu->em.s.cHistoryExecMaxInstructions,
+                                                    pVCpu->em.s.cHistoryProbeMaxInstructionsWithoutExit,
+                                                    &ExecStats);
+            LogFlow(("EMHistoryExec/EXEC_PROBE: %Rrc cExits=%u cMaxExitDistance=%u cInstructions=%u\n",
+                     VBOXSTRICTRC_VAL(rcStrict), ExecStats.cExits, ExecStats.cMaxExitDistance, ExecStats.cInstructions));
+            emHistoryExecSetContinueExitRecIdx(pVCpu, rcStrict, pExitRecUnconst);
+            if (   ExecStats.cExits >= 2
+                && RT_SUCCESS(rcStrict))
+            {
+                Assert(ExecStats.cMaxExitDistance > 0 && ExecStats.cMaxExitDistance <= 32);
+                pExitRecUnconst->cMaxInstructionsWithoutExit = ExecStats.cMaxExitDistance;
+                pExitRecUnconst->enmAction = EMEXITACTION_EXEC_WITH_MAX;
+                LogFlow(("EMHistoryExec/EXEC_PROBE: -> EXEC_WITH_MAX %u\n", ExecStats.cMaxExitDistance));
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.StatHistoryProbedExecWithMax);
+            }
+#ifndef IN_RING3
+            else if (   pVCpu->em.s.idxContinueExitRec != UINT16_MAX
+                     && RT_SUCCESS(rcStrict))
+            {
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.StatHistoryProbedToRing3);
+                LogFlow(("EMHistoryExec/EXEC_PROBE: -> ring-3\n"));
+            }
+#endif
+            else
+            {
+                pExitRecUnconst->enmAction = EMEXITACTION_NORMAL_PROBED;
+                pVCpu->em.s.idxContinueExitRec = UINT16_MAX;
+                LogFlow(("EMHistoryExec/EXEC_PROBE: -> PROBED\n"));
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.StatHistoryProbedNormal);
+                if (   rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED
+                    || rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED)
+                    rcStrict = VINF_SUCCESS;
+            }
+            STAM_REL_COUNTER_ADD(&pVCpu->em.s.StatHistoryProbeInstructions, ExecStats.cInstructions);
+            STAM_REL_PROFILE_STOP(&pVCpu->em.s.StatHistoryProbe, b);
+            return rcStrict;
+        }
+
+        /* We shouldn't ever see these here! */
+        case EMEXITACTION_FREE_RECORD:
+        case EMEXITACTION_NORMAL:
+        case EMEXITACTION_NORMAL_PROBED:
+            break;
+
+        /* No default case, want compiler warnings. */
+    }
+    AssertLogRelFailedReturn(VERR_EM_INTERNAL_ERROR);
+}
+
+
+/**
+ * Worker for emHistoryAddOrUpdateRecord.
+ */
+DECL_FORCE_INLINE(PCEMEXITREC) emHistoryRecordInit(PEMEXITREC pExitRec, uint64_t uFlatPC, uint32_t uFlagsAndType, uint64_t uExitNo)
+{
+    pExitRec->uFlatPC                     = uFlatPC;
+    pExitRec->uFlagsAndType               = uFlagsAndType;
+    pExitRec->enmAction                   = EMEXITACTION_NORMAL;
+    pExitRec->bUnused                     = 0;
+    pExitRec->cMaxInstructionsWithoutExit = 64;
+    pExitRec->uLastExitNo                 = uExitNo;
+    pExitRec->cHits                       = 1;
+    return NULL;
+}
+
+
+/**
+ * Worker for emHistoryAddOrUpdateRecord.
+ */
+DECL_FORCE_INLINE(PCEMEXITREC) emHistoryRecordInitNew(PVMCPU pVCpu, PEMEXITENTRY pHistEntry, uintptr_t idxSlot,
+                                                      PEMEXITREC pExitRec, uint64_t uFlatPC,
+                                                      uint32_t uFlagsAndType, uint64_t uExitNo)
+{
+    pHistEntry->idxSlot = (uint32_t)idxSlot;
+    pVCpu->em.s.cExitRecordUsed++;
+    LogFlow(("emHistoryRecordInitNew: [%#x] = %#07x %016RX64; (%u of %u used)\n", idxSlot, uFlagsAndType, uFlatPC,
+             pVCpu->em.s.cExitRecordUsed, RT_ELEMENTS(pVCpu->em.s.aExitRecords) ));
+    return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+}
+
+
+/**
+ * Worker for emHistoryAddOrUpdateRecord.
+ */
+DECL_FORCE_INLINE(PCEMEXITREC) emHistoryRecordInitReplacement(PEMEXITENTRY pHistEntry, uintptr_t idxSlot,
+                                                              PEMEXITREC pExitRec, uint64_t uFlatPC,
+                                                              uint32_t uFlagsAndType, uint64_t uExitNo)
+{
+    pHistEntry->idxSlot = (uint32_t)idxSlot;
+    LogFlow(("emHistoryRecordInitReplacement: [%#x] = %#07x %016RX64 replacing %#07x %016RX64 with %u hits, %u exits old\n",
+             idxSlot, uFlagsAndType, uFlatPC, pExitRec->uFlagsAndType, pExitRec->uFlatPC, pExitRec->cHits,
+             uExitNo - pExitRec->uLastExitNo));
+    return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+}
+
+
+/**
+ * Adds or updates the EMEXITREC for this PC/type and decide on an action.
+ *
+ * @returns Pointer to an exit record if special action should be taken using
+ *          EMHistoryExec().  Take normal exit action when NULL.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uFlagsAndType   Combined flags and type, EMEXIT_F_KIND_EM set and
+ *                          both EMEXIT_F_CS_EIP and EMEXIT_F_UNFLATTENED_PC are clear.
+ * @param   uFlatPC         The flattened program counter.
+ * @param   pHistEntry      The exit history entry.
+ * @param   uExitNo         The current exit number.
+ */
+static PCEMEXITREC emHistoryAddOrUpdateRecord(PVMCPU pVCpu, uint64_t uFlagsAndType, uint64_t uFlatPC,
+                                              PEMEXITENTRY pHistEntry, uint64_t uExitNo)
+{
+# ifdef IN_RING0
+    /* Disregard the hm flag. */
+    uFlagsAndType &= ~EMEXIT_F_HM;
+# endif
+
+    /*
+     * Work the hash table.
+     */
+    AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitRecords) == 1024);
+# define EM_EXIT_RECORDS_IDX_MASK 0x3ff
+    uintptr_t  idxSlot  = ((uintptr_t)uFlatPC >> 1) & EM_EXIT_RECORDS_IDX_MASK;
+    PEMEXITREC pExitRec = &pVCpu->em.s.aExitRecords[idxSlot];
+    if (pExitRec->uFlatPC == uFlatPC)
+    {
+        Assert(pExitRec->enmAction != EMEXITACTION_FREE_RECORD);
+        pHistEntry->idxSlot = (uint32_t)idxSlot;
+        if (pExitRec->uFlagsAndType == uFlagsAndType)
+        {
+            pExitRec->uLastExitNo = uExitNo;
+            STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecHits[0]);
+        }
+        else
+        {
+            STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecTypeChanged[0]);
+            return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+        }
+    }
+    else if (pExitRec->enmAction == EMEXITACTION_FREE_RECORD)
+    {
+        STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecNew[0]);
+        return emHistoryRecordInitNew(pVCpu, pHistEntry, idxSlot, pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+    }
+    else
+    {
+        /*
+         * Collision.  We calculate a new hash for stepping away from the first,
+         * doing up to 8 steps away before replacing the least recently used record.
+         */
+        uintptr_t idxOldest     = idxSlot;
+        uint64_t  uOldestExitNo = pExitRec->uLastExitNo;
+        unsigned  iOldestStep   = 0;
+        unsigned  iStep         = 1;
+        uintptr_t const idxAdd  = (uintptr_t)(uFlatPC >> 11) & (EM_EXIT_RECORDS_IDX_MASK / 4);
+        for (;;)
+        {
+            Assert(iStep < RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits));
+            AssertCompile(RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecNew)         == RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits));
+            AssertCompile(RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecReplaced)    == RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits));
+            AssertCompile(RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecTypeChanged) == RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecHits));
+
+            /* Step to the next slot. */
+            idxSlot += idxAdd;
+            idxSlot &= EM_EXIT_RECORDS_IDX_MASK;
+            pExitRec = &pVCpu->em.s.aExitRecords[idxSlot];
+
+            /* Does it match? */
+            if (pExitRec->uFlatPC == uFlatPC)
+            {
+                Assert(pExitRec->enmAction != EMEXITACTION_FREE_RECORD);
+                pHistEntry->idxSlot = (uint32_t)idxSlot;
+                if (pExitRec->uFlagsAndType == uFlagsAndType)
+                {
+                    pExitRec->uLastExitNo = uExitNo;
+                    STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecHits[iStep]);
+                    break;
+                }
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecTypeChanged[iStep]);
+                return emHistoryRecordInit(pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+            }
+
+            /* Is it free? */
+            if (pExitRec->enmAction == EMEXITACTION_FREE_RECORD)
+            {
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecNew[iStep]);
+                return emHistoryRecordInitNew(pVCpu, pHistEntry, idxSlot, pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+            }
+
+            /* Is it the least recently used one? */
+            if (pExitRec->uLastExitNo < uOldestExitNo)
+            {
+                uOldestExitNo = pExitRec->uLastExitNo;
+                idxOldest     = idxSlot;
+                iOldestStep   = iStep;
+            }
+
+            /* Next iteration? */
+            iStep++;
+            Assert(iStep < RT_ELEMENTS(pVCpu->em.s.aStatHistoryRecReplaced));
+            if (RT_LIKELY(iStep < 8 + 1))
+            { /* likely */ }
+            else
+            {
+                /* Replace the least recently used slot. */
+                STAM_REL_COUNTER_INC(&pVCpu->em.s.aStatHistoryRecReplaced[iOldestStep]);
+                pExitRec = &pVCpu->em.s.aExitRecords[idxOldest];
+                return emHistoryRecordInitReplacement(pHistEntry, idxOldest, pExitRec, uFlatPC, uFlagsAndType, uExitNo);
+            }
+        }
+    }
+
+    /*
+     * Found an existing record.
+     */
+    switch (pExitRec->enmAction)
+    {
+        case EMEXITACTION_NORMAL:
+        {
+            uint64_t const cHits = ++pExitRec->cHits;
+            if (cHits < 256)
+                return NULL;
+            LogFlow(("emHistoryAddOrUpdateRecord: [%#x] %#07x %16RX64: -> EXEC_PROBE\n", idxSlot, uFlagsAndType, uFlatPC));
+            pExitRec->enmAction = EMEXITACTION_EXEC_PROBE;
+            return pExitRec;
+        }
+
+        case EMEXITACTION_NORMAL_PROBED:
+            pExitRec->cHits += 1;
+            return NULL;
+
+        default:
+            pExitRec->cHits += 1;
+            return pExitRec;
+
+        /* This will happen if the caller ignores or cannot serve the probe
+           request (forced to ring-3, whatever).  We retry this 256 times. */
+        case EMEXITACTION_EXEC_PROBE:
+        {
+            uint64_t const cHits = ++pExitRec->cHits;
+            if (cHits < 512)
+                return pExitRec;
+            pExitRec->enmAction = EMEXITACTION_NORMAL_PROBED;
+            LogFlow(("emHistoryAddOrUpdateRecord: [%#x] %#07x %16RX64: -> PROBED\n", idxSlot, uFlagsAndType, uFlatPC));
+            return NULL;
+        }
+    }
+}
+
+
+/**
+ * Adds an exit to the history for this CPU.
+ *
+ * @returns Pointer to an exit record if special action should be taken using
+ *          EMHistoryExec().  Take normal exit action when NULL.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uFlagsAndType   Combined flags and type (see EMEXIT_MAKE_FT).
+ * @param   uFlatPC         The flattened program counter (RIP).  UINT64_MAX if not available.
+ * @param   uTimestamp      The TSC value for the exit, 0 if not available.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(PCEMEXITREC) EMHistoryAddExit(PVMCPUCC pVCpu, uint32_t uFlagsAndType, uint64_t uFlatPC, uint64_t uTimestamp)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Add the exit history entry.
+     */
+    AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256);
+    uint64_t uExitNo = pVCpu->em.s.iNextExit++;
+    PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff];
+    pHistEntry->uFlatPC       = uFlatPC;
+    pHistEntry->uTimestamp    = uTimestamp;
+    pHistEntry->uFlagsAndType = uFlagsAndType;
+    pHistEntry->idxSlot       = UINT32_MAX;
+
+    /*
+     * If common exit type, we will insert/update the exit into the exit record hash table.
+     */
+    if (   (uFlagsAndType & (EMEXIT_F_KIND_MASK | EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)) == EMEXIT_F_KIND_EM
+#ifdef IN_RING0
+        && pVCpu->em.s.fExitOptimizationEnabledR0
+        && ( !(uFlagsAndType & EMEXIT_F_HM) || pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled)
+#else
+        && pVCpu->em.s.fExitOptimizationEnabled
+#endif
+        && uFlatPC != UINT64_MAX
+       )
+        return emHistoryAddOrUpdateRecord(pVCpu, uFlagsAndType, uFlatPC, pHistEntry, uExitNo);
+    return NULL;
+}
+
+
+/**
+ * Interface that VT-x uses to supply the PC of an exit when CS:RIP is being read.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uFlatPC         The flattened program counter (RIP).
+ * @param   fFlattened      Set if RIP was subjected to CS.BASE, clear if not.
+ */
+VMM_INT_DECL(void) EMHistoryUpdatePC(PVMCPUCC pVCpu, uint64_t uFlatPC, bool fFlattened)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256);
+    uint64_t     uExitNo    = pVCpu->em.s.iNextExit - 1;
+    PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff];
+    pHistEntry->uFlatPC = uFlatPC;
+    if (fFlattened)
+        pHistEntry->uFlagsAndType &= ~EMEXIT_F_UNFLATTENED_PC;
+    else
+        pHistEntry->uFlagsAndType |= EMEXIT_F_UNFLATTENED_PC;
+}
+
+
+/**
+ * Interface for convering a engine specific exit to a generic one and get guidance.
+ *
+ * @returns Pointer to an exit record if special action should be taken using
+ *          EMHistoryExec().  Take normal exit action when NULL.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uFlagsAndType   Combined flags and type (see EMEXIT_MAKE_FLAGS_AND_TYPE).
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(PCEMEXITREC) EMHistoryUpdateFlagsAndType(PVMCPUCC pVCpu, uint32_t uFlagsAndType)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Do the updating.
+     */
+    AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256);
+    uint64_t     uExitNo    = pVCpu->em.s.iNextExit - 1;
+    PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff];
+    pHistEntry->uFlagsAndType = uFlagsAndType | (pHistEntry->uFlagsAndType & (EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC));
+
+    /*
+     * If common exit type, we will insert/update the exit into the exit record hash table.
+     */
+    if (   (uFlagsAndType & (EMEXIT_F_KIND_MASK | EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)) == EMEXIT_F_KIND_EM
+#ifdef IN_RING0
+        && pVCpu->em.s.fExitOptimizationEnabledR0
+        && ( !(uFlagsAndType & EMEXIT_F_HM) || pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled)
+#else
+        && pVCpu->em.s.fExitOptimizationEnabled
+#endif
+        && pHistEntry->uFlatPC != UINT64_MAX
+       )
+        return emHistoryAddOrUpdateRecord(pVCpu, uFlagsAndType, pHistEntry->uFlatPC, pHistEntry, uExitNo);
+    return NULL;
+}
+
+
+/**
+ * Interface for convering a engine specific exit to a generic one and get
+ * guidance, supplying flattened PC too.
+ *
+ * @returns Pointer to an exit record if special action should be taken using
+ *          EMHistoryExec().  Take normal exit action when NULL.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uFlagsAndType   Combined flags and type (see EMEXIT_MAKE_FLAGS_AND_TYPE).
+ * @param   uFlatPC         The flattened program counter (RIP).
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(PCEMEXITREC) EMHistoryUpdateFlagsAndTypeAndPC(PVMCPUCC pVCpu, uint32_t uFlagsAndType, uint64_t uFlatPC)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    //Assert(uFlatPC != UINT64_MAX); - disable to make the pc wrapping tests in bs3-cpu-weird-1 work.
+
+    /*
+     * Do the updating.
+     */
+    AssertCompile(RT_ELEMENTS(pVCpu->em.s.aExitHistory) == 256);
+    uint64_t     uExitNo    = pVCpu->em.s.iNextExit - 1;
+    PEMEXITENTRY pHistEntry = &pVCpu->em.s.aExitHistory[(uintptr_t)uExitNo & 0xff];
+    pHistEntry->uFlagsAndType = uFlagsAndType;
+    pHistEntry->uFlatPC       = uFlatPC;
+
+    /*
+     * If common exit type, we will insert/update the exit into the exit record hash table.
+     */
+    if (   (uFlagsAndType & (EMEXIT_F_KIND_MASK | EMEXIT_F_CS_EIP | EMEXIT_F_UNFLATTENED_PC)) == EMEXIT_F_KIND_EM
+#ifdef IN_RING0
+        && pVCpu->em.s.fExitOptimizationEnabledR0
+        && ( !(uFlagsAndType & EMEXIT_F_HM) || pVCpu->em.s.fExitOptimizationEnabledR0PreemptDisabled)
+#else
+        && pVCpu->em.s.fExitOptimizationEnabled
+#endif
+       )
+        return emHistoryAddOrUpdateRecord(pVCpu, uFlagsAndType, uFlatPC, pHistEntry, uExitNo);
+    return NULL;
+}
+
+
+/**
+ * @callback_method_impl{FNDISREADBYTES}
+ */
+static DECLCALLBACK(int) emReadBytes(PDISCPUSTATE pDis, uint8_t offInstr, uint8_t cbMinRead, uint8_t cbMaxRead)
+{
+    PVMCPUCC    pVCpu    = (PVMCPUCC)pDis->pvUser;
+    RTUINTPTR   uSrcAddr = pDis->uInstrAddr + offInstr;
+
+    /*
+     * Figure how much we can or must read.
+     */
+    size_t      cbToRead = GUEST_PAGE_SIZE - (uSrcAddr & (GUEST_PAGE_SIZE - 1));
+    if (cbToRead > cbMaxRead)
+        cbToRead = cbMaxRead;
+    else if (cbToRead < cbMinRead)
+        cbToRead = cbMinRead;
+
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, &pDis->abInstr[offInstr], uSrcAddr, cbToRead);
+    if (RT_FAILURE(rc))
+    {
+        if (cbToRead > cbMinRead)
+        {
+            cbToRead = cbMinRead;
+            rc = PGMPhysSimpleReadGCPtr(pVCpu, &pDis->abInstr[offInstr], uSrcAddr, cbToRead);
+        }
+        if (RT_FAILURE(rc))
+        {
+            /*
+             * If we fail to find the page via the guest's page tables
+             * we invalidate the page in the host TLB (pertaining to
+             * the guest in the NestedPaging case). See @bugref{6043}.
+             */
+            if (rc == VERR_PAGE_TABLE_NOT_PRESENT || rc == VERR_PAGE_NOT_PRESENT)
+            {
+                HMInvalidatePage(pVCpu, uSrcAddr);
+                if (((uSrcAddr + cbToRead - 1) >> GUEST_PAGE_SHIFT) != (uSrcAddr >> GUEST_PAGE_SHIFT))
+                    HMInvalidatePage(pVCpu, uSrcAddr + cbToRead - 1);
+            }
+        }
+    }
+
+    pDis->cbCachedInstr = offInstr + (uint8_t)cbToRead;
+    return rc;
+}
+
+
+/**
+ * Disassembles the current instruction.
+ *
+ * @returns VBox status code, see SELMToFlatEx and EMInterpretDisasOneEx for
+ *          details.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pDis            Where to return the parsed instruction info.
+ * @param   pcbInstr        Where to return the instruction size. (optional)
+ */
+VMM_INT_DECL(int) EMInterpretDisasCurrent(PVMCPUCC pVCpu, PDISCPUSTATE pDis, unsigned *pcbInstr)
+{
+    PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
+    RTGCPTR  GCPtrInstr;
+#if 0
+    int rc = SELMToFlatEx(pVCpu, DISSELREG_CS, pCtx, pCtx->rip, 0, &GCPtrInstr);
+#else
+/** @todo Get the CPU mode as well while we're at it! */
+    int rc = SELMValidateAndConvertCSAddr(pVCpu, pCtx->eflags.u, pCtx->ss.Sel, pCtx->cs.Sel, &pCtx->cs, pCtx->rip, &GCPtrInstr);
+#endif
+    if (RT_SUCCESS(rc))
+        return EMInterpretDisasOneEx(pVCpu, (RTGCUINTPTR)GCPtrInstr, pDis, pcbInstr);
+
+    Log(("EMInterpretDisasOne: Failed to convert %RTsel:%RGv (cpl=%d) - rc=%Rrc !!\n",
+         pCtx->cs.Sel, (RTGCPTR)pCtx->rip, pCtx->ss.Sel & X86_SEL_RPL, rc));
+    return rc;
+}
+
+
+/**
+ * Disassembles one instruction.
+ *
+ * This is used by internally by the interpreter and by trap/access handlers.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPtrInstr      The flat address of the instruction.
+ * @param   pDis            Where to return the parsed instruction info.
+ * @param   pcbInstr        Where to return the instruction size. (optional)
+ */
+VMM_INT_DECL(int) EMInterpretDisasOneEx(PVMCPUCC pVCpu, RTGCUINTPTR GCPtrInstr, PDISCPUSTATE pDis, unsigned *pcbInstr)
+{
+    DISCPUMODE enmCpuMode = CPUMGetGuestDisMode(pVCpu);
+    /** @todo Deal with too long instruction (=> \#GP), opcode read errors (=>
+     *        \#PF, \#GP, \#??), undefined opcodes (=> \#UD), and such. */
+    int rc = DISInstrWithReader(GCPtrInstr, enmCpuMode, emReadBytes, pVCpu, pDis, pcbInstr);
+    if (RT_SUCCESS(rc))
+        return VINF_SUCCESS;
+    AssertMsg(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT, ("DISCoreOne failed to GCPtrInstr=%RGv rc=%Rrc\n", GCPtrInstr, rc));
+    return rc;
+}
+
+
+/**
+ * Interprets the current instruction.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_*                  Scheduling instructions.
+ * @retval  VERR_EM_INTERPRETER     Something we can't cope with.
+ * @retval  VERR_*                  Fatal errors.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ *
+ * @remark  Invalid opcode exceptions have a higher priority than \#GP (see
+ *          Intel Architecture System Developers Manual, Vol 3, 5.5) so we don't
+ *          need to worry about e.g. invalid modrm combinations (!)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) EMInterpretInstruction(PVMCPUCC pVCpu)
+{
+    LogFlow(("EMInterpretInstruction %RGv\n", (RTGCPTR)CPUMGetGuestRIP(pVCpu)));
+
+    VBOXSTRICTRC rc = IEMExecOneBypassEx(pVCpu, NULL /*pcbWritten*/);
+    if (RT_UNLIKELY(   rc == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                    || rc == VERR_IEM_INSTR_NOT_IMPLEMENTED))
+        rc = VERR_EM_INTERPRETER;
+    if (rc != VINF_SUCCESS)
+        Log(("EMInterpretInstruction: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
+
+    return rc;
+}
+
+
+/**
+ * Interprets the current instruction using the supplied DISCPUSTATE structure.
+ *
+ * IP/EIP/RIP *IS* updated!
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_*                  Scheduling instructions. When these are returned, it
+ *                                  starts to get a bit tricky to know whether code was
+ *                                  executed or not... We'll address this when it becomes a problem.
+ * @retval  VERR_EM_INTERPRETER     Something we can't cope with.
+ * @retval  VERR_*                  Fatal errors.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pDis        The disassembler cpu state for the instruction to be
+ *                      interpreted.
+ * @param   rip         The instruction pointer value.
+ *
+ * @remark  Invalid opcode exceptions have a higher priority than GP (see Intel
+ *          Architecture System Developers Manual, Vol 3, 5.5) so we don't need
+ *          to worry about e.g. invalid modrm combinations (!)
+ *
+ * @todo    At this time we do NOT check if the instruction overwrites vital information.
+ *          Make sure this can't happen!! (will add some assertions/checks later)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) EMInterpretInstructionDisasState(PVMCPUCC pVCpu, PDISCPUSTATE pDis, uint64_t rip)
+{
+    LogFlow(("EMInterpretInstructionDisasState %RGv\n", (RTGCPTR)rip));
+
+    VBOXSTRICTRC rc = IEMExecOneBypassWithPrefetchedByPC(pVCpu, rip, pDis->abInstr, pDis->cbCachedInstr);
+    if (RT_UNLIKELY(   rc == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                    || rc == VERR_IEM_INSTR_NOT_IMPLEMENTED))
+        rc = VERR_EM_INTERPRETER;
+
+    if (rc != VINF_SUCCESS)
+        Log(("EMInterpretInstructionDisasState: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
+
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMAll/GCMAll.cpp b/src/VBox/VMM/VMMAll/GCMAll.cpp
new file mode 100644
index 00000000..6773e8c3
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/GCMAll.cpp
@@ -0,0 +1,245 @@
+/** @file
+ * GCM - Guest Compatibility Manager - All Contexts.
+ */
+
+/*
+ * Copyright (C) 2022-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gcm.h>
+#include <VBox/vmm/em.h>    /* For EMInterpretDisasCurrent */
+#include "GCMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/dis.h>       /* For DISCPUSTATE */
+#include <iprt/errcore.h>
+#include <iprt/string.h>
+
+
+/**
+ * Checks whether GCM is enabled for this VM.
+ *
+ * @retval  true if GCM is on.
+ * @retval  false if no GCM fixer is enabled.
+ *
+ * @param   pVM       The cross context VM structure.
+ */
+VMMDECL(bool) GCMIsEnabled(PVM pVM)
+{
+    return pVM->gcm.s.enmFixerIds != GCMFIXER_NONE;
+}
+
+
+/**
+ * Gets the GCM fixers configured for this VM.
+ *
+ * @returns The GCM provider Id.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(int32_t) GCMGetFixers(PVM pVM)
+{
+    return pVM->gcm.s.enmFixerIds;
+}
+
+
+/**
+ * Whether \#DE exceptions in the guest should be intercepted by GCM and
+ * possibly fixed up.
+ *
+ * @returns true if needed, false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) GCMShouldTrapXcptDE(PVMCPUCC pVCpu)
+{
+    LogFlowFunc(("entered\n"));
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    if (!GCMIsEnabled(pVM))
+        return false;
+
+    LogFunc(("GCM checking if #DE needs trapping\n"));
+
+    /* See if the enabled fixers need to intercept #DE. */
+    if (  pVM->gcm.s.enmFixerIds
+        & (GCMFIXER_DBZ_DOS |  GCMFIXER_DBZ_OS2 | GCMFIXER_DBZ_WIN9X))
+    {
+        LogRel(("GCM: #DE should be trapped\n"));
+        return true;
+    }
+
+    return false;
+}
+
+
+/**
+ * Exception handler for \#DE when registered by GCM.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS retry division and continue.
+ * @retval  VERR_NOT_FOUND deliver exception to guest.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   pDis        Pointer to the disassembled instruction state at RIP.
+ *                      If NULL is passed, it implies the disassembly of the
+ *                      the instruction at RIP is the
+ *                      responsibility of GCM.
+ * @param   pcbInstr    Where to store the instruction length of
+ *                      the divide instruction. Optional, can be
+ *                      NULL.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GCMXcptDE(PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    Assert(GCMIsEnabled(pVM));
+    Assert(pDis || pcbInstr);
+    RT_NOREF(pDis);
+    RT_NOREF(pcbInstr);
+
+    LogRel(("GCM: Intercepted #DE at CS:RIP=%04x:%RX64 (%RX64 linear) RDX:RAX=%RX64:%RX64 RCX=%RX64 RBX=%RX64\n",
+            pCtx->cs.Sel, pCtx->rip, pCtx->cs.u64Base + pCtx->rip, pCtx->rdx, pCtx->rax, pCtx->rcx, pCtx->rbx));
+
+    if (pVM->gcm.s.enmFixerIds & GCMFIXER_DBZ_OS2)
+    {
+        if (pCtx->rcx == 0 && pCtx->rdx == 1 && pCtx->rax == 0x86a0)
+        {
+            /* OS/2 1.x drivers loaded during boot: DX:AX = 100,000, CX < 2 causes overflow. */
+            /* Example: OS/2 1.0 KBD01.SYS, 16,945 bytes, dated 10/21/1987, div cx at offset 2:2ffeh */
+            /* Code later merged into BASEDD01.SYS, crash fixed in OS/2 1.30.1; this should
+             * fix all affected versions of OS/2 1.x.
+             */
+            pCtx->rcx = 2;
+            return VINF_SUCCESS;
+        }
+        if ((uint16_t)pCtx->rbx == 0 && (uint16_t)pCtx->rdx == 0 && (uint16_t)pCtx->rax == 0x1000)
+        {
+            /* OS/2 2.1 and later boot loader: DX:AX = 0x1000, zero BX. May have junk in high words of all registers. */
+            /* Example: OS/2 MCP2 OS2LDR, 44,544 bytes, dated 03/08/2002, idiv bx at offset 847ah */
+            pCtx->rbx = (pCtx->rbx & ~0xffff) | 2;
+            return VINF_SUCCESS;
+        }
+        if (pCtx->rbx == 0 && pCtx->rdx == 0 && pCtx->rax == 0x100)
+        {
+            /* OS/2 2.0 boot loader: DX:AX = 0x100, zero BX. May have junk in high words of registers. */
+            /* Example: OS/2 2.0 OS2LDR, 32,256 bytes, dated 03/30/1992, idiv bx at offset 2298h */
+            pCtx->rbx = 2;
+            return VINF_SUCCESS;
+        }
+    }
+
+    if (pVM->gcm.s.enmFixerIds & GCMFIXER_DBZ_DOS)
+    {
+        /* NB: For 16-bit DOS software, we must generally only compare 16-bit registers.
+         * The contents of the high words may be unpredictable depending on the environment.
+         * For 32-bit Windows 3.x code that is not the case.
+         */
+        if (pCtx->rcx == 0 && pCtx->rdx == 0 && pCtx->rax == 0x100000)
+        {
+            /* NDIS.386 in WfW 3.11: CalibrateStall, EDX:EAX = 0x100000, zero ECX.
+             * Occurs when NDIS.386 loads.
+             */
+            pCtx->rcx = 0x20000;    /* Want a large divisor to shorten stalls. */
+            return VINF_SUCCESS;
+        }
+        if (pCtx->rcx == 0 && pCtx->rdx == 0 && pCtx->rax > 0x100000)
+        {
+            /* NDIS.386 in WfW 3.11: NdisStallExecution, EDX:EAX = 0xYY00000, zero ECX.
+             * EDX:EAX is variable, but low 20 bits of EAX must be zero and EDX is likely
+             * to be zero as well.
+             * Only occurs if NdisStallExecution is called to do a longish stall.
+             */
+            pCtx->rcx = 22;
+            return VINF_SUCCESS;
+        }
+        if ((uint16_t)pCtx->rbx == 0 && (uint16_t)pCtx->rdx == 0 && (uint16_t)pCtx->rax == 0x64)
+        {
+            /* Norton Sysinfo or Diagnostics 8.0 DX:AX = 0x64 (100 decimal), zero BX. */
+            pCtx->rbx = (pCtx->rbx & 0xffff0000) | 1;   /* BX = 1 */
+            return VINF_SUCCESS;
+        }
+        if ((uint16_t)pCtx->rbx == 0 && (uint16_t)pCtx->rdx == 0 && (uint16_t)pCtx->rax == 0xff)
+        {
+            /* IBM PC LAN Program 1.3: DX:AX=0xff (255 decimal), zero BX. */
+            /* NETWORK1.CMD, 64,324 bytes, dated 06/06/1988, div bx at offset 0xa400 in file. */
+            pCtx->rbx = (pCtx->rbx & 0xffff0000) | 1;   /* BX = 1 */
+            return VINF_SUCCESS;
+        }
+        if ((uint16_t)pCtx->rdx == 0xffff && (uint16_t)pCtx->rax == 0xffff && (uint16_t)pCtx->rcx == 0xa8c0)
+        {
+            /* QNX 2.15C: DX:AX=0xffffffff (-1), constant CX = 0xa8c0 (43200). */
+            /* div cx at e.g. 2220:fa5 and 2220:10a0 in memory. */
+            pCtx->rdx = (pCtx->rdx & 0xffff0000) | 8;   /* DX = 8 */
+            return VINF_SUCCESS;
+        }
+        if ((uint16_t)pCtx->rax > 0x1800 && ((uint16_t)pCtx->rax & 0x3f) == 0 && (uint16_t)pCtx->rbx == 0x19)
+        {
+            /* 3C501.COM ODI driver v1.21: AX > ~0x1900 (-1), BX = 0x19 (25). */
+            /* AX was shifted left by 6 bits so low bits must be zero. */
+            /* div bl at e.g. 06b3:2f80 and offset 0x2E80 in file. */
+            pCtx->rax = (pCtx->rax & 0xffff0000) | 0x8c0;   /* AX = 0x8c0 */
+            return VINF_SUCCESS;
+        }
+        if ((uint16_t)pCtx->rcx == 0x37 && ((uint16_t)pCtx->rdx > 0x34))
+        {
+            /* Turbo Pascal, classic Runtime Error 200: CX = 55, DX > ~54, AX/BX variable. */
+            /* div cx at variable offset in file. */
+            pCtx->rdx = (pCtx->rdx & 0xffff0000) | 0x30;    /* DX = 48 */
+            return VINF_SUCCESS;
+        }
+    }
+
+    if (pVM->gcm.s.enmFixerIds & GCMFIXER_DBZ_WIN9X)
+    {
+        if (pCtx->rcx == 0 && pCtx->rdx == 0 && pCtx->rax == 0x100000)
+        {
+            /* NDIS.VXD in Win9x: EDX:EAX = 0x100000, zero ECX. */
+            /* Example: Windows 95 NDIS.VXD, 99,084 bytes, dated 07/11/1994, div ecx at 28:Cxxxx80B */
+            /* Crash fixed in Windows 98 SE. */
+            pCtx->rcx = 0x20000;    /* Want a large divisor to shorten stalls. */
+            return VINF_SUCCESS;
+        }
+        if (pCtx->rcx < 3 && pCtx->rdx == 2 && pCtx->rax == 0x540be400)
+        {
+            /* SCSI.PDR, ESDI506.PDR in Win95: EDX:EAX = 0x2540be400 (10,000,000,000 decimal), ECX < 3. */
+            /* Example: Windows 95 SCSIPORT.PDR, 23,133 bytes, dated 07/11/1995, div ecx at 28:Cxxxx876  */
+            /* Example: Win95 OSR2  ESDI506.PDR, 24,390 bytes, dated 04/24/1996, div ecx at 28:Cxxxx8E3 */
+            /* Crash fixed in Windows 98. */
+            pCtx->rcx = 1000;
+            return VINF_SUCCESS;
+        }
+        if (pCtx->rcx == 0 && pCtx->rdx == 0x3d && pCtx->rax == 0x9000000)
+        {
+            /* Unknown source, Win9x shutdown, div ecx. */
+            /* GCM: Intercepted #DE at CS:RIP=0028:c0050f8e RDX:RAX=3d:9000000 (250000*1024*1024) RCX=0 RBX=c19200e8 [RBX variable] */
+            pCtx->rcx = 4096;
+            return VINF_SUCCESS;
+        }
+    }
+
+    /* If we got this far, deliver exception to guest. */
+    return VERR_NOT_FOUND;
+}
diff --git a/src/VBox/VMM/VMMAll/GIMAll.cpp b/src/VBox/VMM/VMMAll/GIMAll.cpp
new file mode 100644
index 00000000..470e653e
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/GIMAll.cpp
@@ -0,0 +1,506 @@
+/* $Id: GIMAll.cpp $ */
+/** @file
+ * GIM - Guest Interface Manager - All Contexts.
+ */
+
+/*
+ * Copyright (C) 2014-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/em.h>    /* For EMInterpretDisasCurrent */
+#include "GIMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/dis.h>       /* For DISCPUSTATE */
+#include <VBox/err.h>
+#include <iprt/string.h>
+
+/* Include all the providers. */
+#include "GIMHvInternal.h"
+#include "GIMMinimalInternal.h"
+
+
+/**
+ * Checks whether GIM is being used by this VM.
+ *
+ * @retval  true if used.
+ * @retval  false if no GIM provider ("none") is used.
+ *
+ * @param   pVM       The cross context VM structure.
+ */
+VMMDECL(bool) GIMIsEnabled(PVM pVM)
+{
+    return pVM->gim.s.enmProviderId != GIMPROVIDERID_NONE;
+}
+
+
+/**
+ * Gets the GIM provider configured for this VM.
+ *
+ * @returns The GIM provider Id.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(GIMPROVIDERID) GIMGetProvider(PVM pVM)
+{
+    return pVM->gim.s.enmProviderId;
+}
+
+
+/**
+ * Returns the array of MMIO2 regions that are expected to be registered and
+ * later mapped into the guest-physical address space for the GIM provider
+ * configured for the VM.
+ *
+ * @returns Pointer to an array of GIM MMIO2 regions, may return NULL.
+ * @param   pVM         The cross context VM structure.
+ * @param   pcRegions   Where to store the number of items in the array.
+ *
+ * @remarks The caller does not own and therefore must -NOT- try to free the
+ *          returned pointer.
+ */
+VMMDECL(PGIMMMIO2REGION) GIMGetMmio2Regions(PVMCC pVM, uint32_t *pcRegions)
+{
+    Assert(pVM);
+    Assert(pcRegions);
+
+    *pcRegions = 0;
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvGetMmio2Regions(pVM, pcRegions);
+
+        default:
+            break;
+    }
+
+    return NULL;
+}
+
+
+/**
+ * Returns whether the guest has configured and enabled calls to the hypervisor.
+ *
+ * @returns true if hypercalls are enabled and usable, false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) GIMAreHypercallsEnabled(PVMCPUCC pVCpu)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    if (!GIMIsEnabled(pVM))
+        return false;
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvAreHypercallsEnabled(pVM);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmAreHypercallsEnabled(pVCpu);
+
+        default:
+            return false;
+    }
+}
+
+
+/**
+ * Implements a GIM hypercall with the provider configured for the VM.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating
+ *          RIP.
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_HYPERCALLS_NOT_AVAILABLE hypercalls unavailable.
+ * @retval  VERR_GIM_NOT_ENABLED GIM is not enabled (shouldn't really happen)
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_READ_FAILED hypercall failed while reading
+ *          memory.
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED hypercall failed while
+ *          writing memory.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ *
+ * @remarks The caller of this function needs to advance RIP as required.
+ * @thread  EMT.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    if (RT_UNLIKELY(!GIMIsEnabled(pVM)))
+        return VERR_GIM_NOT_ENABLED;
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvHypercall(pVCpu, pCtx);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmHypercall(pVCpu, pCtx);
+
+        default:
+            AssertMsgFailed(("GIMHypercall: for provider %u not available/implemented\n", pVM->gim.s.enmProviderId));
+            return VERR_GIM_HYPERCALLS_NOT_AVAILABLE;
+    }
+}
+
+
+/**
+ * Same as GIMHypercall, except with disassembler opcode and instruction length.
+ *
+ * This is the interface used by IEM.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating
+ *          RIP.
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_HYPERCALLS_NOT_AVAILABLE hypercalls unavailable.
+ * @retval  VERR_GIM_NOT_ENABLED GIM is not enabled (shouldn't really happen)
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_READ_FAILED hypercall failed while reading
+ *          memory.
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED hypercall failed while
+ *          writing memory.
+ * @retval  VERR_GIM_INVALID_HYPERCALL_INSTR if uDisOpcode is the wrong one; raise \#UD.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   uDisOpcode  The disassembler opcode.
+ * @param   cbInstr     The instruction length.
+ *
+ * @remarks The caller of this function needs to advance RIP as required.
+ * @thread  EMT.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    if (RT_UNLIKELY(!GIMIsEnabled(pVM)))
+        return VERR_GIM_NOT_ENABLED;
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvHypercallEx(pVCpu, pCtx, uDisOpcode, cbInstr);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmHypercallEx(pVCpu, pCtx, uDisOpcode, cbInstr);
+
+        default:
+            AssertMsgFailedReturn(("enmProviderId=%u\n", pVM->gim.s.enmProviderId), VERR_GIM_HYPERCALLS_NOT_AVAILABLE);
+    }
+}
+
+
+/**
+ * Disassembles the instruction at RIP and if it's a hypercall
+ * instruction, performs the hypercall.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   pcbInstr    Where to store the disassembled instruction length.
+ *                      Optional, can be NULL.
+ *
+ * @todo    This interface should disappear when IEM/REM execution engines
+ *          handle VMCALL/VMMCALL instructions to call into GIM when
+ *          required. See @bugref{7270#c168}.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMExecHypercallInstr(PVMCPUCC pVCpu, PCPUMCTX pCtx, uint8_t *pcbInstr)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    if (RT_UNLIKELY(!GIMIsEnabled(pVM)))
+        return VERR_GIM_NOT_ENABLED;
+
+    unsigned    cbInstr;
+    DISCPUSTATE Dis;
+    int rc = EMInterpretDisasCurrent(pVCpu, &Dis, &cbInstr);
+    if (RT_SUCCESS(rc))
+    {
+        if (pcbInstr)
+            *pcbInstr = (uint8_t)cbInstr;
+        switch (pVM->gim.s.enmProviderId)
+        {
+            case GIMPROVIDERID_HYPERV:
+                return gimHvHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr);
+
+            case GIMPROVIDERID_KVM:
+                return gimKvmHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr);
+
+            default:
+                AssertMsgFailed(("GIMExecHypercallInstr: for provider %u not available/implemented\n", pVM->gim.s.enmProviderId));
+                return VERR_GIM_HYPERCALLS_NOT_AVAILABLE;
+        }
+    }
+
+    Log(("GIM: GIMExecHypercallInstr: Failed to disassemble CS:RIP=%04x:%08RX64. rc=%Rrc\n", pCtx->cs.Sel, pCtx->rip, rc));
+    return rc;
+}
+
+
+/**
+ * Returns whether the guest has configured and setup the use of paravirtualized
+ * TSC.
+ *
+ * Paravirtualized TSCs are per-VM and the rest of the execution engine logic
+ * relies on that.
+ *
+ * @returns true if enabled and usable, false otherwise.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(bool) GIMIsParavirtTscEnabled(PVMCC pVM)
+{
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvIsParavirtTscEnabled(pVM);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmIsParavirtTscEnabled(pVM);
+
+        default:
+            break;
+    }
+    return false;
+}
+
+
+/**
+ * Whether \#UD exceptions in the guest needs to be intercepted by the GIM
+ * provider.
+ *
+ * At the moment, the reason why this isn't a more generic interface wrt to
+ * exceptions is because of performance (each VM-exit would have to manually
+ * check whether or not GIM needs to be notified). Left as a todo for later if
+ * really required.
+ *
+ * @returns true if needed, false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) GIMShouldTrapXcptUD(PVMCPUCC pVCpu)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    if (!GIMIsEnabled(pVM))
+        return false;
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_KVM:
+            return gimKvmShouldTrapXcptUD(pVM);
+
+        case GIMPROVIDERID_HYPERV:
+            return gimHvShouldTrapXcptUD(pVCpu);
+
+        default:
+            return false;
+    }
+}
+
+
+/**
+ * Exception handler for \#UD when requested by the GIM provider.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_R3_HYPERCALL restart the hypercall from ring-3.
+ * @retval  VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating
+ *          RIP.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_INVALID_HYPERCALL_INSTR instruction at RIP is not a valid
+ *          hypercall instruction.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   pDis        Pointer to the disassembled instruction state at RIP.
+ *                      If NULL is passed, it implies the disassembly of the
+ *                      the instruction at RIP is the responsibility of the
+ *                      GIM provider.
+ * @param   pcbInstr    Where to store the instruction length of the hypercall
+ *                      instruction. Optional, can be NULL.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMXcptUD(PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    Assert(GIMIsEnabled(pVM));
+    Assert(pDis || pcbInstr);
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_KVM:
+            return gimKvmXcptUD(pVM, pVCpu, pCtx, pDis, pcbInstr);
+
+        case GIMPROVIDERID_HYPERV:
+            return gimHvXcptUD(pVCpu, pCtx, pDis, pcbInstr);
+
+        default:
+            return VERR_GIM_OPERATION_FAILED;
+    }
+}
+
+
+/**
+ * Invokes the read-MSR handler for the GIM provider configured for the VM.
+ *
+ * @returns Strict VBox status code like CPUMQueryGuestMsr.
+ * @retval  VINF_CPUM_R3_MSR_READ
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR to read.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   puValue     Where to store the MSR value read.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    Assert(pVCpu);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    Assert(GIMIsEnabled(pVM));
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvReadMsr(pVCpu, idMsr, pRange, puValue);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmReadMsr(pVCpu, idMsr, pRange, puValue);
+
+        default:
+            AssertMsgFailed(("GIMReadMsr: for unknown provider %u idMsr=%#RX32 -> #GP(0)", pVM->gim.s.enmProviderId, idMsr));
+            return VERR_CPUM_RAISE_GP_0;
+    }
+}
+
+
+/**
+ * Invokes the write-MSR handler for the GIM provider configured for the VM.
+ *
+ * @returns Strict VBox status code like CPUMSetGuestMsr.
+ * @retval  VINF_CPUM_R3_MSR_WRITE
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR to write.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   uValue      The value to set, ignored bits masked.
+ * @param   uRawValue   The raw value with the ignored bits not masked.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) GIMWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uValue, uint64_t uRawValue)
+{
+    AssertPtr(pVCpu);
+    NOREF(uValue);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    Assert(GIMIsEnabled(pVM));
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    switch (pVM->gim.s.enmProviderId)
+    {
+        case GIMPROVIDERID_HYPERV:
+            return gimHvWriteMsr(pVCpu, idMsr, pRange, uRawValue);
+
+        case GIMPROVIDERID_KVM:
+            return gimKvmWriteMsr(pVCpu, idMsr, pRange, uRawValue);
+
+        default:
+            AssertMsgFailed(("GIMWriteMsr: for unknown provider %u idMsr=%#RX32 -> #GP(0)", pVM->gim.s.enmProviderId, idMsr));
+            return VERR_CPUM_RAISE_GP_0;
+    }
+}
+
+
+/**
+ * Queries the opcode bytes for a native hypercall.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pvBuf           The destination buffer.
+ * @param   cbBuf           The size of the buffer.
+ * @param   pcbWritten      Where to return the number of bytes written.  This is
+ *                          reliably updated only on successful return.  Optional.
+ * @param   puDisOpcode     Where to return the disassembler opcode.  Optional.
+ */
+VMM_INT_DECL(int) GIMQueryHypercallOpcodeBytes(PVM pVM, void *pvBuf, size_t cbBuf, size_t *pcbWritten, uint16_t *puDisOpcode)
+{
+    AssertPtrReturn(pvBuf, VERR_INVALID_POINTER);
+
+#if defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)
+    CPUMCPUVENDOR  enmCpuVendor = CPUMGetHostCpuVendor(pVM);
+#else
+    CPUMCPUVENDOR  enmCpuVendor = CPUMGetGuestCpuVendor(pVM); /* Use what is presented to the guest. */
+#endif
+    uint8_t const *pbSrc;
+    size_t         cbSrc;
+    switch (enmCpuVendor)
+    {
+        case CPUMCPUVENDOR_AMD:
+        case CPUMCPUVENDOR_HYGON:
+        {
+            if (puDisOpcode)
+                *puDisOpcode = OP_VMMCALL;
+            static uint8_t const s_abHypercall[] = { 0x0F, 0x01, 0xD9 };   /* VMMCALL */
+            pbSrc = s_abHypercall;
+            cbSrc = sizeof(s_abHypercall);
+            break;
+        }
+
+        case CPUMCPUVENDOR_INTEL:
+        case CPUMCPUVENDOR_VIA:
+        case CPUMCPUVENDOR_SHANGHAI:
+        {
+            if (puDisOpcode)
+                *puDisOpcode = OP_VMCALL;
+            static uint8_t const s_abHypercall[] = { 0x0F, 0x01, 0xC1 };   /* VMCALL */
+            pbSrc = s_abHypercall;
+            cbSrc = sizeof(s_abHypercall);
+            break;
+        }
+
+        default:
+            AssertMsgFailedReturn(("%d\n", enmCpuVendor), VERR_UNSUPPORTED_CPU);
+    }
+    if (RT_LIKELY(cbBuf >= cbSrc))
+    {
+        memcpy(pvBuf, pbSrc, cbSrc);
+        if (pcbWritten)
+            *pcbWritten = cbSrc;
+        return VINF_SUCCESS;
+    }
+    return VERR_BUFFER_OVERFLOW;
+}
+
diff --git a/src/VBox/VMM/VMMAll/GIMAllHv.cpp b/src/VBox/VMM/VMMAll/GIMAllHv.cpp
new file mode 100644
index 00000000..2f71989a
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/GIMAllHv.cpp
@@ -0,0 +1,1495 @@
+/* $Id: GIMAllHv.cpp $ */
+/** @file
+ * GIM - Guest Interface Manager, Microsoft Hyper-V, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2014-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/pdmapi.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/em.h>
+#include "GIMHvInternal.h"
+#include "GIMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+
+#ifdef IN_RING3
+# include <iprt/mem.h>
+#endif
+
+
+#ifdef IN_RING3
+/**
+ * Read and validate slow hypercall parameters.
+ *
+ * @returns VBox status code.
+ * @param   pVM               The cross context VM structure.
+ * @param   pCtx              Pointer to the guest-CPU context.
+ * @param   fIs64BitMode      Whether the guest is currently in 64-bit mode or not.
+ * @param   enmParam          The hypercall parameter type.
+ * @param   prcHv             Where to store the Hyper-V status code. Only valid
+ *                            to the caller when this function returns
+ *                            VINF_SUCCESS.
+ */
+static int gimHvReadSlowHypercallParam(PVM pVM, PCPUMCTX pCtx, bool fIs64BitMode, GIMHVHYPERCALLPARAM enmParam, int *prcHv)
+{
+    int       rc = VINF_SUCCESS;
+    PGIMHV    pHv = &pVM->gim.s.u.Hv;
+    RTGCPHYS  GCPhysParam;
+    void     *pvDst;
+    if (enmParam == GIMHVHYPERCALLPARAM_IN)
+    {
+        GCPhysParam = fIs64BitMode ? pCtx->rdx : (pCtx->rbx << 32) | pCtx->ecx;
+        pvDst = pHv->pbHypercallIn;
+        pHv->GCPhysHypercallIn = GCPhysParam;
+    }
+    else
+    {
+        GCPhysParam = fIs64BitMode ? pCtx->r8 : (pCtx->rdi << 32) | pCtx->esi;
+        pvDst = pHv->pbHypercallOut;
+        pHv->GCPhysHypercallOut = GCPhysParam;
+        Assert(enmParam == GIMHVHYPERCALLPARAM_OUT);
+    }
+
+    const char *pcszParam = enmParam == GIMHVHYPERCALLPARAM_IN ? "input" : "output";  NOREF(pcszParam);
+    if (RT_ALIGN_64(GCPhysParam, 8) == GCPhysParam)
+    {
+        if (PGMPhysIsGCPhysNormal(pVM, GCPhysParam))
+        {
+            rc = PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysParam, GIM_HV_PAGE_SIZE);
+            if (RT_SUCCESS(rc))
+            {
+                *prcHv = GIM_HV_STATUS_SUCCESS;
+                return VINF_SUCCESS;
+            }
+            LogRel(("GIM: HyperV: Failed reading %s param at %#RGp. rc=%Rrc\n", pcszParam, GCPhysParam, rc));
+            rc = VERR_GIM_HYPERCALL_MEMORY_READ_FAILED;
+        }
+        else
+        {
+            Log(("GIM: HyperV: Invalid %s param address %#RGp\n", pcszParam, GCPhysParam));
+            *prcHv = GIM_HV_STATUS_INVALID_PARAMETER;
+        }
+    }
+    else
+    {
+        Log(("GIM: HyperV: Misaligned %s param address %#RGp\n", pcszParam, GCPhysParam));
+        *prcHv = GIM_HV_STATUS_INVALID_ALIGNMENT;
+    }
+    return rc;
+}
+
+
+/**
+ * Helper for reading and validating slow hypercall input and output parameters.
+ *
+ * @returns VBox status code.
+ * @param   pVM               The cross context VM structure.
+ * @param   pCtx              Pointer to the guest-CPU context.
+ * @param   fIs64BitMode      Whether the guest is currently in 64-bit mode or not.
+ * @param   prcHv             Where to store the Hyper-V status code. Only valid
+ *                            to the caller when this function returns
+ *                            VINF_SUCCESS.
+ */
+static int gimHvReadSlowHypercallParamsInOut(PVM pVM, PCPUMCTX pCtx, bool fIs64BitMode, int *prcHv)
+{
+    int rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_IN, prcHv);
+    if (   RT_SUCCESS(rc)
+        && *prcHv == GIM_HV_STATUS_SUCCESS)
+        rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_OUT, prcHv);
+    return rc;
+}
+#endif
+
+
+/**
+ * Handles all Hyper-V hypercalls.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VERR_GIM_HYPERCALLS_NOT_ENABLED hypercalls are disabled by the
+ *          guest.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_READ_FAILED hypercall failed while reading
+ *          memory.
+ * @retval  VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED hypercall failed while
+ *          writing memory.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Pointer to the guest-CPU context.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimHvHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+#ifndef IN_RING3
+    RT_NOREF_PV(pVCpu);
+    RT_NOREF_PV(pCtx);
+    return VINF_GIM_R3_HYPERCALL;
+#else
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    STAM_REL_COUNTER_INC(&pVM->gim.s.StatHypercalls);
+
+    /*
+     * Verify that hypercalls are enabled by the guest.
+     */
+    if (!gimHvAreHypercallsEnabled(pVM))
+        return VERR_GIM_HYPERCALLS_NOT_ENABLED;
+
+    /*
+     * Verify guest is in ring-0 protected mode.
+     */
+    uint32_t uCpl = CPUMGetGuestCPL(pVCpu);
+    if (   uCpl
+        || CPUMIsGuestInRealModeEx(pCtx))
+    {
+        return VERR_GIM_HYPERCALL_ACCESS_DENIED;
+    }
+
+    /*
+     * Get the hypercall operation code and modes.
+     * Fast hypercalls have only two or fewer inputs but no output parameters.
+     */
+    const bool       fIs64BitMode     = CPUMIsGuestIn64BitCodeEx(pCtx);
+    const uint64_t   uHyperIn         = fIs64BitMode ? pCtx->rcx : (pCtx->rdx << 32) | pCtx->eax;
+    const uint16_t   uHyperOp         = GIM_HV_HYPERCALL_IN_CALL_CODE(uHyperIn);
+    const bool       fHyperFast       = GIM_HV_HYPERCALL_IN_IS_FAST(uHyperIn);
+    const uint16_t   cHyperReps       = GIM_HV_HYPERCALL_IN_REP_COUNT(uHyperIn);
+    const uint16_t   idxHyperRepStart = GIM_HV_HYPERCALL_IN_REP_START_IDX(uHyperIn);
+    uint64_t         cHyperRepsDone   = 0;
+
+    /* Currently no repeating hypercalls are supported. */
+    RT_NOREF2(cHyperReps, idxHyperRepStart);
+
+    int rc     = VINF_SUCCESS;
+    int rcHv   = GIM_HV_STATUS_OPERATION_DENIED;
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+
+    /*
+     * Validate common hypercall input parameters.
+     */
+    if (   !GIM_HV_HYPERCALL_IN_RSVD_1(uHyperIn)
+        && !GIM_HV_HYPERCALL_IN_RSVD_2(uHyperIn)
+        && !GIM_HV_HYPERCALL_IN_RSVD_3(uHyperIn))
+    {
+        /*
+         * Perform the hypercall.
+         */
+        switch (uHyperOp)
+        {
+            case GIM_HV_HYPERCALL_OP_RETREIVE_DEBUG_DATA:   /* Non-rep, memory IO. */
+            {
+                if (pHv->uPartFlags & GIM_HV_PART_FLAGS_DEBUGGING)
+                {
+                    rc  = gimHvReadSlowHypercallParamsInOut(pVM, pCtx, fIs64BitMode, &rcHv);
+                    if (   RT_SUCCESS(rc)
+                        && rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        LogRelMax(1, ("GIM: HyperV: Initiated debug data reception via hypercall\n"));
+                        rc = gimR3HvHypercallRetrieveDebugData(pVM, &rcHv);
+                        if (RT_FAILURE(rc))
+                            LogRelMax(10, ("GIM: HyperV: gimR3HvHypercallRetrieveDebugData failed. rc=%Rrc\n", rc));
+                    }
+                }
+                else
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                break;
+            }
+
+            case GIM_HV_HYPERCALL_OP_POST_DEBUG_DATA:   /* Non-rep, memory IO. */
+            {
+                if (pHv->uPartFlags & GIM_HV_PART_FLAGS_DEBUGGING)
+                {
+                    rc = gimHvReadSlowHypercallParamsInOut(pVM, pCtx, fIs64BitMode, &rcHv);
+                    if (   RT_SUCCESS(rc)
+                        && rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        LogRelMax(1, ("GIM: HyperV: Initiated debug data transmission via hypercall\n"));
+                        rc = gimR3HvHypercallPostDebugData(pVM, &rcHv);
+                        if (RT_FAILURE(rc))
+                            LogRelMax(10, ("GIM: HyperV: gimR3HvHypercallPostDebugData failed. rc=%Rrc\n", rc));
+                    }
+                }
+                else
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                break;
+            }
+
+            case GIM_HV_HYPERCALL_OP_RESET_DEBUG_SESSION:   /* Non-rep, fast (register IO). */
+            {
+                if (pHv->uPartFlags & GIM_HV_PART_FLAGS_DEBUGGING)
+                {
+                    uint32_t fFlags = 0;
+                    if (!fHyperFast)
+                    {
+                        rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_IN, &rcHv);
+                        if (   RT_SUCCESS(rc)
+                            && rcHv == GIM_HV_STATUS_SUCCESS)
+                        {
+                            PGIMHVDEBUGRESETIN pIn = (PGIMHVDEBUGRESETIN)pHv->pbHypercallIn;
+                            fFlags = pIn->fFlags;
+                        }
+                    }
+                    else
+                    {
+                        rcHv = GIM_HV_STATUS_SUCCESS;
+                        fFlags = fIs64BitMode ? pCtx->rdx : pCtx->ebx;
+                    }
+
+                    /*
+                     * Nothing to flush on the sending side as we don't maintain our own buffers.
+                     */
+                    /** @todo We should probably ask the debug receive thread to flush it's buffer. */
+                    if (rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        if (fFlags)
+                            LogRel(("GIM: HyperV: Resetting debug session via hypercall\n"));
+                        else
+                            rcHv = GIM_HV_STATUS_INVALID_PARAMETER;
+                    }
+                }
+                else
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                break;
+            }
+
+            case GIM_HV_HYPERCALL_OP_POST_MESSAGE:      /* Non-rep, memory IO. */
+            {
+                if (pHv->fIsInterfaceVs)
+                {
+                    rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_IN, &rcHv);
+                    if (   RT_SUCCESS(rc)
+                        && rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        PGIMHVPOSTMESSAGEIN pMsgIn = (PGIMHVPOSTMESSAGEIN)pHv->pbHypercallIn;
+                        PCGIMHVCPU          pHvCpu = &pVCpu->gim.s.u.HvCpu;
+                        if (    pMsgIn->uConnectionId  == GIM_HV_VMBUS_MSG_CONNECTION_ID
+                            &&  pMsgIn->enmMessageType == GIMHVMSGTYPE_VMBUS
+                            && !MSR_GIM_HV_SINT_IS_MASKED(pHvCpu->auSintMsrs[GIM_HV_VMBUS_MSG_SINT])
+                            &&  MSR_GIM_HV_SIMP_IS_ENABLED(pHvCpu->uSimpMsr))
+                        {
+                            RTGCPHYS GCPhysSimp = MSR_GIM_HV_SIMP_GPA(pHvCpu->uSimpMsr);
+                            if (PGMPhysIsGCPhysNormal(pVM, GCPhysSimp))
+                            {
+                                /*
+                                 * The VMBus client (guest) expects to see 0xf at offsets 4 and 16 and 1 at offset 0.
+                                 */
+                                GIMHVMSG HvMsg;
+                                RT_ZERO(HvMsg);
+                                HvMsg.MsgHdr.enmMessageType = GIMHVMSGTYPE_VMBUS;
+                                HvMsg.MsgHdr.cbPayload = 0xf;
+                                HvMsg.aPayload[0]      = 0xf;
+                                uint16_t const offMsg = GIM_HV_VMBUS_MSG_SINT * sizeof(GIMHVMSG);
+                                int rc2 = PGMPhysSimpleWriteGCPhys(pVM, GCPhysSimp + offMsg, &HvMsg, sizeof(HvMsg));
+                                if (RT_SUCCESS(rc2))
+                                    LogRel(("GIM: HyperV: SIMP hypercall faking message at %#RGp:%u\n", GCPhysSimp, offMsg));
+                                else
+                                {
+                                    LogRel(("GIM: HyperV: Failed to write SIMP message at %#RGp:%u, rc=%Rrc\n", GCPhysSimp,
+                                            offMsg, rc));
+                                }
+                            }
+                        }
+
+                        /*
+                         * Make the call fail after updating the SIMP, so the guest can go back to using
+                         * the Hyper-V debug MSR interface. Any error code below GIM_HV_STATUS_NOT_ACKNOWLEDGED
+                         * and the guest tries to proceed with initializing VMBus which is totally unnecessary
+                         * for what we're trying to accomplish, i.e. convince guest to use Hyper-V debugging. Also,
+                         * we don't implement other VMBus/SynIC functionality so the guest would #GP and die.
+                         */
+                        rcHv = GIM_HV_STATUS_NOT_ACKNOWLEDGED;
+                    }
+                    else
+                        rcHv = GIM_HV_STATUS_INVALID_PARAMETER;
+                }
+                else
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                break;
+            }
+
+            case GIM_HV_EXT_HYPERCALL_OP_QUERY_CAP:              /* Non-rep, extended hypercall. */
+            {
+                if (pHv->uPartFlags & GIM_HV_PART_FLAGS_EXTENDED_HYPERCALLS)
+                {
+                    rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_OUT, &rcHv);
+                    if (   RT_SUCCESS(rc)
+                        && rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        rc = gimR3HvHypercallExtQueryCap(pVM, &rcHv);
+                    }
+                }
+                else
+                {
+                    LogRel(("GIM: HyperV: Denied HvExtCallQueryCapabilities when the feature is not exposed\n"));
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                }
+                break;
+            }
+
+            case GIM_HV_EXT_HYPERCALL_OP_GET_BOOT_ZEROED_MEM:    /* Non-rep, extended hypercall. */
+            {
+                if (pHv->uPartFlags & GIM_HV_PART_FLAGS_EXTENDED_HYPERCALLS)
+                {
+                    rc = gimHvReadSlowHypercallParam(pVM, pCtx, fIs64BitMode, GIMHVHYPERCALLPARAM_OUT, &rcHv);
+                    if (   RT_SUCCESS(rc)
+                        && rcHv == GIM_HV_STATUS_SUCCESS)
+                    {
+                        rc = gimR3HvHypercallExtGetBootZeroedMem(pVM, &rcHv);
+                    }
+                }
+                else
+                {
+                    LogRel(("GIM: HyperV: Denied HvExtCallGetBootZeroedMemory when the feature is not exposed\n"));
+                    rcHv = GIM_HV_STATUS_ACCESS_DENIED;
+                }
+                break;
+            }
+
+            default:
+            {
+                LogRel(("GIM: HyperV: Unknown/invalid hypercall opcode %#x (%u)\n", uHyperOp, uHyperOp));
+                rcHv = GIM_HV_STATUS_INVALID_HYPERCALL_CODE;
+                break;
+            }
+        }
+    }
+    else
+        rcHv = GIM_HV_STATUS_INVALID_HYPERCALL_INPUT;
+
+    /*
+     * Update the guest with results of the hypercall.
+     */
+    if (RT_SUCCESS(rc))
+    {
+        if (fIs64BitMode)
+            pCtx->rax = (cHyperRepsDone << 32) | rcHv;
+        else
+        {
+            pCtx->edx = cHyperRepsDone;
+            pCtx->eax = rcHv;
+        }
+    }
+
+    return rc;
+#endif
+}
+
+
+/**
+ * Returns a pointer to the MMIO2 regions supported by Hyper-V.
+ *
+ * @returns Pointer to an array of MMIO2 regions.
+ * @param   pVM         The cross context VM structure.
+ * @param   pcRegions   Where to store the number of regions in the array.
+ */
+VMM_INT_DECL(PGIMMMIO2REGION) gimHvGetMmio2Regions(PVM pVM, uint32_t *pcRegions)
+{
+    Assert(GIMIsEnabled(pVM));
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+
+    AssertCompile(RT_ELEMENTS(pHv->aMmio2Regions) <= 8);
+    *pcRegions = RT_ELEMENTS(pHv->aMmio2Regions);
+    return pHv->aMmio2Regions;
+}
+
+
+/**
+ * Returns whether the guest has configured and enabled the use of Hyper-V's
+ * hypercall interface.
+ *
+ * @returns true if hypercalls are enabled, false otherwise.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(bool) gimHvAreHypercallsEnabled(PCVM pVM)
+{
+    return RT_BOOL(pVM->gim.s.u.Hv.u64GuestOsIdMsr != 0);
+}
+
+
+/**
+ * Returns whether the guest has configured and enabled the use of Hyper-V's
+ * paravirtualized TSC.
+ *
+ * @returns true if paravirt. TSC is enabled, false otherwise.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(bool) gimHvIsParavirtTscEnabled(PVM pVM)
+{
+    return MSR_GIM_HV_REF_TSC_IS_ENABLED(pVM->gim.s.u.Hv.u64TscPageMsr);
+}
+
+
+#ifdef IN_RING3
+/**
+ * Gets the descriptive OS ID variant as identified via the
+ * MSR_GIM_HV_GUEST_OS_ID MSR.
+ *
+ * @returns The name.
+ * @param   uGuestOsIdMsr     The MSR_GIM_HV_GUEST_OS_ID MSR.
+ */
+static const char *gimHvGetGuestOsIdVariantName(uint64_t uGuestOsIdMsr)
+{
+    /* Refer the Hyper-V spec, section 3.6 "Reporting the Guest OS Identity". */
+    uint32_t uVendor = MSR_GIM_HV_GUEST_OS_ID_VENDOR(uGuestOsIdMsr);
+    if (uVendor == 1 /* Microsoft */)
+    {
+        uint32_t uOsVariant = MSR_GIM_HV_GUEST_OS_ID_OS_VARIANT(uGuestOsIdMsr);
+        switch (uOsVariant)
+        {
+            case 0:  return "Undefined";
+            case 1:  return "MS-DOS";
+            case 2:  return "Windows 3.x";
+            case 3:  return "Windows 9x";
+            case 4:  return "Windows NT or derivative";
+            case 5:  return "Windows CE";
+            default: return "Unknown";
+        }
+    }
+    return "Unknown";
+}
+#endif
+
+/**
+ * Gets the time reference count for the current VM.
+ *
+ * @returns The time reference count.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+DECLINLINE(uint64_t) gimHvGetTimeRefCount(PVMCPUCC pVCpu)
+{
+    /* Hyper-V reports the time in 100 ns units (10 MHz). */
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+    PCGIMHV pHv = &pVCpu->CTX_SUFF(pVM)->gim.s.u.Hv;
+    uint64_t const u64Tsc        = TMCpuTickGet(pVCpu);     /** @todo should we be passing VCPU0 always? */
+    uint64_t const u64TscHz      = pHv->cTscTicksPerSecond;
+    uint64_t const u64Tsc100NS   = u64TscHz / UINT64_C(10000000); /* 100 ns */
+    uint64_t const uTimeRefCount = (u64Tsc / u64Tsc100NS);
+    return uTimeRefCount;
+}
+
+
+/**
+ * Starts the synthetic timer.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pHvStimer   Pointer to the Hyper-V synthetic timer.
+ *
+ * @remarks Caller needs to hold the timer critical section.
+ * @thread  Any.
+ */
+VMM_INT_DECL(void) gimHvStartStimer(PVMCPUCC pVCpu, PCGIMHVSTIMER pHvStimer)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    TMTIMERHANDLE hTimer = pHvStimer->hTimer;
+    Assert(TMTimerIsLockOwner(pVM, hTimer));
+
+    uint64_t const uTimerCount = pHvStimer->uStimerCountMsr;
+    if (uTimerCount)
+    {
+        uint64_t const uTimerCountNS = uTimerCount * 100;
+
+        /* For periodic timers, 'uTimerCountNS' represents the relative interval. */
+        if (MSR_GIM_HV_STIMER_IS_PERIODIC(pHvStimer->uStimerConfigMsr))
+        {
+            TMTimerSetNano(pVM, hTimer, uTimerCountNS);
+            LogFlow(("GIM%u: HyperV: Started relative periodic STIMER%u with uTimerCountNS=%RU64\n", pVCpu->idCpu,
+                     pHvStimer->idxStimer, uTimerCountNS));
+        }
+        else
+        {
+            /* For one-shot timers, 'uTimerCountNS' represents an absolute expiration wrt to Hyper-V reference time,
+               we convert it to a relative time and program the timer. */
+            uint64_t const uCurRefTimeNS = gimHvGetTimeRefCount(pVCpu) * 100;
+            if (uTimerCountNS > uCurRefTimeNS)
+            {
+                uint64_t const uRelativeNS = uTimerCountNS - uCurRefTimeNS;
+                TMTimerSetNano(pVM, hTimer, uRelativeNS);
+                LogFlow(("GIM%u: HyperV: Started one-shot relative STIMER%u with uRelativeNS=%RU64\n", pVCpu->idCpu,
+                         pHvStimer->idxStimer, uRelativeNS));
+            }
+        }
+        /** @todo frequency hinting? */
+    }
+}
+
+
+/**
+ * Stops the synthetic timer for the given VCPU.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pHvStimer   Pointer to the Hyper-V synthetic timer.
+ *
+ * @remarks Caller needs to the hold the timer critical section.
+ * @thread  EMT(pVCpu).
+ */
+static void gimHvStopStimer(PVMCPUCC pVCpu, PGIMHVSTIMER pHvStimer)
+{
+    VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    TMTIMERHANDLE hTimer = pHvStimer->hTimer;
+    Assert(TMTimerIsLockOwner(pVM, hTimer));
+
+    if (TMTimerIsActive(pVM, hTimer))
+        TMTimerStop(pVM, hTimer);
+}
+
+
+/**
+ * MSR read handler for Hyper-V.
+ *
+ * @returns Strict VBox status code like CPUMQueryGuestMsr().
+ * @retval  VINF_CPUM_R3_MSR_READ
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR being read.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   puValue     Where to store the MSR value read.
+ *
+ * @thread  EMT.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimHvReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    NOREF(pRange);
+    PVMCC   pVM = pVCpu->CTX_SUFF(pVM);
+    PCGIMHV pHv = &pVM->gim.s.u.Hv;
+
+    switch (idMsr)
+    {
+        case MSR_GIM_HV_TIME_REF_COUNT:
+            *puValue = gimHvGetTimeRefCount(pVCpu);
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_VP_INDEX:
+            *puValue = pVCpu->idCpu;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_TPR:
+            *puValue = APICHvGetTpr(pVCpu);
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_ICR:
+            *puValue = APICHvGetIcr(pVCpu);
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_GUEST_OS_ID:
+            *puValue = pHv->u64GuestOsIdMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_HYPERCALL:
+            *puValue = pHv->u64HypercallMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_REF_TSC:
+            *puValue = pHv->u64TscPageMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_TSC_FREQ:
+            *puValue = TMCpuTicksPerSecond(pVM);
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_APIC_FREQ:
+        {
+            int rc = APICGetTimerFreq(pVM, puValue);
+            if (RT_FAILURE(rc))
+                return VERR_CPUM_RAISE_GP_0;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SYNTH_DEBUG_STATUS:
+            *puValue = pHv->uDbgStatusMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_SINT0:   case MSR_GIM_HV_SINT1:   case MSR_GIM_HV_SINT2:   case MSR_GIM_HV_SINT3:
+        case MSR_GIM_HV_SINT4:   case MSR_GIM_HV_SINT5:   case MSR_GIM_HV_SINT6:   case MSR_GIM_HV_SINT7:
+        case MSR_GIM_HV_SINT8:   case MSR_GIM_HV_SINT9:   case MSR_GIM_HV_SINT10:  case MSR_GIM_HV_SINT11:
+        case MSR_GIM_HV_SINT12:  case MSR_GIM_HV_SINT13:  case MSR_GIM_HV_SINT14:  case MSR_GIM_HV_SINT15:
+        {
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            *puValue = pHvCpu->auSintMsrs[idMsr - MSR_GIM_HV_SINT0];
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_STIMER0_CONFIG:
+        case MSR_GIM_HV_STIMER1_CONFIG:
+        case MSR_GIM_HV_STIMER2_CONFIG:
+        case MSR_GIM_HV_STIMER3_CONFIG:
+        {
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            uint8_t const idxStimer  = (idMsr - MSR_GIM_HV_STIMER0_CONFIG) >> 1;
+            PCGIMHVSTIMER pcHvStimer = &pHvCpu->aStimers[idxStimer];
+            *puValue = pcHvStimer->uStimerConfigMsr;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_STIMER0_COUNT:
+        case MSR_GIM_HV_STIMER1_COUNT:
+        case MSR_GIM_HV_STIMER2_COUNT:
+        case MSR_GIM_HV_STIMER3_COUNT:
+        {
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            uint8_t const idxStimer  = (idMsr - MSR_GIM_HV_STIMER0_COUNT) >> 1;
+            PCGIMHVSTIMER pcHvStimer = &pHvCpu->aStimers[idxStimer];
+            *puValue = pcHvStimer->uStimerCountMsr;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_EOM:
+        {
+            *puValue = 0;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SCONTROL:
+        {
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            *puValue = pHvCpu->uSControlMsr;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SIMP:
+        {
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            *puValue = pHvCpu->uSimpMsr;
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SVERSION:
+            *puValue = GIM_HV_SVERSION;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_RESET:
+            *puValue = 0;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_CRASH_CTL:
+            *puValue = pHv->uCrashCtlMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_HV_CRASH_P0: *puValue = pHv->uCrashP0Msr;   return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P1: *puValue = pHv->uCrashP1Msr;   return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P2: *puValue = pHv->uCrashP2Msr;   return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P3: *puValue = pHv->uCrashP3Msr;   return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P4: *puValue = pHv->uCrashP4Msr;   return VINF_SUCCESS;
+
+        case MSR_GIM_HV_DEBUG_OPTIONS_MSR:
+        {
+            if (pHv->fIsVendorMsHv)
+            {
+#ifndef IN_RING3
+                return VINF_CPUM_R3_MSR_READ;
+#else
+                LogRelMax(1, ("GIM: HyperV: Guest querying debug options, suggesting %s interface\n",
+                              pHv->fDbgHypercallInterface ? "hypercall" : "MSR"));
+                *puValue = pHv->fDbgHypercallInterface ? GIM_HV_DEBUG_OPTIONS_USE_HYPERCALLS : 0;
+                return VINF_SUCCESS;
+#endif
+            }
+            break;
+        }
+
+        /* Write-only MSRs: */
+        case MSR_GIM_HV_EOI:
+        /* Reserved/unknown MSRs: */
+        default:
+        {
+#ifdef IN_RING3
+            static uint32_t s_cTimes = 0;
+            if (s_cTimes++ < 20)
+                LogRel(("GIM: HyperV: Unknown/invalid RdMsr (%#x) -> #GP(0)\n", idMsr));
+            LogFunc(("Unknown/invalid RdMsr (%#RX32) -> #GP(0)\n", idMsr));
+            break;
+#else
+            return VINF_CPUM_R3_MSR_READ;
+#endif
+        }
+    }
+
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * MSR write handler for Hyper-V.
+ *
+ * @returns Strict VBox status code like CPUMSetGuestMsr().
+ * @retval  VINF_CPUM_R3_MSR_WRITE
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR being written.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   uRawValue   The raw value with the ignored bits not masked.
+ *
+ * @thread  EMT.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimHvWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uRawValue)
+{
+    NOREF(pRange);
+    PVMCC  pVM = pVCpu->CTX_SUFF(pVM);
+    PGIMHV pHv = &pVM->gim.s.u.Hv;
+
+    switch (idMsr)
+    {
+        case MSR_GIM_HV_TPR:
+            return APICHvSetTpr(pVCpu, uRawValue);
+
+        case MSR_GIM_HV_EOI:
+            return APICHvSetEoi(pVCpu, uRawValue);
+
+        case MSR_GIM_HV_ICR:
+            return APICHvSetIcr(pVCpu, uRawValue);
+
+        case MSR_GIM_HV_GUEST_OS_ID:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            /* Disable the hypercall-page and hypercalls if 0 is written to this MSR. */
+            if (!uRawValue)
+            {
+                if (MSR_GIM_HV_HYPERCALL_PAGE_IS_ENABLED(pHv->u64HypercallMsr))
+                {
+                    gimR3HvDisableHypercallPage(pVM);
+                    pHv->u64HypercallMsr &= ~MSR_GIM_HV_HYPERCALL_PAGE_ENABLE;
+                    LogRel(("GIM: HyperV: Hypercall page disabled via Guest OS ID MSR\n"));
+                }
+            }
+            else
+            {
+                LogRel(("GIM: HyperV: Guest OS reported ID %#RX64\n", uRawValue));
+                LogRel(("GIM: HyperV: Open-source=%RTbool Vendor=%#x OS=%#x (%s) Major=%u Minor=%u ServicePack=%u Build=%u\n",
+                        MSR_GIM_HV_GUEST_OS_ID_IS_OPENSOURCE(uRawValue),   MSR_GIM_HV_GUEST_OS_ID_VENDOR(uRawValue),
+                        MSR_GIM_HV_GUEST_OS_ID_OS_VARIANT(uRawValue),      gimHvGetGuestOsIdVariantName(uRawValue),
+                        MSR_GIM_HV_GUEST_OS_ID_MAJOR_VERSION(uRawValue),   MSR_GIM_HV_GUEST_OS_ID_MINOR_VERSION(uRawValue),
+                        MSR_GIM_HV_GUEST_OS_ID_SERVICE_VERSION(uRawValue), MSR_GIM_HV_GUEST_OS_ID_BUILD(uRawValue)));
+
+                /* Update the CPUID leaf, see Hyper-V spec. "Microsoft Hypervisor CPUID Leaves". */
+                CPUMCPUIDLEAF HyperLeaf;
+                RT_ZERO(HyperLeaf);
+                HyperLeaf.uLeaf = UINT32_C(0x40000002);
+                HyperLeaf.uEax  = MSR_GIM_HV_GUEST_OS_ID_BUILD(uRawValue);
+                HyperLeaf.uEbx  =  MSR_GIM_HV_GUEST_OS_ID_MINOR_VERSION(uRawValue)
+                                | (MSR_GIM_HV_GUEST_OS_ID_MAJOR_VERSION(uRawValue) << 16);
+                HyperLeaf.uEcx  = MSR_GIM_HV_GUEST_OS_ID_SERVICE_VERSION(uRawValue);
+                HyperLeaf.uEdx  =  MSR_GIM_HV_GUEST_OS_ID_SERVICE_VERSION(uRawValue)
+                                | (MSR_GIM_HV_GUEST_OS_ID_BUILD(uRawValue) << 24);
+                int rc2 = CPUMR3CpuIdInsert(pVM, &HyperLeaf);
+                AssertRC(rc2);
+            }
+
+            pHv->u64GuestOsIdMsr = uRawValue;
+
+            /*
+             * Update EM on hypercall instruction enabled state.
+             */
+            if (uRawValue)
+                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                    EMSetHypercallInstructionsEnabled(pVM->CTX_SUFF(apCpus)[idCpu], true);
+            else
+                for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+                    EMSetHypercallInstructionsEnabled(pVM->CTX_SUFF(apCpus)[idCpu], false);
+
+            return VINF_SUCCESS;
+#endif /* IN_RING3 */
+        }
+
+        case MSR_GIM_HV_HYPERCALL:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            /** @todo There is/was a problem with hypercalls for FreeBSD 10.1 guests,
+             *  see @bugref{7270#c116}. */
+            /* First, update all but the hypercall page enable bit. */
+            pHv->u64HypercallMsr = (uRawValue & ~MSR_GIM_HV_HYPERCALL_PAGE_ENABLE);
+
+            /* Hypercall page can only be enabled when the guest has enabled hypercalls. */
+            bool fEnable = MSR_GIM_HV_HYPERCALL_PAGE_IS_ENABLED(uRawValue);
+            if (   fEnable
+                && !gimHvAreHypercallsEnabled(pVM))
+            {
+                return VINF_SUCCESS;
+            }
+
+            /* Is the guest disabling the hypercall-page? Allow it regardless of the Guest-OS Id Msr. */
+            if (!fEnable)
+            {
+                gimR3HvDisableHypercallPage(pVM);
+                pHv->u64HypercallMsr = uRawValue;
+                return VINF_SUCCESS;
+            }
+
+            /* Enable the hypercall-page. */
+            RTGCPHYS GCPhysHypercallPage = MSR_GIM_HV_HYPERCALL_GUEST_PFN(uRawValue) << GUEST_PAGE_SHIFT;
+            int rc = gimR3HvEnableHypercallPage(pVM, GCPhysHypercallPage);
+            if (RT_SUCCESS(rc))
+            {
+                pHv->u64HypercallMsr = uRawValue;
+                return VINF_SUCCESS;
+            }
+
+            return VERR_CPUM_RAISE_GP_0;
+#endif
+        }
+
+        case MSR_GIM_HV_REF_TSC:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else  /* IN_RING3 */
+            /* First, update all but the TSC page enable bit. */
+            pHv->u64TscPageMsr = (uRawValue & ~MSR_GIM_HV_REF_TSC_ENABLE);
+
+            /* Is the guest disabling the TSC page? */
+            bool fEnable = MSR_GIM_HV_REF_TSC_IS_ENABLED(uRawValue);
+            if (!fEnable)
+            {
+                gimR3HvDisableTscPage(pVM);
+                pHv->u64TscPageMsr = uRawValue;
+                return VINF_SUCCESS;
+            }
+
+            /* Enable the TSC page. */
+            RTGCPHYS GCPhysTscPage = MSR_GIM_HV_REF_TSC_GUEST_PFN(uRawValue) << GUEST_PAGE_SHIFT;
+            int rc = gimR3HvEnableTscPage(pVM, GCPhysTscPage, false /* fUseThisTscSequence */, 0 /* uTscSequence */);
+            if (RT_SUCCESS(rc))
+            {
+                pHv->u64TscPageMsr = uRawValue;
+                return VINF_SUCCESS;
+            }
+
+            return VERR_CPUM_RAISE_GP_0;
+#endif /* IN_RING3 */
+        }
+
+        case MSR_GIM_HV_APIC_ASSIST_PAGE:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else  /* IN_RING3 */
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            pHvCpu->uApicAssistPageMsr = uRawValue;
+
+            if (MSR_GIM_HV_APICASSIST_PAGE_IS_ENABLED(uRawValue))
+            {
+                RTGCPHYS GCPhysApicAssistPage = MSR_GIM_HV_APICASSIST_GUEST_PFN(uRawValue) << GUEST_PAGE_SHIFT;
+                if (PGMPhysIsGCPhysNormal(pVM, GCPhysApicAssistPage))
+                {
+                    int rc = gimR3HvEnableApicAssistPage(pVCpu, GCPhysApicAssistPage);
+                    if (RT_SUCCESS(rc))
+                    {
+                        pHvCpu->uApicAssistPageMsr = uRawValue;
+                        return VINF_SUCCESS;
+                    }
+                }
+                else
+                {
+                    LogRelMax(5, ("GIM%u: HyperV: APIC-assist page address %#RGp invalid!\n", pVCpu->idCpu,
+                                  GCPhysApicAssistPage));
+                }
+            }
+            else
+                gimR3HvDisableApicAssistPage(pVCpu);
+
+            return VERR_CPUM_RAISE_GP_0;
+#endif /* IN_RING3 */
+        }
+
+        case MSR_GIM_HV_RESET:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            if (MSR_GIM_HV_RESET_IS_ENABLED(uRawValue))
+            {
+                LogRel(("GIM: HyperV: Reset initiated through MSR\n"));
+                int rc = PDMDevHlpVMReset(pVM->gim.s.pDevInsR3, PDMVMRESET_F_GIM);
+                AssertRC(rc); /* Note! Not allowed to return VINF_EM_RESET / VINF_EM_HALT here, so ignore them. */
+            }
+            /* else: Ignore writes to other bits. */
+            return VINF_SUCCESS;
+#endif /* IN_RING3 */
+        }
+
+        case MSR_GIM_HV_CRASH_CTL:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            if (uRawValue & MSR_GIM_HV_CRASH_CTL_NOTIFY)
+            {
+                LogRel(("GIM: HyperV: Guest indicates a fatal condition! P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64\n",
+                        pHv->uCrashP0Msr, pHv->uCrashP1Msr, pHv->uCrashP2Msr, pHv->uCrashP3Msr, pHv->uCrashP4Msr));
+                DBGFR3ReportBugCheck(pVM, pVCpu, DBGFEVENT_BSOD_MSR, pHv->uCrashP0Msr, pHv->uCrashP1Msr,
+                                     pHv->uCrashP2Msr, pHv->uCrashP3Msr, pHv->uCrashP4Msr);
+                /* (Do not try pass VINF_EM_DBG_EVENT, doesn't work from here!) */
+            }
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_SYNTH_DEBUG_SEND_BUFFER:
+        {
+            if (!pHv->fDbgEnabled)
+                return VERR_CPUM_RAISE_GP_0;
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            RTGCPHYS GCPhysBuffer    = (RTGCPHYS)uRawValue;
+            pHv->uDbgSendBufferMsr = GCPhysBuffer;
+            if (PGMPhysIsGCPhysNormal(pVM, GCPhysBuffer))
+                LogRel(("GIM: HyperV: Set up debug send buffer at %#RGp\n", GCPhysBuffer));
+            else
+                LogRel(("GIM: HyperV: Destroyed debug send buffer\n"));
+            pHv->uDbgSendBufferMsr = uRawValue;
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_SYNTH_DEBUG_RECEIVE_BUFFER:
+        {
+            if (!pHv->fDbgEnabled)
+                return VERR_CPUM_RAISE_GP_0;
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            RTGCPHYS GCPhysBuffer  = (RTGCPHYS)uRawValue;
+            pHv->uDbgRecvBufferMsr = GCPhysBuffer;
+            if (PGMPhysIsGCPhysNormal(pVM, GCPhysBuffer))
+                LogRel(("GIM: HyperV: Set up debug receive buffer at %#RGp\n", GCPhysBuffer));
+            else
+                LogRel(("GIM: HyperV: Destroyed debug receive buffer\n"));
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_SYNTH_DEBUG_PENDING_BUFFER:
+        {
+            if (!pHv->fDbgEnabled)
+                return VERR_CPUM_RAISE_GP_0;
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            RTGCPHYS GCPhysBuffer      = (RTGCPHYS)uRawValue;
+            pHv->uDbgPendingBufferMsr  = GCPhysBuffer;
+            if (PGMPhysIsGCPhysNormal(pVM, GCPhysBuffer))
+                LogRel(("GIM: HyperV: Set up debug pending buffer at %#RGp\n", uRawValue));
+            else
+                LogRel(("GIM: HyperV: Destroyed debug pending buffer\n"));
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_SYNTH_DEBUG_CONTROL:
+        {
+            if (!pHv->fDbgEnabled)
+                return VERR_CPUM_RAISE_GP_0;
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            if (   MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_WRITE(uRawValue)
+                && MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_READ(uRawValue))
+            {
+                LogRel(("GIM: HyperV: Requesting both read and write through debug control MSR -> #GP(0)\n"));
+                return VERR_CPUM_RAISE_GP_0;
+            }
+
+            if (MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_WRITE(uRawValue))
+            {
+                uint32_t cbWrite = MSR_GIM_HV_SYNTH_DEBUG_CONTROL_W_LEN(uRawValue);
+                if (   cbWrite > 0
+                    && cbWrite < GIM_HV_PAGE_SIZE)
+                {
+                    if (PGMPhysIsGCPhysNormal(pVM, (RTGCPHYS)pHv->uDbgSendBufferMsr))
+                    {
+                        Assert(pHv->pvDbgBuffer);
+                        int rc = PGMPhysSimpleReadGCPhys(pVM, pHv->pvDbgBuffer, (RTGCPHYS)pHv->uDbgSendBufferMsr, cbWrite);
+                        if (RT_SUCCESS(rc))
+                        {
+                            LogRelMax(1, ("GIM: HyperV: Initiated debug data transmission via MSR\n"));
+                            uint32_t cbWritten = 0;
+                            rc = gimR3HvDebugWrite(pVM, pHv->pvDbgBuffer, cbWrite, &cbWritten, false /*fUdpPkt*/);
+                            if (   RT_SUCCESS(rc)
+                                && cbWrite == cbWritten)
+                                pHv->uDbgStatusMsr = MSR_GIM_HV_SYNTH_DEBUG_STATUS_W_SUCCESS;
+                            else
+                                pHv->uDbgStatusMsr = 0;
+                        }
+                        else
+                            LogRelMax(5, ("GIM: HyperV: Failed to read debug send buffer at %#RGp, rc=%Rrc\n",
+                                          (RTGCPHYS)pHv->uDbgSendBufferMsr, rc));
+                    }
+                    else
+                        LogRelMax(5, ("GIM: HyperV: Debug send buffer address %#RGp invalid! Ignoring debug write!\n",
+                                      (RTGCPHYS)pHv->uDbgSendBufferMsr));
+                }
+                else
+                    LogRelMax(5, ("GIM: HyperV: Invalid write size %u specified in MSR, ignoring debug write!\n",
+                                  MSR_GIM_HV_SYNTH_DEBUG_CONTROL_W_LEN(uRawValue)));
+            }
+            else if (MSR_GIM_HV_SYNTH_DEBUG_CONTROL_IS_READ(uRawValue))
+            {
+                if (PGMPhysIsGCPhysNormal(pVM, (RTGCPHYS)pHv->uDbgRecvBufferMsr))
+                {
+                    LogRelMax(1, ("GIM: HyperV: Initiated debug data reception via MSR\n"));
+                    uint32_t cbReallyRead;
+                    Assert(pHv->pvDbgBuffer);
+                    int rc = gimR3HvDebugRead(pVM, pHv->pvDbgBuffer, GIM_HV_PAGE_SIZE, GIM_HV_PAGE_SIZE,
+                                              &cbReallyRead, 0, false /*fUdpPkt*/);
+                    if (   RT_SUCCESS(rc)
+                        && cbReallyRead > 0)
+                    {
+                        rc = PGMPhysSimpleWriteGCPhys(pVM, (RTGCPHYS)pHv->uDbgRecvBufferMsr, pHv->pvDbgBuffer, cbReallyRead);
+                        if (RT_SUCCESS(rc))
+                        {
+                            pHv->uDbgStatusMsr  = ((uint16_t)cbReallyRead) << 16;
+                            pHv->uDbgStatusMsr |= MSR_GIM_HV_SYNTH_DEBUG_STATUS_R_SUCCESS;
+                        }
+                        else
+                        {
+                            pHv->uDbgStatusMsr = 0;
+                            LogRelMax(5, ("GIM: HyperV: PGMPhysSimpleWriteGCPhys failed. rc=%Rrc\n", rc));
+                        }
+                    }
+                    else
+                        pHv->uDbgStatusMsr = 0;
+                }
+                else
+                {
+                    LogRelMax(5, ("GIM: HyperV: Debug receive buffer address %#RGp invalid! Ignoring debug read!\n",
+                                  (RTGCPHYS)pHv->uDbgRecvBufferMsr));
+                }
+            }
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_SINT0:    case MSR_GIM_HV_SINT1:    case MSR_GIM_HV_SINT2:    case MSR_GIM_HV_SINT3:
+        case MSR_GIM_HV_SINT4:    case MSR_GIM_HV_SINT5:    case MSR_GIM_HV_SINT6:    case MSR_GIM_HV_SINT7:
+        case MSR_GIM_HV_SINT8:    case MSR_GIM_HV_SINT9:    case MSR_GIM_HV_SINT10:   case MSR_GIM_HV_SINT11:
+        case MSR_GIM_HV_SINT12:   case MSR_GIM_HV_SINT13:   case MSR_GIM_HV_SINT14:   case MSR_GIM_HV_SINT15:
+        {
+            uint8_t      uVector    = MSR_GIM_HV_SINT_GET_VECTOR(uRawValue);
+            bool const   fVMBusMsg  = RT_BOOL(idMsr == GIM_HV_VMBUS_MSG_SINT);
+            size_t const idxSintMsr = idMsr - MSR_GIM_HV_SINT0;
+            const char  *pszDesc    = fVMBusMsg ? "VMBus Message" : "Generic";
+            if (uVector < GIM_HV_SINT_VECTOR_VALID_MIN)
+            {
+                LogRel(("GIM%u: HyperV: Programmed an invalid vector in SINT%u (%s), uVector=%u -> #GP(0)\n", pVCpu->idCpu,
+                        idxSintMsr, pszDesc, uVector));
+                return VERR_CPUM_RAISE_GP_0;
+            }
+
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            pHvCpu->auSintMsrs[idxSintMsr] = uRawValue;
+            if (fVMBusMsg)
+            {
+                if (MSR_GIM_HV_SINT_IS_MASKED(uRawValue))
+                    Log(("GIM%u: HyperV: Masked SINT%u (%s)\n", pVCpu->idCpu, idxSintMsr, pszDesc));
+                else
+                    Log(("GIM%u: HyperV: Unmasked SINT%u (%s), uVector=%u\n", pVCpu->idCpu, idxSintMsr, pszDesc, uVector));
+            }
+            Log(("GIM%u: HyperV: Written SINT%u=%#RX64\n", pVCpu->idCpu, idxSintMsr, uRawValue));
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SCONTROL:
+        {
+#ifndef IN_RING3
+            /** @todo make this RZ later? */
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            pHvCpu->uSControlMsr = uRawValue;
+            if (MSR_GIM_HV_SCONTROL_IS_ENABLED(uRawValue))
+                LogRel(("GIM%u: HyperV: Synthetic interrupt control enabled\n", pVCpu->idCpu));
+            else
+                LogRel(("GIM%u: HyperV: Synthetic interrupt control disabled\n", pVCpu->idCpu));
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_STIMER0_CONFIG:
+        case MSR_GIM_HV_STIMER1_CONFIG:
+        case MSR_GIM_HV_STIMER2_CONFIG:
+        case MSR_GIM_HV_STIMER3_CONFIG:
+        {
+            PGIMHVCPU     pHvCpu    = &pVCpu->gim.s.u.HvCpu;
+            uint8_t const idxStimer = (idMsr - MSR_GIM_HV_STIMER0_CONFIG) >> 1;
+
+            /* Validate the writable bits. */
+            if (RT_LIKELY(!(uRawValue & ~MSR_GIM_HV_STIMER_RW_VALID)))
+            {
+                Assert(idxStimer < RT_ELEMENTS(pHvCpu->aStimers));
+                PGIMHVSTIMER pHvStimer = &pHvCpu->aStimers[idxStimer];
+
+                /* Lock to prevent concurrent access from the timer callback. */
+                int rc = TMTimerLock(pVM, pHvStimer->hTimer, VERR_IGNORED);
+                if (rc == VINF_SUCCESS)
+                {
+                    /* Update the MSR value. */
+                    pHvStimer->uStimerConfigMsr = uRawValue;
+                    Log(("GIM%u: HyperV: Set STIMER_CONFIG%u=%#RX64\n", pVCpu->idCpu, idxStimer, uRawValue));
+
+                    /* Process the MSR bits. */
+                    if (   !MSR_GIM_HV_STIMER_GET_SINTX(uRawValue)   /* Writing SINTx as 0 causes the timer to be disabled. */
+                        || !MSR_GIM_HV_STIMER_IS_ENABLED(uRawValue))
+                    {
+                        pHvStimer->uStimerConfigMsr &= ~MSR_GIM_HV_STIMER_ENABLE;
+                        gimHvStopStimer(pVCpu, pHvStimer);
+                        Log(("GIM%u: HyperV: Disabled STIMER_CONFIG%u\n", pVCpu->idCpu, idxStimer));
+                    }
+                    else if (MSR_GIM_HV_STIMER_IS_ENABLED(uRawValue))
+                    {
+                        /* Auto-enable implies writing to the STIMERx_COUNT MSR is what starts the timer. */
+                        if (!MSR_GIM_HV_STIMER_IS_AUTO_ENABLED(uRawValue))
+                        {
+                            if (!TMTimerIsActive(pVM, pHvStimer->hTimer))
+                            {
+                                gimHvStartStimer(pVCpu, pHvStimer);
+                                Log(("GIM%u: HyperV: Started STIMER%u\n", pVCpu->idCpu, idxStimer));
+                            }
+                            else
+                            {
+                                /*
+                                 * Enabling a timer that's already enabled is undefined behaviour,
+                                 * see Hyper-V spec. 15.3.1 "Synthetic Timer Configuration Register".
+                                 *
+                                 * Our implementation just re-starts the timer. Guests that comform to
+                                 * the Hyper-V specs. should not be doing this anyway.
+                                 */
+                                AssertFailed();
+                                gimHvStopStimer(pVCpu, pHvStimer);
+                                gimHvStartStimer(pVCpu, pHvStimer);
+                            }
+                        }
+                    }
+
+                    TMTimerUnlock(pVM, pHvStimer->hTimer);
+                }
+                return rc;
+            }
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            LogRel(("GIM%u: HyperV: Setting reserved bits of STIMER%u MSR (uRawValue=%#RX64) -> #GP(0)\n", pVCpu->idCpu,
+                    idxStimer, uRawValue));
+            return VERR_CPUM_RAISE_GP_0;
+#endif
+        }
+
+        case MSR_GIM_HV_STIMER0_COUNT:
+        case MSR_GIM_HV_STIMER1_COUNT:
+        case MSR_GIM_HV_STIMER2_COUNT:
+        case MSR_GIM_HV_STIMER3_COUNT:
+        {
+            PGIMHVCPU     pHvCpu    = &pVCpu->gim.s.u.HvCpu;
+            uint8_t const idxStimer = (idMsr - MSR_GIM_HV_STIMER0_CONFIG) >> 1;
+            Assert(idxStimer < RT_ELEMENTS(pHvCpu->aStimers));
+            PGIMHVSTIMER  pHvStimer = &pHvCpu->aStimers[idxStimer];
+            int const     rcBusy    = VINF_CPUM_R3_MSR_WRITE;
+
+            /*
+             * Writing zero to this MSR disables the timer regardless of whether the auto-enable
+             * flag is set in the config MSR corresponding to the timer.
+             */
+            if (!uRawValue)
+            {
+                gimHvStopStimer(pVCpu, pHvStimer);
+                pHvStimer->uStimerCountMsr = 0;
+                Log(("GIM%u: HyperV: Set STIMER_COUNT%u=%RU64, stopped timer\n", pVCpu->idCpu, idxStimer, uRawValue));
+                return VINF_SUCCESS;
+            }
+
+            /*
+             * Concurrent writes to the config. MSR can't happen as it's serialized by way
+             * of being done on the same EMT as this.
+             */
+            if (MSR_GIM_HV_STIMER_IS_AUTO_ENABLED(pHvStimer->uStimerConfigMsr))
+            {
+                int rc = TMTimerLock(pVM, pHvStimer->hTimer, rcBusy);
+                if (rc == VINF_SUCCESS)
+                {
+                    pHvStimer->uStimerCountMsr = uRawValue;
+                    gimHvStartStimer(pVCpu, pHvStimer);
+                    TMTimerUnlock(pVM, pHvStimer->hTimer);
+                    Log(("GIM%u: HyperV: Set STIMER_COUNT%u=%RU64 %RU64 msec, auto-started timer\n", pVCpu->idCpu, idxStimer,
+                         uRawValue, (uRawValue * 100) / RT_NS_1MS_64));
+                }
+                return rc;
+            }
+
+            /* Simple update of the counter without any timer start/stop side-effects. */
+            pHvStimer->uStimerCountMsr = uRawValue;
+            Log(("GIM%u: HyperV: Set STIMER_COUNT%u=%RU64\n", pVCpu->idCpu, idxStimer, uRawValue));
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_EOM:
+        {
+            /** @todo implement EOM. */
+            Log(("GIM%u: HyperV: EOM\n", pVCpu->idCpu));
+            return VINF_SUCCESS;
+        }
+
+        case MSR_GIM_HV_SIEFP:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            pHvCpu->uSiefpMsr = uRawValue;
+            if (MSR_GIM_HV_SIEF_PAGE_IS_ENABLED(uRawValue))
+            {
+                RTGCPHYS GCPhysSiefPage = MSR_GIM_HV_SIEF_GUEST_PFN(uRawValue) << GUEST_PAGE_SHIFT;
+                if (PGMPhysIsGCPhysNormal(pVM, GCPhysSiefPage))
+                {
+                    int rc = gimR3HvEnableSiefPage(pVCpu, GCPhysSiefPage);
+                    if (RT_SUCCESS(rc))
+                    {
+                        LogRel(("GIM%u: HyperV: Enabled synthetic interrupt event flags page at %#RGp\n", pVCpu->idCpu,
+                                GCPhysSiefPage));
+                        /** @todo SIEF setup. */
+                        return VINF_SUCCESS;
+                    }
+                }
+                else
+                    LogRelMax(5, ("GIM%u: HyperV: SIEF page address %#RGp invalid!\n", pVCpu->idCpu, GCPhysSiefPage));
+            }
+            else
+                gimR3HvDisableSiefPage(pVCpu);
+
+            return VERR_CPUM_RAISE_GP_0;
+#endif
+            break;
+        }
+
+        case MSR_GIM_HV_SIMP:
+        {
+#ifndef IN_RING3
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            PGIMHVCPU pHvCpu = &pVCpu->gim.s.u.HvCpu;
+            pHvCpu->uSimpMsr = uRawValue;
+            if (MSR_GIM_HV_SIMP_IS_ENABLED(uRawValue))
+            {
+                RTGCPHYS GCPhysSimp = MSR_GIM_HV_SIMP_GPA(uRawValue);
+                if (PGMPhysIsGCPhysNormal(pVM, GCPhysSimp))
+                {
+                    uint8_t abSimp[GIM_HV_PAGE_SIZE];
+                    RT_ZERO(abSimp);
+                    int rc2 = PGMPhysSimpleWriteGCPhys(pVM, GCPhysSimp, &abSimp[0], sizeof(abSimp));
+                    if (RT_SUCCESS(rc2))
+                        LogRel(("GIM%u: HyperV: Enabled synthetic interrupt message page at %#RGp\n", pVCpu->idCpu, GCPhysSimp));
+                    else
+                    {
+                        LogRel(("GIM%u: HyperV: Failed to update synthetic interrupt message page at %#RGp. uSimpMsr=%#RX64 rc=%Rrc\n",
+                                pVCpu->idCpu, pHvCpu->uSimpMsr, GCPhysSimp, rc2));
+                        return VERR_CPUM_RAISE_GP_0;
+                    }
+                }
+                else
+                {
+                    LogRel(("GIM%u: HyperV: Enabled synthetic interrupt message page at invalid address %#RGp\n", pVCpu->idCpu,
+                            GCPhysSimp));
+                }
+            }
+            else
+                LogRel(("GIM%u: HyperV: Disabled synthetic interrupt message page\n", pVCpu->idCpu));
+            return VINF_SUCCESS;
+#endif
+        }
+
+        case MSR_GIM_HV_CRASH_P0:  pHv->uCrashP0Msr = uRawValue;  return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P1:  pHv->uCrashP1Msr = uRawValue;  return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P2:  pHv->uCrashP2Msr = uRawValue;  return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P3:  pHv->uCrashP3Msr = uRawValue;  return VINF_SUCCESS;
+        case MSR_GIM_HV_CRASH_P4:  pHv->uCrashP4Msr = uRawValue;  return VINF_SUCCESS;
+
+        case MSR_GIM_HV_TIME_REF_COUNT:     /* Read-only MSRs. */
+        case MSR_GIM_HV_VP_INDEX:
+        case MSR_GIM_HV_TSC_FREQ:
+        case MSR_GIM_HV_APIC_FREQ:
+            LogFunc(("WrMsr on read-only MSR %#RX32 -> #GP(0)\n", idMsr));
+            break;
+
+        case MSR_GIM_HV_DEBUG_OPTIONS_MSR:
+        {
+            if (pHv->fIsVendorMsHv)
+            {
+#ifndef IN_RING3
+                return VINF_CPUM_R3_MSR_WRITE;
+#else
+                LogRelMax(5, ("GIM: HyperV: Write debug options MSR with %#RX64 ignored\n", uRawValue));
+                return VINF_SUCCESS;
+#endif
+            }
+            return VERR_CPUM_RAISE_GP_0;
+        }
+
+        default:
+        {
+#ifdef IN_RING3
+            static uint32_t s_cTimes = 0;
+            if (s_cTimes++ < 20)
+                LogRel(("GIM: HyperV: Unknown/invalid WrMsr (%#x,%#x`%08x) -> #GP(0)\n", idMsr,
+                        uRawValue & UINT64_C(0xffffffff00000000), uRawValue & UINT64_C(0xffffffff)));
+            LogFunc(("Unknown/invalid WrMsr (%#RX32,%#RX64) -> #GP(0)\n", idMsr, uRawValue));
+            break;
+#else
+            return VINF_CPUM_R3_MSR_WRITE;
+#endif
+        }
+    }
+
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * Whether we need to trap \#UD exceptions in the guest.
+ *
+ * We only needed to trap \#UD exceptions for the old raw-mode guests when
+ * hypercalls are enabled. For HM VMs, the hypercall would be handled via the
+ * VMCALL/VMMCALL VM-exit.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) gimHvShouldTrapXcptUD(PVMCPU pVCpu)
+{
+    RT_NOREF(pVCpu);
+    return false;
+}
+
+
+/**
+ * Checks the instruction and executes the hypercall if it's a valid hypercall
+ * instruction.
+ *
+ * This interface is used by \#UD handlers and IEM.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   uDisOpcode  The disassembler opcode.
+ * @param   cbInstr     The instruction length.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimHvHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr)
+{
+    Assert(pVCpu);
+    Assert(pCtx);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    CPUMCPUVENDOR const enmGuestCpuVendor = (CPUMCPUVENDOR)pVM->cpum.ro.GuestFeatures.enmCpuVendor;
+    if (   (   uDisOpcode == OP_VMCALL
+            && (   enmGuestCpuVendor == CPUMCPUVENDOR_INTEL
+                || enmGuestCpuVendor == CPUMCPUVENDOR_VIA
+                || enmGuestCpuVendor == CPUMCPUVENDOR_SHANGHAI))
+        || (   uDisOpcode == OP_VMMCALL
+            && (   enmGuestCpuVendor == CPUMCPUVENDOR_AMD
+                || enmGuestCpuVendor == CPUMCPUVENDOR_HYGON)) )
+        return gimHvHypercall(pVCpu, pCtx);
+
+    RT_NOREF_PV(cbInstr);
+    return VERR_GIM_INVALID_HYPERCALL_INSTR;
+}
+
+
+/**
+ * Exception handler for \#UD.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VINF_GIM_HYPERCALL_CONTINUING continue hypercall without updating
+ *          RIP.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_INVALID_HYPERCALL_INSTR instruction at RIP is not a valid
+ *          hypercall instruction.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   pDis        Pointer to the disassembled instruction state at RIP.
+ *                      Optional, can be NULL.
+ * @param   pcbInstr    Where to store the instruction length of the hypercall
+ *                      instruction. Optional, can be NULL.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimHvXcptUD(PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * If we didn't ask for #UD to be trapped, bail.
+     */
+    if (!gimHvShouldTrapXcptUD(pVCpu))
+        return VERR_GIM_IPE_1;
+
+    if (!pDis)
+    {
+        /*
+         * Disassemble the instruction at RIP to figure out if it's the Intel VMCALL instruction
+         * or the AMD VMMCALL instruction and if so, handle it as a hypercall.
+         */
+        unsigned    cbInstr;
+        DISCPUSTATE Dis;
+        int rc = EMInterpretDisasCurrent(pVCpu, &Dis, &cbInstr);
+        if (RT_SUCCESS(rc))
+        {
+            if (pcbInstr)
+                *pcbInstr = (uint8_t)cbInstr;
+            return gimHvHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr);
+        }
+
+        Log(("GIM: HyperV: Failed to disassemble instruction at CS:RIP=%04x:%08RX64. rc=%Rrc\n", pCtx->cs.Sel, pCtx->rip, rc));
+        return rc;
+    }
+
+    return gimHvHypercallEx(pVCpu, pCtx, pDis->pCurInstr->uOpcode, pDis->cbInstr);
+}
+
diff --git a/src/VBox/VMM/VMMAll/GIMAllKvm.cpp b/src/VBox/VMM/VMMAll/GIMAllKvm.cpp
new file mode 100644
index 00000000..91d858ee
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/GIMAllKvm.cpp
@@ -0,0 +1,450 @@
+/* $Id: GIMAllKvm.cpp $ */
+/** @file
+ * GIM - Guest Interface Manager, KVM, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2015-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_GIM
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/pdmapi.h>
+#include "GIMKvmInternal.h"
+#include "GIMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/dis.h>
+#include <VBox/err.h>
+#include <VBox/sup.h>
+
+#include <iprt/time.h>
+
+
+/**
+ * Handles the KVM hypercall.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Pointer to the guest-CPU context.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimKvmHypercall(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    STAM_REL_COUNTER_INC(&pVM->gim.s.StatHypercalls);
+
+    /*
+     * Get the hypercall operation and arguments.
+     */
+    bool const fIs64BitMode = CPUMIsGuestIn64BitCodeEx(pCtx);
+    uint64_t uHyperOp       = pCtx->rax;
+    uint64_t uHyperArg0     = pCtx->rbx;
+    uint64_t uHyperArg1     = pCtx->rcx;
+    uint64_t uHyperArg2     = pCtx->rdi;
+    uint64_t uHyperArg3     = pCtx->rsi;
+    uint64_t uHyperRet      = KVM_HYPERCALL_RET_ENOSYS;
+    uint64_t uAndMask       = UINT64_C(0xffffffffffffffff);
+    if (!fIs64BitMode)
+    {
+        uAndMask    = UINT64_C(0xffffffff);
+        uHyperOp   &= UINT64_C(0xffffffff);
+        uHyperArg0 &= UINT64_C(0xffffffff);
+        uHyperArg1 &= UINT64_C(0xffffffff);
+        uHyperArg2 &= UINT64_C(0xffffffff);
+        uHyperArg3 &= UINT64_C(0xffffffff);
+        uHyperRet  &= UINT64_C(0xffffffff);
+    }
+
+    /*
+     * Verify that guest ring-0 is the one making the hypercall.
+     */
+    uint32_t uCpl = CPUMGetGuestCPL(pVCpu);
+    if (RT_UNLIKELY(uCpl))
+    {
+        pCtx->rax = KVM_HYPERCALL_RET_EPERM & uAndMask;
+        return VERR_GIM_HYPERCALL_ACCESS_DENIED;
+    }
+
+    /*
+     * Do the work.
+     */
+    int rc = VINF_SUCCESS;
+    switch (uHyperOp)
+    {
+        case KVM_HYPERCALL_OP_KICK_CPU:
+        {
+            if (uHyperArg1 < pVM->cCpus)
+            {
+                PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, uHyperArg1); /* ASSUMES pVCpu index == ApicId of the VCPU. */
+                EMUnhaltAndWakeUp(pVM, pVCpuDst);
+                uHyperRet = KVM_HYPERCALL_RET_SUCCESS;
+            }
+            else
+            {
+                /* Shouldn't ever happen! If it does, throw a guru, as otherwise it'll lead to deadlocks in the guest anyway! */
+                rc = VERR_GIM_HYPERCALL_FAILED;
+            }
+            break;
+        }
+
+        case KVM_HYPERCALL_OP_VAPIC_POLL_IRQ:
+            uHyperRet = KVM_HYPERCALL_RET_SUCCESS;
+            break;
+
+        default:
+            break;
+    }
+
+    /*
+     * Place the result in rax/eax.
+     */
+    pCtx->rax = uHyperRet & uAndMask;
+    return rc;
+}
+
+
+/**
+ * Returns whether the guest has configured and enabled the use of KVM's
+ * hypercall interface.
+ *
+ * @returns true if hypercalls are enabled, false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) gimKvmAreHypercallsEnabled(PVMCPU pVCpu)
+{
+    NOREF(pVCpu);
+    /* KVM paravirt interface doesn't have hypercall control bits (like Hyper-V does)
+       that guests can control, i.e. hypercalls are always enabled. */
+    return true;
+}
+
+
+/**
+ * Returns whether the guest has configured and enabled the use of KVM's
+ * paravirtualized TSC.
+ *
+ * @returns true if paravirt. TSC is enabled, false otherwise.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(bool) gimKvmIsParavirtTscEnabled(PVMCC pVM)
+{
+    uint32_t const cCpus = pVM->cCpus;
+    for (uint32_t idCpu = 0; idCpu < cCpus; idCpu++)
+    {
+        PVMCPUCC   pVCpu      = pVM->CTX_SUFF(apCpus)[idCpu];
+        PGIMKVMCPU pGimKvmCpu = &pVCpu->gim.s.u.KvmCpu;
+        if (MSR_GIM_KVM_SYSTEM_TIME_IS_ENABLED(pGimKvmCpu->u64SystemTimeMsr))
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * MSR read handler for KVM.
+ *
+ * @returns Strict VBox status code like CPUMQueryGuestMsr().
+ * @retval  VINF_CPUM_R3_MSR_READ
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR being read.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   puValue     Where to store the MSR value read.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimKvmReadMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t *puValue)
+{
+    NOREF(pRange);
+    PVM     pVM        = pVCpu->CTX_SUFF(pVM);
+    PGIMKVM pKvm       = &pVM->gim.s.u.Kvm;
+    PGIMKVMCPU pKvmCpu = &pVCpu->gim.s.u.KvmCpu;
+
+    switch (idMsr)
+    {
+        case MSR_GIM_KVM_SYSTEM_TIME:
+        case MSR_GIM_KVM_SYSTEM_TIME_OLD:
+            *puValue = pKvmCpu->u64SystemTimeMsr;
+            return VINF_SUCCESS;
+
+        case MSR_GIM_KVM_WALL_CLOCK:
+        case MSR_GIM_KVM_WALL_CLOCK_OLD:
+            *puValue = pKvm->u64WallClockMsr;
+            return VINF_SUCCESS;
+
+        default:
+        {
+#ifdef IN_RING3
+            static uint32_t s_cTimes = 0;
+            if (s_cTimes++ < 20)
+                LogRel(("GIM: KVM: Unknown/invalid RdMsr (%#x) -> #GP(0)\n", idMsr));
+#endif
+            LogFunc(("Unknown/invalid RdMsr (%#RX32) -> #GP(0)\n", idMsr));
+            break;
+        }
+    }
+
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * MSR write handler for KVM.
+ *
+ * @returns Strict VBox status code like CPUMSetGuestMsr().
+ * @retval  VINF_CPUM_R3_MSR_WRITE
+ * @retval  VERR_CPUM_RAISE_GP_0
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR being written.
+ * @param   pRange      The range this MSR belongs to.
+ * @param   uRawValue   The raw value with the ignored bits not masked.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimKvmWriteMsr(PVMCPUCC pVCpu, uint32_t idMsr, PCCPUMMSRRANGE pRange, uint64_t uRawValue)
+{
+    NOREF(pRange);
+    switch (idMsr)
+    {
+        case MSR_GIM_KVM_SYSTEM_TIME:
+        case MSR_GIM_KVM_SYSTEM_TIME_OLD:
+        {
+#ifndef IN_RING3
+            RT_NOREF2(pVCpu, uRawValue);
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            PVMCC      pVM = pVCpu->CTX_SUFF(pVM);
+            PGIMKVMCPU pKvmCpu = &pVCpu->gim.s.u.KvmCpu;
+            if (uRawValue & MSR_GIM_KVM_SYSTEM_TIME_ENABLE_BIT)
+                gimR3KvmEnableSystemTime(pVM, pVCpu, uRawValue);
+            else
+                gimR3KvmDisableSystemTime(pVM);
+
+            pKvmCpu->u64SystemTimeMsr = uRawValue;
+            return VINF_SUCCESS;
+#endif /* IN_RING3 */
+        }
+
+        case MSR_GIM_KVM_WALL_CLOCK:
+        case MSR_GIM_KVM_WALL_CLOCK_OLD:
+        {
+#ifndef IN_RING3
+            RT_NOREF2(pVCpu, uRawValue);
+            return VINF_CPUM_R3_MSR_WRITE;
+#else
+            /* Enable the wall-clock struct. */
+            RTGCPHYS GCPhysWallClock = MSR_GIM_KVM_WALL_CLOCK_GUEST_GPA(uRawValue);
+            if (RT_LIKELY(RT_ALIGN_64(GCPhysWallClock, 4) == GCPhysWallClock))
+            {
+                PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+                int rc = gimR3KvmEnableWallClock(pVM, GCPhysWallClock);
+                if (RT_SUCCESS(rc))
+                {
+                    PGIMKVM pKvm = &pVM->gim.s.u.Kvm;
+                    pKvm->u64WallClockMsr = uRawValue;
+                    return VINF_SUCCESS;
+                }
+            }
+            return VERR_CPUM_RAISE_GP_0;
+#endif /* IN_RING3 */
+        }
+
+        default:
+        {
+#ifdef IN_RING3
+            static uint32_t s_cTimes = 0;
+            if (s_cTimes++ < 20)
+                LogRel(("GIM: KVM: Unknown/invalid WrMsr (%#x,%#x`%08x) -> #GP(0)\n", idMsr,
+                        uRawValue & UINT64_C(0xffffffff00000000), uRawValue & UINT64_C(0xffffffff)));
+#endif
+            LogFunc(("Unknown/invalid WrMsr (%#RX32,%#RX64) -> #GP(0)\n", idMsr, uRawValue));
+            break;
+        }
+    }
+
+    return VERR_CPUM_RAISE_GP_0;
+}
+
+
+/**
+ * Whether we need to trap \#UD exceptions in the guest.
+ *
+ * On AMD-V we need to trap them because paravirtualized Linux/KVM guests use
+ * the Intel VMCALL instruction to make hypercalls and we need to trap and
+ * optionally patch them to the AMD-V VMMCALL instruction and handle the
+ * hypercall.
+ *
+ * I guess this was done so that guest teleporation between an AMD and an Intel
+ * machine would working without any changes at the time of teleporation.
+ * However, this also means we -always- need to intercept \#UD exceptions on one
+ * of the two CPU models (Intel or AMD). Hyper-V solves this problem more
+ * elegantly by letting the hypervisor supply an opaque hypercall page.
+ *
+ * For raw-mode VMs, this function will always return true. See gimR3KvmInit().
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(bool) gimKvmShouldTrapXcptUD(PVM pVM)
+{
+    return pVM->gim.s.u.Kvm.fTrapXcptUD;
+}
+
+
+/**
+ * Checks the instruction and executes the hypercall if it's a valid hypercall
+ * instruction.
+ *
+ * This interface is used by \#UD handlers and IEM.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   uDisOpcode  The disassembler opcode.
+ * @param   cbInstr     The instruction length.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimKvmHypercallEx(PVMCPUCC pVCpu, PCPUMCTX pCtx, unsigned uDisOpcode, uint8_t cbInstr)
+{
+    Assert(pVCpu);
+    Assert(pCtx);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * If the instruction at RIP is the Intel VMCALL instruction or
+     * the AMD VMMCALL instruction handle it as a hypercall.
+     *
+     * Linux/KVM guests always uses the Intel VMCALL instruction but we patch
+     * it to the host-native one whenever we encounter it so subsequent calls
+     * will not require disassembly (when coming from HM).
+     */
+    if (   uDisOpcode == OP_VMCALL
+        || uDisOpcode == OP_VMMCALL)
+    {
+        /*
+         * Perform the hypercall.
+         *
+         * For HM, we can simply resume guest execution without performing the hypercall now and
+         * do it on the next VMCALL/VMMCALL exit handler on the patched instruction.
+         *
+         * For raw-mode we need to do this now anyway. So we do it here regardless with an added
+         * advantage is that it saves one world-switch for the HM case.
+         */
+        VBOXSTRICTRC rcStrict = gimKvmHypercall(pVCpu, pCtx);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /*
+             * Patch the instruction to so we don't have to spend time disassembling it each time.
+             * Makes sense only for HM as with raw-mode we will be getting a #UD regardless.
+             */
+            PVM      pVM  = pVCpu->CTX_SUFF(pVM);
+            PCGIMKVM pKvm = &pVM->gim.s.u.Kvm;
+            if (   uDisOpcode != pKvm->uOpcodeNative
+                && cbInstr == sizeof(pKvm->abOpcodeNative) )
+            {
+                /** @todo r=ramshankar: we probably should be doing this in an
+                 *        EMT rendezvous. */
+                /** @todo Add stats for patching. */
+                int rc = PGMPhysSimpleWriteGCPtr(pVCpu, pCtx->rip, pKvm->abOpcodeNative, sizeof(pKvm->abOpcodeNative));
+                AssertRC(rc);
+            }
+        }
+        else
+        {
+            /* The KVM provider doesn't have any concept of continuing hypercalls. */
+            Assert(rcStrict != VINF_GIM_HYPERCALL_CONTINUING);
+#ifdef IN_RING3
+            Assert(rcStrict != VINF_GIM_R3_HYPERCALL);
+#endif
+        }
+        return rcStrict;
+    }
+
+    return VERR_GIM_INVALID_HYPERCALL_INSTR;
+}
+
+
+/**
+ * Exception handler for \#UD.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS if the hypercall succeeded (even if its operation
+ *          failed).
+ * @retval  VINF_GIM_R3_HYPERCALL re-start the hypercall from ring-3.
+ * @retval  VERR_GIM_HYPERCALL_ACCESS_DENIED CPL is insufficient.
+ * @retval  VERR_GIM_INVALID_HYPERCALL_INSTR instruction at RIP is not a valid
+ *          hypercall instruction.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the guest-CPU context.
+ * @param   pDis        Pointer to the disassembled instruction state at RIP.
+ *                      Optional, can be NULL.
+ * @param   pcbInstr    Where to store the instruction length of the hypercall
+ *                      instruction. Optional, can be NULL.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) gimKvmXcptUD(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, uint8_t *pcbInstr)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * If we didn't ask for #UD to be trapped, bail.
+     */
+    if (RT_UNLIKELY(!pVM->gim.s.u.Kvm.fTrapXcptUD))
+        return VERR_GIM_IPE_3;
+
+    if (!pDis)
+    {
+        unsigned    cbInstr;
+        DISCPUSTATE Dis;
+        int rc = EMInterpretDisasCurrent(pVCpu, &Dis, &cbInstr);
+        if (RT_SUCCESS(rc))
+        {
+            if (pcbInstr)
+                *pcbInstr = (uint8_t)cbInstr;
+            return gimKvmHypercallEx(pVCpu, pCtx, Dis.pCurInstr->uOpcode, Dis.cbInstr);
+        }
+
+        Log(("GIM: KVM: Failed to disassemble instruction at CS:RIP=%04x:%08RX64. rc=%Rrc\n", pCtx->cs.Sel, pCtx->rip, rc));
+        return rc;
+    }
+
+    return gimKvmHypercallEx(pVCpu, pCtx, pDis->pCurInstr->uOpcode, pDis->cbInstr);
+}
+
diff --git a/src/VBox/VMM/VMMAll/HMAll.cpp b/src/VBox/VMM/VMMAll/HMAll.cpp
new file mode 100644
index 00000000..1d2f4170
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/HMAll.cpp
@@ -0,0 +1,904 @@
+/* $Id: HMAll.cpp $ */
+/** @file
+ * HM - All contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_HM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pgm.h>
+#include "HMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/hm_vmx.h>
+#include <VBox/vmm/hm_svm.h>
+#include <iprt/errcore.h>
+#include <VBox/log.h>
+#include <iprt/param.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+#include <iprt/x86.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#define EXIT_REASON(a_Def, a_Val, a_Str)      #a_Def " - " #a_Val " - " a_Str
+#define EXIT_REASON_NIL()                     NULL
+
+/** Exit reason descriptions for VT-x, used to describe statistics and exit
+ *  history. */
+static const char * const g_apszVmxExitReasons[MAX_EXITREASON_STAT] =
+{
+    EXIT_REASON(VMX_EXIT_XCPT_OR_NMI            ,   0, "Exception or non-maskable interrupt (NMI)."),
+    EXIT_REASON(VMX_EXIT_EXT_INT                ,   1, "External interrupt."),
+    EXIT_REASON(VMX_EXIT_TRIPLE_FAULT           ,   2, "Triple fault."),
+    EXIT_REASON(VMX_EXIT_INIT_SIGNAL            ,   3, "INIT signal."),
+    EXIT_REASON(VMX_EXIT_SIPI                   ,   4, "Start-up IPI (SIPI)."),
+    EXIT_REASON(VMX_EXIT_IO_SMI_IRQ             ,   5, "I/O system-management interrupt (SMI)."),
+    EXIT_REASON(VMX_EXIT_SMI_IRQ                ,   6, "Other SMI."),
+    EXIT_REASON(VMX_EXIT_INT_WINDOW             ,   7, "Interrupt window."),
+    EXIT_REASON(VMX_EXIT_NMI_WINDOW             ,   8, "NMI window."),
+    EXIT_REASON(VMX_EXIT_TASK_SWITCH            ,   9, "Task switch."),
+    EXIT_REASON(VMX_EXIT_CPUID                  ,  10, "CPUID instruction."),
+    EXIT_REASON(VMX_EXIT_GETSEC                 ,  11, "GETSEC instruction."),
+    EXIT_REASON(VMX_EXIT_HLT                    ,  12, "HLT instruction."),
+    EXIT_REASON(VMX_EXIT_INVD                   ,  13, "INVD instruction."),
+    EXIT_REASON(VMX_EXIT_INVLPG                 ,  14, "INVLPG instruction."),
+    EXIT_REASON(VMX_EXIT_RDPMC                  ,  15, "RDPMC instruction."),
+    EXIT_REASON(VMX_EXIT_RDTSC                  ,  16, "RDTSC instruction."),
+    EXIT_REASON(VMX_EXIT_RSM                    ,  17, "RSM instruction in SMM."),
+    EXIT_REASON(VMX_EXIT_VMCALL                 ,  18, "VMCALL instruction."),
+    EXIT_REASON(VMX_EXIT_VMCLEAR                ,  19, "VMCLEAR instruction."),
+    EXIT_REASON(VMX_EXIT_VMLAUNCH               ,  20, "VMLAUNCH instruction."),
+    EXIT_REASON(VMX_EXIT_VMPTRLD                ,  21, "VMPTRLD instruction."),
+    EXIT_REASON(VMX_EXIT_VMPTRST                ,  22, "VMPTRST instruction."),
+    EXIT_REASON(VMX_EXIT_VMREAD                 ,  23, "VMREAD instruction."),
+    EXIT_REASON(VMX_EXIT_VMRESUME               ,  24, "VMRESUME instruction."),
+    EXIT_REASON(VMX_EXIT_VMWRITE                ,  25, "VMWRITE instruction."),
+    EXIT_REASON(VMX_EXIT_VMXOFF                 ,  26, "VMXOFF instruction."),
+    EXIT_REASON(VMX_EXIT_VMXON                  ,  27, "VMXON instruction."),
+    EXIT_REASON(VMX_EXIT_MOV_CRX                ,  28, "Control-register accesses."),
+    EXIT_REASON(VMX_EXIT_MOV_DRX                ,  29, "Debug-register accesses."),
+    EXIT_REASON(VMX_EXIT_PORT_IO                ,  30, "I/O instruction."),
+    EXIT_REASON(VMX_EXIT_RDMSR                  ,  31, "RDMSR instruction."),
+    EXIT_REASON(VMX_EXIT_WRMSR                  ,  32, "WRMSR instruction."),
+    EXIT_REASON(VMX_EXIT_ERR_INVALID_GUEST_STATE,  33, "VM-entry failure due to invalid guest state."),
+    EXIT_REASON(VMX_EXIT_ERR_MSR_LOAD           ,  34, "VM-entry failure due to MSR loading."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_MWAIT                  ,  36, "MWAIT instruction."),
+    EXIT_REASON(VMX_EXIT_MTF                    ,  37, "Monitor Trap Flag."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_MONITOR                ,  39, "MONITOR instruction."),
+    EXIT_REASON(VMX_EXIT_PAUSE                  ,  40, "PAUSE instruction."),
+    EXIT_REASON(VMX_EXIT_ERR_MACHINE_CHECK      ,  41, "VM-entry failure due to machine-check."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_TPR_BELOW_THRESHOLD    ,  43, "TPR below threshold (MOV to CR8)."),
+    EXIT_REASON(VMX_EXIT_APIC_ACCESS            ,  44, "APIC access."),
+    EXIT_REASON(VMX_EXIT_VIRTUALIZED_EOI        ,  45, "Virtualized EOI."),
+    EXIT_REASON(VMX_EXIT_GDTR_IDTR_ACCESS       ,  46, "GDTR/IDTR access using LGDT/SGDT/LIDT/SIDT."),
+    EXIT_REASON(VMX_EXIT_LDTR_TR_ACCESS         ,  47, "LDTR/TR access using LLDT/SLDT/LTR/STR."),
+    EXIT_REASON(VMX_EXIT_EPT_VIOLATION          ,  48, "EPT violation."),
+    EXIT_REASON(VMX_EXIT_EPT_MISCONFIG          ,  49, "EPT misconfiguration."),
+    EXIT_REASON(VMX_EXIT_INVEPT                 ,  50, "INVEPT instruction."),
+    EXIT_REASON(VMX_EXIT_RDTSCP                 ,  51, "RDTSCP instruction."),
+    EXIT_REASON(VMX_EXIT_PREEMPT_TIMER          ,  52, "VMX-preemption timer expired."),
+    EXIT_REASON(VMX_EXIT_INVVPID                ,  53, "INVVPID instruction."),
+    EXIT_REASON(VMX_EXIT_WBINVD                 ,  54, "WBINVD instruction."),
+    EXIT_REASON(VMX_EXIT_XSETBV                 ,  55, "XSETBV instruction."),
+    EXIT_REASON(VMX_EXIT_APIC_WRITE             ,  56, "APIC write completed to virtual-APIC page."),
+    EXIT_REASON(VMX_EXIT_RDRAND                 ,  57, "RDRAND instruction."),
+    EXIT_REASON(VMX_EXIT_INVPCID                ,  58, "INVPCID instruction."),
+    EXIT_REASON(VMX_EXIT_VMFUNC                 ,  59, "VMFUNC instruction."),
+    EXIT_REASON(VMX_EXIT_ENCLS                  ,  60, "ENCLS instruction."),
+    EXIT_REASON(VMX_EXIT_RDSEED                 ,  61, "RDSEED instruction."),
+    EXIT_REASON(VMX_EXIT_PML_FULL               ,  62, "Page-modification log full."),
+    EXIT_REASON(VMX_EXIT_XSAVES                 ,  63, "XSAVES instruction."),
+    EXIT_REASON(VMX_EXIT_XRSTORS                ,  64, "XRSTORS instruction."),
+    EXIT_REASON_NIL(),
+    EXIT_REASON(VMX_EXIT_SPP_EVENT              ,  66, "SPP-related event."),
+    EXIT_REASON(VMX_EXIT_UMWAIT                 ,  67, "UMWAIT instruction."),
+    EXIT_REASON(VMX_EXIT_TPAUSE                 ,  68, "TPAUSE instruction.")
+};
+/** Array index of the last valid VT-x exit reason. */
+#define MAX_EXITREASON_VTX                         68
+
+/** A partial list of \#EXIT reason descriptions for AMD-V, used to describe
+ *  statistics and exit history.
+ *
+ *  @note AMD-V have annoyingly large gaps (e.g. \#NPF VMEXIT comes at 1024),
+ *        this array doesn't contain the entire set of exit reasons, we
+ *        handle them via hmSvmGetSpecialExitReasonDesc(). */
+static const char * const g_apszSvmExitReasons[MAX_EXITREASON_STAT] =
+{
+    EXIT_REASON(SVM_EXIT_READ_CR0       ,    0, "Read CR0."),
+    EXIT_REASON(SVM_EXIT_READ_CR1       ,    1, "Read CR1."),
+    EXIT_REASON(SVM_EXIT_READ_CR2       ,    2, "Read CR2."),
+    EXIT_REASON(SVM_EXIT_READ_CR3       ,    3, "Read CR3."),
+    EXIT_REASON(SVM_EXIT_READ_CR4       ,    4, "Read CR4."),
+    EXIT_REASON(SVM_EXIT_READ_CR5       ,    5, "Read CR5."),
+    EXIT_REASON(SVM_EXIT_READ_CR6       ,    6, "Read CR6."),
+    EXIT_REASON(SVM_EXIT_READ_CR7       ,    7, "Read CR7."),
+    EXIT_REASON(SVM_EXIT_READ_CR8       ,    8, "Read CR8."),
+    EXIT_REASON(SVM_EXIT_READ_CR9       ,    9, "Read CR9."),
+    EXIT_REASON(SVM_EXIT_READ_CR10      ,   10, "Read CR10."),
+    EXIT_REASON(SVM_EXIT_READ_CR11      ,   11, "Read CR11."),
+    EXIT_REASON(SVM_EXIT_READ_CR12      ,   12, "Read CR12."),
+    EXIT_REASON(SVM_EXIT_READ_CR13      ,   13, "Read CR13."),
+    EXIT_REASON(SVM_EXIT_READ_CR14      ,   14, "Read CR14."),
+    EXIT_REASON(SVM_EXIT_READ_CR15      ,   15, "Read CR15."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR0      ,   16, "Write CR0."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR1      ,   17, "Write CR1."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR2      ,   18, "Write CR2."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR3      ,   19, "Write CR3."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR4      ,   20, "Write CR4."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR5      ,   21, "Write CR5."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR6      ,   22, "Write CR6."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR7      ,   23, "Write CR7."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR8      ,   24, "Write CR8."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR9      ,   25, "Write CR9."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR10     ,   26, "Write CR10."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR11     ,   27, "Write CR11."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR12     ,   28, "Write CR12."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR13     ,   29, "Write CR13."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR14     ,   30, "Write CR14."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR15     ,   31, "Write CR15."),
+    EXIT_REASON(SVM_EXIT_READ_DR0       ,   32, "Read DR0."),
+    EXIT_REASON(SVM_EXIT_READ_DR1       ,   33, "Read DR1."),
+    EXIT_REASON(SVM_EXIT_READ_DR2       ,   34, "Read DR2."),
+    EXIT_REASON(SVM_EXIT_READ_DR3       ,   35, "Read DR3."),
+    EXIT_REASON(SVM_EXIT_READ_DR4       ,   36, "Read DR4."),
+    EXIT_REASON(SVM_EXIT_READ_DR5       ,   37, "Read DR5."),
+    EXIT_REASON(SVM_EXIT_READ_DR6       ,   38, "Read DR6."),
+    EXIT_REASON(SVM_EXIT_READ_DR7       ,   39, "Read DR7."),
+    EXIT_REASON(SVM_EXIT_READ_DR8       ,   40, "Read DR8."),
+    EXIT_REASON(SVM_EXIT_READ_DR9       ,   41, "Read DR9."),
+    EXIT_REASON(SVM_EXIT_READ_DR10      ,   42, "Read DR10."),
+    EXIT_REASON(SVM_EXIT_READ_DR11      ,   43, "Read DR11"),
+    EXIT_REASON(SVM_EXIT_READ_DR12      ,   44, "Read DR12."),
+    EXIT_REASON(SVM_EXIT_READ_DR13      ,   45, "Read DR13."),
+    EXIT_REASON(SVM_EXIT_READ_DR14      ,   46, "Read DR14."),
+    EXIT_REASON(SVM_EXIT_READ_DR15      ,   47, "Read DR15."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR0      ,   48, "Write DR0."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR1      ,   49, "Write DR1."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR2      ,   50, "Write DR2."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR3      ,   51, "Write DR3."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR4      ,   52, "Write DR4."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR5      ,   53, "Write DR5."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR6      ,   54, "Write DR6."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR7      ,   55, "Write DR7."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR8      ,   56, "Write DR8."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR9      ,   57, "Write DR9."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR10     ,   58, "Write DR10."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR11     ,   59, "Write DR11."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR12     ,   60, "Write DR12."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR13     ,   61, "Write DR13."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR14     ,   62, "Write DR14."),
+    EXIT_REASON(SVM_EXIT_WRITE_DR15     ,   63, "Write DR15."),
+    EXIT_REASON(SVM_EXIT_XCPT_0         ,   64, "Exception 0  (#DE)."),
+    EXIT_REASON(SVM_EXIT_XCPT_1         ,   65, "Exception 1  (#DB)."),
+    EXIT_REASON(SVM_EXIT_XCPT_2         ,   66, "Exception 2  (#NMI)."),
+    EXIT_REASON(SVM_EXIT_XCPT_3         ,   67, "Exception 3  (#BP)."),
+    EXIT_REASON(SVM_EXIT_XCPT_4         ,   68, "Exception 4  (#OF)."),
+    EXIT_REASON(SVM_EXIT_XCPT_5         ,   69, "Exception 5  (#BR)."),
+    EXIT_REASON(SVM_EXIT_XCPT_6         ,   70, "Exception 6  (#UD)."),
+    EXIT_REASON(SVM_EXIT_XCPT_7         ,   71, "Exception 7  (#NM)."),
+    EXIT_REASON(SVM_EXIT_XCPT_8         ,   72, "Exception 8  (#DF)."),
+    EXIT_REASON(SVM_EXIT_XCPT_9         ,   73, "Exception 9  (#CO_SEG_OVERRUN)."),
+    EXIT_REASON(SVM_EXIT_XCPT_10        ,   74, "Exception 10 (#TS)."),
+    EXIT_REASON(SVM_EXIT_XCPT_11        ,   75, "Exception 11 (#NP)."),
+    EXIT_REASON(SVM_EXIT_XCPT_12        ,   76, "Exception 12 (#SS)."),
+    EXIT_REASON(SVM_EXIT_XCPT_13        ,   77, "Exception 13 (#GP)."),
+    EXIT_REASON(SVM_EXIT_XCPT_14        ,   78, "Exception 14 (#PF)."),
+    EXIT_REASON(SVM_EXIT_XCPT_15        ,   79, "Exception 15 (0x0f)."),
+    EXIT_REASON(SVM_EXIT_XCPT_16        ,   80, "Exception 16 (#MF)."),
+    EXIT_REASON(SVM_EXIT_XCPT_17        ,   81, "Exception 17 (#AC)."),
+    EXIT_REASON(SVM_EXIT_XCPT_18        ,   82, "Exception 18 (#MC)."),
+    EXIT_REASON(SVM_EXIT_XCPT_19        ,   83, "Exception 19 (#XF)."),
+    EXIT_REASON(SVM_EXIT_XCPT_20        ,   84, "Exception 20 (#VE)."),
+    EXIT_REASON(SVM_EXIT_XCPT_21        ,   85, "Exception 22 (0x15)."),
+    EXIT_REASON(SVM_EXIT_XCPT_22        ,   86, "Exception 22 (0x16)."),
+    EXIT_REASON(SVM_EXIT_XCPT_23        ,   87, "Exception 23 (0x17)."),
+    EXIT_REASON(SVM_EXIT_XCPT_24        ,   88, "Exception 24 (0x18)."),
+    EXIT_REASON(SVM_EXIT_XCPT_25        ,   89, "Exception 25 (0x19)."),
+    EXIT_REASON(SVM_EXIT_XCPT_26        ,   90, "Exception 26 (0x1a)."),
+    EXIT_REASON(SVM_EXIT_XCPT_27        ,   91, "Exception 27 (0x1b)."),
+    EXIT_REASON(SVM_EXIT_XCPT_28        ,   92, "Exception 28 (0x1c)."),
+    EXIT_REASON(SVM_EXIT_XCPT_29        ,   93, "Exception 29 (0x1d)."),
+    EXIT_REASON(SVM_EXIT_XCPT_30        ,   94, "Exception 30 (#SX)."),
+    EXIT_REASON(SVM_EXIT_XCPT_31        ,   95, "Exception 31 (0x1F)."),
+    EXIT_REASON(SVM_EXIT_INTR           ,   96, "Physical maskable interrupt (host)."),
+    EXIT_REASON(SVM_EXIT_NMI            ,   97, "Physical non-maskable interrupt (host)."),
+    EXIT_REASON(SVM_EXIT_SMI            ,   98, "System management interrupt (host)."),
+    EXIT_REASON(SVM_EXIT_INIT           ,   99, "Physical INIT signal (host)."),
+    EXIT_REASON(SVM_EXIT_VINTR          ,  100, "Virtual interrupt-window exit."),
+    EXIT_REASON(SVM_EXIT_CR0_SEL_WRITE  ,  101, "Selective CR0 Write (to bits other than CR0.TS and CR0.MP)."),
+    EXIT_REASON(SVM_EXIT_IDTR_READ      ,  102, "Read IDTR."),
+    EXIT_REASON(SVM_EXIT_GDTR_READ      ,  103, "Read GDTR."),
+    EXIT_REASON(SVM_EXIT_LDTR_READ      ,  104, "Read LDTR."),
+    EXIT_REASON(SVM_EXIT_TR_READ        ,  105, "Read TR."),
+    EXIT_REASON(SVM_EXIT_IDTR_WRITE     ,  106, "Write IDTR."),
+    EXIT_REASON(SVM_EXIT_GDTR_WRITE     ,  107, "Write GDTR."),
+    EXIT_REASON(SVM_EXIT_LDTR_WRITE     ,  108, "Write LDTR."),
+    EXIT_REASON(SVM_EXIT_TR_WRITE       ,  109, "Write TR."),
+    EXIT_REASON(SVM_EXIT_RDTSC          ,  110, "RDTSC instruction."),
+    EXIT_REASON(SVM_EXIT_RDPMC          ,  111, "RDPMC instruction."),
+    EXIT_REASON(SVM_EXIT_PUSHF          ,  112, "PUSHF instruction."),
+    EXIT_REASON(SVM_EXIT_POPF           ,  113, "POPF instruction."),
+    EXIT_REASON(SVM_EXIT_CPUID          ,  114, "CPUID instruction."),
+    EXIT_REASON(SVM_EXIT_RSM            ,  115, "RSM instruction."),
+    EXIT_REASON(SVM_EXIT_IRET           ,  116, "IRET instruction."),
+    EXIT_REASON(SVM_EXIT_SWINT          ,  117, "Software interrupt (INTn instructions)."),
+    EXIT_REASON(SVM_EXIT_INVD           ,  118, "INVD instruction."),
+    EXIT_REASON(SVM_EXIT_PAUSE          ,  119, "PAUSE instruction."),
+    EXIT_REASON(SVM_EXIT_HLT            ,  120, "HLT instruction."),
+    EXIT_REASON(SVM_EXIT_INVLPG         ,  121, "INVLPG instruction."),
+    EXIT_REASON(SVM_EXIT_INVLPGA        ,  122, "INVLPGA instruction."),
+    EXIT_REASON(SVM_EXIT_IOIO           ,  123, "IN/OUT/INS/OUTS instruction."),
+    EXIT_REASON(SVM_EXIT_MSR            ,  124, "RDMSR or WRMSR access to protected MSR."),
+    EXIT_REASON(SVM_EXIT_TASK_SWITCH    ,  125, "Task switch."),
+    EXIT_REASON(SVM_EXIT_FERR_FREEZE    ,  126, "FERR Freeze; CPU frozen in an x87/mmx instruction waiting for interrupt."),
+    EXIT_REASON(SVM_EXIT_SHUTDOWN       ,  127, "Shutdown."),
+    EXIT_REASON(SVM_EXIT_VMRUN          ,  128, "VMRUN instruction."),
+    EXIT_REASON(SVM_EXIT_VMMCALL        ,  129, "VMCALL instruction."),
+    EXIT_REASON(SVM_EXIT_VMLOAD         ,  130, "VMLOAD instruction."),
+    EXIT_REASON(SVM_EXIT_VMSAVE         ,  131, "VMSAVE instruction."),
+    EXIT_REASON(SVM_EXIT_STGI           ,  132, "STGI instruction."),
+    EXIT_REASON(SVM_EXIT_CLGI           ,  133, "CLGI instruction."),
+    EXIT_REASON(SVM_EXIT_SKINIT         ,  134, "SKINIT instruction."),
+    EXIT_REASON(SVM_EXIT_RDTSCP         ,  135, "RDTSCP instruction."),
+    EXIT_REASON(SVM_EXIT_ICEBP          ,  136, "ICEBP instruction."),
+    EXIT_REASON(SVM_EXIT_WBINVD         ,  137, "WBINVD instruction."),
+    EXIT_REASON(SVM_EXIT_MONITOR        ,  138, "MONITOR instruction."),
+    EXIT_REASON(SVM_EXIT_MWAIT          ,  139, "MWAIT instruction."),
+    EXIT_REASON(SVM_EXIT_MWAIT_ARMED    ,  140, "MWAIT instruction when armed."),
+    EXIT_REASON(SVM_EXIT_XSETBV         ,  141, "XSETBV instruction."),
+    EXIT_REASON(SVM_EXIT_RDPRU          ,  142, "RDPRU instruction."),
+    EXIT_REASON(SVM_EXIT_WRITE_EFER_TRAP,  143, "Write EFER (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR0_TRAP ,  144, "Write CR0 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR1_TRAP ,  145, "Write CR1 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR2_TRAP ,  146, "Write CR2 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR3_TRAP ,  147, "Write CR3 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR4_TRAP ,  148, "Write CR4 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR5_TRAP ,  149, "Write CR5 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR6_TRAP ,  150, "Write CR6 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR7_TRAP ,  151, "Write CR7 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR8_TRAP ,  152, "Write CR8 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR9_TRAP ,  153, "Write CR9 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR10_TRAP,  154, "Write CR10 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR11_TRAP,  155, "Write CR11 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR12_TRAP,  156, "Write CR12 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR13_TRAP,  157, "Write CR13 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR14_TRAP,  158, "Write CR14 (trap-like)."),
+    EXIT_REASON(SVM_EXIT_WRITE_CR15_TRAP,  159, "Write CR15 (trap-like)."),
+    EXIT_REASON_NIL()                   ,
+    EXIT_REASON_NIL()                   ,
+    EXIT_REASON_NIL()                   ,
+    EXIT_REASON(SVM_EXIT_MCOMMIT        ,  163, "MCOMMIT instruction."),
+};
+/** Array index of the last valid AMD-V exit reason. */
+#define MAX_EXITREASON_AMDV              163
+
+/** Special exit reasons not covered in the array above. */
+#define SVM_EXIT_REASON_NPF                  EXIT_REASON(SVM_EXIT_NPF                , 1024, "Nested Page Fault.")
+#define SVM_EXIT_REASON_AVIC_INCOMPLETE_IPI  EXIT_REASON(SVM_EXIT_AVIC_INCOMPLETE_IPI, 1025, "AVIC - Incomplete IPI delivery.")
+#define SVM_EXIT_REASON_AVIC_NOACCEL         EXIT_REASON(SVM_EXIT_AVIC_NOACCEL       , 1026, "AVIC - Unhandled register.")
+
+/**
+ * Gets the SVM exit reason if it's one of the reasons not present in the @c
+ * g_apszSvmExitReasons array.
+ *
+ * @returns The exit reason or NULL if unknown.
+ * @param   uExit       The exit.
+ */
+DECLINLINE(const char *) hmSvmGetSpecialExitReasonDesc(uint16_t uExit)
+{
+    switch (uExit)
+    {
+        case SVM_EXIT_NPF:                 return SVM_EXIT_REASON_NPF;
+        case SVM_EXIT_AVIC_INCOMPLETE_IPI: return SVM_EXIT_REASON_AVIC_INCOMPLETE_IPI;
+        case SVM_EXIT_AVIC_NOACCEL:        return SVM_EXIT_REASON_AVIC_NOACCEL;
+    }
+    return EXIT_REASON_NIL();
+}
+#undef EXIT_REASON_NIL
+#undef EXIT_REASON
+
+
+/**
+ * Checks whether HM (VT-x/AMD-V) is being used by this VM.
+ *
+ * @retval  true if used.
+ * @retval  false if software virtualization (raw-mode) is used.
+ * @param   pVM        The cross context VM structure.
+ * @sa      HMIsEnabled, HMR3IsEnabled
+ * @internal
+ */
+VMMDECL(bool) HMIsEnabledNotMacro(PVM pVM)
+{
+    Assert(pVM->bMainExecutionEngine != VM_EXEC_ENGINE_NOT_SET);
+    return pVM->fHMEnabled;
+}
+
+
+/**
+ * Checks if the guest is in a suitable state for hardware-assisted execution.
+ *
+ * @returns @c true if it is suitable, @c false otherwise.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pCtx    Pointer to the guest CPU context.
+ *
+ * @remarks @a pCtx can be a partial context created and not necessarily the same as
+ *          pVCpu->cpum.GstCtx.
+ */
+VMMDECL(bool) HMCanExecuteGuest(PVMCC pVM, PVMCPUCC pVCpu, PCCPUMCTX pCtx)
+{
+    Assert(HMIsEnabled(pVM));
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
+    if (   CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
+        || CPUMIsGuestInVmxNonRootMode(pCtx))
+    {
+        LogFunc(("In nested-guest mode - returning false"));
+        return false;
+    }
+#endif
+
+    /* AMD-V supports real & protected mode with or without paging. */
+    if (pVM->hm.s.svm.fEnabled)
+    {
+        pVCpu->hm.s.fActive = true;
+        return true;
+    }
+
+    bool rc = HMCanExecuteVmxGuest(pVM, pVCpu, pCtx);
+    LogFlowFunc(("returning %RTbool\n", rc));
+    return rc;
+}
+
+
+/**
+ * Queues a guest page for invalidation.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCVirt      Page to invalidate.
+ */
+static void hmQueueInvlPage(PVMCPU pVCpu, RTGCPTR GCVirt)
+{
+    /* Nothing to do if a TLB flush is already pending */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
+        return;
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+    NOREF(GCVirt);
+}
+
+
+/**
+ * Invalidates a guest page.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCVirt      Page to invalidate.
+ */
+VMM_INT_DECL(int) HMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
+{
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushPageManual);
+#ifdef IN_RING0
+    return HMR0InvalidatePage(pVCpu, GCVirt);
+#else
+    hmQueueInvlPage(pVCpu, GCVirt);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+#ifdef IN_RING0
+
+/**
+ * Dummy RTMpOnSpecific handler since RTMpPokeCpu couldn't be used.
+ *
+ */
+static DECLCALLBACK(void) hmFlushHandler(RTCPUID idCpu, void *pvUser1, void *pvUser2)
+{
+    NOREF(idCpu); NOREF(pvUser1); NOREF(pvUser2);
+    return;
+}
+
+
+/**
+ * Wrapper for RTMpPokeCpu to deal with VERR_NOT_SUPPORTED.
+ */
+static void hmR0PokeCpu(PVMCPUCC pVCpu, RTCPUID idHostCpu)
+{
+    uint32_t cWorldSwitchExits = ASMAtomicUoReadU32(&pVCpu->hmr0.s.cWorldSwitchExits);
+
+    STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatPoke, x);
+    int rc = RTMpPokeCpu(idHostCpu);
+    STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPoke, x);
+
+    /* Not implemented on some platforms (Darwin, Linux kernel < 2.6.19); fall
+       back to a less efficient implementation (broadcast). */
+    if (rc == VERR_NOT_SUPPORTED)
+    {
+        STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatSpinPoke, z);
+        /* synchronous. */
+        RTMpOnSpecific(idHostCpu, hmFlushHandler, 0, 0);
+        STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatSpinPoke, z);
+    }
+    else
+    {
+        if (rc == VINF_SUCCESS)
+            STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatSpinPoke, z);
+        else
+            STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatSpinPokeFailed, z);
+
+/** @todo If more than one CPU is going to be poked, we could optimize this
+ *        operation by poking them first and wait afterwards.  Would require
+ *        recording who to poke and their current cWorldSwitchExits values,
+ *        that's something not suitable for stack... So, pVCpu->hm.s.something
+ *        then. */
+        /* Spin until the VCPU has switched back (poking is async). */
+        while (   ASMAtomicUoReadBool(&pVCpu->hm.s.fCheckedTLBFlush)
+               && cWorldSwitchExits == ASMAtomicUoReadU32(&pVCpu->hmr0.s.cWorldSwitchExits))
+            ASMNopPause();
+
+        if (rc == VINF_SUCCESS)
+            STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatSpinPoke, z);
+        else
+            STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatSpinPokeFailed, z);
+    }
+}
+
+#endif /* IN_RING0 */
+
+/**
+ * Flushes the guest TLB.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(int) HMFlushTlb(PVMCPU pVCpu)
+{
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+    STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbManual);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Poke an EMT so it can perform the appropriate TLB shootdowns.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              EMT poke.
+ * @param   fAccountFlushStat   Whether to account the call to
+ *                              StatTlbShootdownFlush or StatTlbShootdown.
+ */
+static void hmPokeCpuForTlbFlush(PVMCPUCC pVCpu, bool fAccountFlushStat)
+{
+    if (ASMAtomicUoReadBool(&pVCpu->hm.s.fCheckedTLBFlush))
+    {
+        if (fAccountFlushStat)
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdownFlush);
+        else
+            STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
+#ifdef IN_RING0
+        RTCPUID idHostCpu = pVCpu->hmr0.s.idEnteredCpu;
+        if (idHostCpu != NIL_RTCPUID)
+            hmR0PokeCpu(pVCpu, idHostCpu);
+#else
+        VMR3NotifyCpuFFU(pVCpu->pUVCpu, VMNOTIFYFF_FLAGS_POKE);
+#endif
+    }
+    else
+        STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushPageManual);
+}
+
+
+/**
+ * Invalidates a guest page on all VCPUs.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCVirt      Page to invalidate.
+ */
+VMM_INT_DECL(int) HMInvalidatePageOnAllVCpus(PVMCC pVM, RTGCPTR GCVirt)
+{
+    /*
+     * The VT-x/AMD-V code will be flushing TLB each time a VCPU migrates to a different
+     * host CPU, see hmR0VmxFlushTaggedTlbBoth() and hmR0SvmFlushTaggedTlb().
+     *
+     * This is the reason why we do not care about thread preemption here and just
+     * execute HMInvalidatePage() assuming it might be the 'right' CPU.
+     */
+    VMCPUID const idCurCpu = VMMGetCpuId(pVM);
+    STAM_COUNTER_INC(&VMCC_GET_CPU(pVM, idCurCpu)->hm.s.StatFlushPage);
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+
+        /* Nothing to do if a TLB flush is already pending; the VCPU should
+           have already been poked if it were active. */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
+            continue;
+
+        if (pVCpu->idCpu == idCurCpu)
+            HMInvalidatePage(pVCpu, GCVirt);
+        else
+        {
+            hmQueueInvlPage(pVCpu, GCVirt);
+            hmPokeCpuForTlbFlush(pVCpu, false /* fAccountFlushStat */);
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Flush the TLBs of all VCPUs.
+ *
+ * @returns VBox status code.
+ * @param   pVM       The cross context VM structure.
+ */
+VMM_INT_DECL(int) HMFlushTlbOnAllVCpus(PVMCC pVM)
+{
+    if (pVM->cCpus == 1)
+        return HMFlushTlb(VMCC_GET_CPU_0(pVM));
+
+    VMCPUID const idThisCpu = VMMGetCpuId(pVM);
+
+    STAM_COUNTER_INC(&VMCC_GET_CPU(pVM, idThisCpu)->hm.s.StatFlushTlb);
+
+    for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
+    {
+        PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
+
+        /* Nothing to do if a TLB flush is already pending; the VCPU should
+           have already been poked if it were active. */
+        if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
+        {
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+            if (idThisCpu != idCpu)
+                hmPokeCpuForTlbFlush(pVCpu, true /* fAccountFlushStat */);
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Invalidates a guest page by physical address.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      Page to invalidate.
+ *
+ * @remarks Assumes the current instruction references this physical page
+ *          though a virtual address!
+ */
+VMM_INT_DECL(int) HMInvalidatePhysPage(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    if (!HMIsNestedPagingActive(pVM))
+        return VINF_SUCCESS;
+
+    /*
+     * AMD-V: Doesn't support invalidation with guest physical addresses.
+     *
+     * VT-x: Doesn't support invalidation with guest physical addresses.
+     * INVVPID instruction takes only a linear address while invept only flushes by EPT
+     * not individual addresses.
+     *
+     * We update the force flag and flush before the next VM-entry, see @bugref{6568}.
+     */
+    RT_NOREF(GCPhys);
+    /** @todo Remove or figure out to way to update the Phys STAT counter.  */
+    /* STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys); */
+    return HMFlushTlbOnAllVCpus(pVM);
+}
+
+
+/**
+ * Checks if nested paging is enabled.
+ *
+ * @returns true if nested paging is active, false otherwise.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks Works before hmR3InitFinalizeR0.
+ */
+VMM_INT_DECL(bool) HMIsNestedPagingActive(PVMCC pVM)
+{
+    return HMIsEnabled(pVM) && CTX_EXPR(pVM->hm.s.fNestedPagingCfg, pVM->hmr0.s.fNestedPaging, RT_NOTHING);
+}
+
+
+/**
+ * Checks if both nested paging and unhampered guest execution are enabled.
+ *
+ * The almost complete guest execution in hardware is only applicable to VT-x.
+ *
+ * @returns true if we have both enabled, otherwise false.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks Works before hmR3InitFinalizeR0.
+ */
+VMM_INT_DECL(bool) HMAreNestedPagingAndFullGuestExecEnabled(PVMCC pVM)
+{
+    return HMIsEnabled(pVM)
+        && CTX_EXPR(pVM->hm.s.fNestedPagingCfg, pVM->hmr0.s.fNestedPaging, RT_NOTHING)
+        && (   CTX_EXPR(pVM->hm.s.vmx.fUnrestrictedGuestCfg, pVM->hmr0.s.vmx.fUnrestrictedGuest, RT_NOTHING)
+            || pVM->hm.s.svm.fSupported);
+}
+
+
+/**
+ * Checks if this VM is using HM and is long-mode capable.
+ *
+ * Use VMR3IsLongModeAllowed() instead of this, when possible.
+ *
+ * @returns true if long mode is allowed, false otherwise.
+ * @param   pVM         The cross context VM structure.
+ * @sa      VMR3IsLongModeAllowed, NEMHCIsLongModeAllowed
+ */
+VMM_INT_DECL(bool) HMIsLongModeAllowed(PVMCC pVM)
+{
+    return HMIsEnabled(pVM) && CTX_EXPR(pVM->hm.s.fAllow64BitGuestsCfg, pVM->hmr0.s.fAllow64BitGuests, RT_NOTHING);
+}
+
+
+/**
+ * Checks if MSR bitmaps are active. It is assumed that when it's available
+ * it will be used as well.
+ *
+ * @returns true if MSR bitmaps are available, false otherwise.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(bool) HMIsMsrBitmapActive(PVM pVM)
+{
+    if (HMIsEnabled(pVM))
+    {
+        if (pVM->hm.s.svm.fSupported)
+            return true;
+
+        if (   pVM->hm.s.vmx.fSupported
+            && (  CTX_EXPR(pVM->hm.s.ForR3.vmx.Msrs.ProcCtls.n.allowed1, g_HmMsrs.u.vmx.ProcCtls.n.allowed1, RT_NOTHING)
+                & VMX_PROC_CTLS_USE_MSR_BITMAPS))
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * Checks if AMD-V is active.
+ *
+ * @returns true if AMD-V is active.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks Works before hmR3InitFinalizeR0.
+ */
+VMM_INT_DECL(bool) HMIsSvmActive(PVM pVM)
+{
+    return pVM->hm.s.svm.fSupported && HMIsEnabled(pVM);
+}
+
+
+/**
+ * Checks if VT-x is active.
+ *
+ * @returns true if VT-x is active.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks Works before hmR3InitFinalizeR0.
+ */
+VMM_INT_DECL(bool) HMIsVmxActive(PVM pVM)
+{
+    return pVM->hm.s.vmx.fSupported && HMIsEnabled(pVM);
+}
+
+
+/**
+ * Checks if an interrupt event is currently pending.
+ *
+ * @returns Interrupt event pending state.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(bool) HMHasPendingIrq(PVMCC pVM)
+{
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    return !!pVCpu->hm.s.Event.fPending;
+}
+
+
+/**
+ * Sets or clears the single instruction flag.
+ *
+ * When set, HM will try its best to return to ring-3 after executing a single
+ * instruction.  This can be used for debugging.  See also
+ * EMR3HmSingleInstruction.
+ *
+ * @returns The old flag state.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   fEnable The new flag state.
+ */
+VMM_INT_DECL(bool) HMSetSingleInstruction(PVMCC pVM, PVMCPUCC pVCpu, bool fEnable)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    bool fOld = pVCpu->hm.s.fSingleInstruction;
+    pVCpu->hm.s.fSingleInstruction = fEnable;
+    pVCpu->hm.s.fUseDebugLoop = fEnable || pVM->hm.s.fUseDebugLoop;
+    return fOld;
+}
+
+
+/**
+ * Notification callback which is called whenever there is a chance that a CR3
+ * value might have changed.
+ *
+ * This is called by PGM.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmShadowMode   New shadow paging mode.
+ * @param   enmGuestMode    New guest paging mode.
+ */
+VMM_INT_DECL(void) HMHCChangedPagingMode(PVM pVM, PVMCPUCC pVCpu, PGMMODE enmShadowMode, PGMMODE enmGuestMode)
+{
+#ifdef IN_RING3
+    /* Ignore page mode changes during state loading. */
+    if (VMR3GetState(pVM) == VMSTATE_LOADING)
+        return;
+#endif
+
+    pVCpu->hm.s.enmShadowMode = enmShadowMode;
+
+    /*
+     * If the guest left protected mode VMX execution, we'll have to be
+     * extra careful if/when the guest switches back to protected mode.
+     */
+    if (enmGuestMode == PGMMODE_REAL)
+    {
+        PVMXVMCSINFOSHARED pVmcsInfoShared = hmGetVmxActiveVmcsInfoShared(pVCpu);
+        pVmcsInfoShared->fWasInRealMode = true;
+    }
+
+#ifdef IN_RING0
+    /*
+     * We need to tickle SVM and VT-x state updates.
+     *
+     * Note! We could probably reduce this depending on what exactly changed.
+     */
+    if (VM_IS_HM_ENABLED(pVM))
+    {
+        CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER); /* No recursion! */
+        uint64_t fChanged = HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_CR3 | HM_CHANGED_GUEST_CR4 | HM_CHANGED_GUEST_EFER_MSR;
+        if (pVM->hm.s.svm.fSupported)
+            fChanged |= HM_CHANGED_SVM_XCPT_INTERCEPTS;
+        else
+            fChanged |= HM_CHANGED_VMX_XCPT_INTERCEPTS | HM_CHANGED_VMX_ENTRY_EXIT_CTLS;
+        ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fChanged);
+    }
+#endif
+
+    Log4(("HMHCChangedPagingMode: Guest paging mode '%s', shadow paging mode '%s'\n", PGMGetModeName(enmGuestMode),
+          PGMGetModeName(enmShadowMode)));
+}
+
+
+/**
+ * Gets VMX MSRs from the provided hardware-virtualization MSRs struct.
+ *
+ * This abstraction exists to insulate the support driver from including VMX
+ * structures from HM headers.
+ *
+ * @param   pHwvirtMsrs     The hardware-virtualization MSRs.
+ * @param   pVmxMsrs        Where to store the VMX MSRs.
+ */
+VMM_INT_DECL(void) HMGetVmxMsrsFromHwvirtMsrs(PCSUPHWVIRTMSRS pHwvirtMsrs, PVMXMSRS pVmxMsrs)
+{
+    AssertReturnVoid(pHwvirtMsrs);
+    AssertReturnVoid(pVmxMsrs);
+    pVmxMsrs->u64Basic         = pHwvirtMsrs->u.vmx.u64Basic;
+    pVmxMsrs->PinCtls.u        = pHwvirtMsrs->u.vmx.PinCtls.u;
+    pVmxMsrs->ProcCtls.u       = pHwvirtMsrs->u.vmx.ProcCtls.u;
+    pVmxMsrs->ProcCtls2.u      = pHwvirtMsrs->u.vmx.ProcCtls2.u;
+    pVmxMsrs->ExitCtls.u       = pHwvirtMsrs->u.vmx.ExitCtls.u;
+    pVmxMsrs->EntryCtls.u      = pHwvirtMsrs->u.vmx.EntryCtls.u;
+    pVmxMsrs->TruePinCtls.u    = pHwvirtMsrs->u.vmx.TruePinCtls.u;
+    pVmxMsrs->TrueProcCtls.u   = pHwvirtMsrs->u.vmx.TrueProcCtls.u;
+    pVmxMsrs->TrueEntryCtls.u  = pHwvirtMsrs->u.vmx.TrueEntryCtls.u;
+    pVmxMsrs->TrueExitCtls.u   = pHwvirtMsrs->u.vmx.TrueExitCtls.u;
+    pVmxMsrs->u64Misc          = pHwvirtMsrs->u.vmx.u64Misc;
+    pVmxMsrs->u64Cr0Fixed0     = pHwvirtMsrs->u.vmx.u64Cr0Fixed0;
+    pVmxMsrs->u64Cr0Fixed1     = pHwvirtMsrs->u.vmx.u64Cr0Fixed1;
+    pVmxMsrs->u64Cr4Fixed0     = pHwvirtMsrs->u.vmx.u64Cr4Fixed0;
+    pVmxMsrs->u64Cr4Fixed1     = pHwvirtMsrs->u.vmx.u64Cr4Fixed1;
+    pVmxMsrs->u64VmcsEnum      = pHwvirtMsrs->u.vmx.u64VmcsEnum;
+    pVmxMsrs->u64VmFunc        = pHwvirtMsrs->u.vmx.u64VmFunc;
+    pVmxMsrs->u64EptVpidCaps   = pHwvirtMsrs->u.vmx.u64EptVpidCaps;
+    pVmxMsrs->u64ProcCtls3     = pHwvirtMsrs->u.vmx.u64ProcCtls3;
+    pVmxMsrs->u64ExitCtls2     = pHwvirtMsrs->u.vmx.u64ExitCtls2;
+}
+
+
+/**
+ * Gets SVM MSRs from the provided hardware-virtualization MSRs struct.
+ *
+ * This abstraction exists to insulate the support driver from including SVM
+ * structures from HM headers.
+ *
+ * @param   pHwvirtMsrs     The hardware-virtualization MSRs.
+ * @param   pSvmMsrs        Where to store the SVM MSRs.
+ */
+VMM_INT_DECL(void) HMGetSvmMsrsFromHwvirtMsrs(PCSUPHWVIRTMSRS pHwvirtMsrs, PSVMMSRS pSvmMsrs)
+{
+    AssertReturnVoid(pHwvirtMsrs);
+    AssertReturnVoid(pSvmMsrs);
+    pSvmMsrs->u64MsrHwcr = pHwvirtMsrs->u.svm.u64MsrHwcr;
+}
+
+
+/**
+ * Gets the name of a VT-x exit code.
+ *
+ * @returns Pointer to read only string if @a uExit is known, otherwise NULL.
+ * @param   uExit               The VT-x exit to name.
+ */
+VMM_INT_DECL(const char *) HMGetVmxExitName(uint32_t uExit)
+{
+    uint16_t const uReason = VMX_EXIT_REASON_BASIC(uExit);
+    if (uReason <= MAX_EXITREASON_VTX)
+    {
+        Assert(uReason < RT_ELEMENTS(g_apszVmxExitReasons));
+        return g_apszVmxExitReasons[uReason];
+    }
+    return NULL;
+}
+
+
+/**
+ * Gets the name of an AMD-V exit code.
+ *
+ * @returns Pointer to read only string if @a uExit is known, otherwise NULL.
+ * @param   uExit               The AMD-V exit to name.
+ */
+VMM_INT_DECL(const char *) HMGetSvmExitName(uint32_t uExit)
+{
+    if (uExit <= MAX_EXITREASON_AMDV)
+    {
+        Assert(uExit < RT_ELEMENTS(g_apszSvmExitReasons));
+        return g_apszSvmExitReasons[uExit];
+    }
+    return hmSvmGetSpecialExitReasonDesc(uExit);
+}
+
diff --git a/src/VBox/VMM/VMMAll/HMSVMAll.cpp b/src/VBox/VMM/VMMAll/HMSVMAll.cpp
new file mode 100644
index 00000000..02e66339
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/HMSVMAll.cpp
@@ -0,0 +1,553 @@
+/* $Id: HMSVMAll.cpp $ */
+/** @file
+ * HM SVM (AMD-V) - All contexts.
+ */
+
+/*
+ * Copyright (C) 2017-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_HM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include "HMInternal.h"
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/err.h>
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+# include <iprt/asm-amd64-x86.h> /* ASMCpuId */
+#endif
+
+
+
+/**
+ * Emulates a simple MOV TPR (CR8) instruction.
+ *
+ * Used for TPR patching on 32-bit guests. This simply looks up the patch record
+ * at EIP and does the required.
+ *
+ * This VMMCALL is used a fallback mechanism when mov to/from cr8 isn't exactly
+ * like how we want it to be (e.g. not followed by shr 4 as is usually done for
+ * TPR). See hmR3ReplaceTprInstr() for the details.
+ *
+ * @returns VBox status code.
+ * @retval VINF_SUCCESS if the access was handled successfully, RIP + RFLAGS updated.
+ * @retval VERR_NOT_FOUND if no patch record for this RIP could be found.
+ * @retval VERR_SVM_UNEXPECTED_PATCH_TYPE if the found patch type is invalid.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(int) hmEmulateSvmMovTpr(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    Log4(("Emulated VMMCall TPR access replacement at RIP=%RGv\n", pCtx->rip));
+
+    AssertCompile(DISGREG_EAX  == X86_GREG_xAX);
+    AssertCompile(DISGREG_ECX  == X86_GREG_xCX);
+    AssertCompile(DISGREG_EDX  == X86_GREG_xDX);
+    AssertCompile(DISGREG_EBX  == X86_GREG_xBX);
+    AssertCompile(DISGREG_ESP  == X86_GREG_xSP);
+    AssertCompile(DISGREG_EBP  == X86_GREG_xBP);
+    AssertCompile(DISGREG_ESI  == X86_GREG_xSI);
+    AssertCompile(DISGREG_EDI  == X86_GREG_xDI);
+    AssertCompile(DISGREG_R8D  == X86_GREG_x8);
+    AssertCompile(DISGREG_R9D  == X86_GREG_x9);
+    AssertCompile(DISGREG_R10D == X86_GREG_x10);
+    AssertCompile(DISGREG_R11D == X86_GREG_x11);
+    AssertCompile(DISGREG_R12D == X86_GREG_x12);
+    AssertCompile(DISGREG_R13D == X86_GREG_x13);
+    AssertCompile(DISGREG_R14D == X86_GREG_x14);
+    AssertCompile(DISGREG_R15D == X86_GREG_x15);
+
+    /*
+     * We do this in a loop as we increment the RIP after a successful emulation
+     * and the new RIP may be a patched instruction which needs emulation as well.
+     */
+    bool fPatchFound = false;
+    for (;;)
+    {
+        PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
+        if (!pPatch)
+            break;
+        fPatchFound = true;
+
+        uint8_t u8Tpr;
+        switch (pPatch->enmType)
+        {
+            case HMTPRINSTR_READ:
+            {
+                bool fPending;
+                int  rc = APICGetTpr(pVCpu, &u8Tpr, &fPending, NULL /* pu8PendingIrq */);
+                AssertRC(rc);
+
+                uint8_t idxReg = pPatch->uDstOperand;
+                AssertStmt(idxReg < RT_ELEMENTS(pCtx->aGRegs), idxReg = RT_ELEMENTS(pCtx->aGRegs) - 1);
+                pCtx->aGRegs[idxReg].u64 = u8Tpr;
+                pCtx->rip += pPatch->cbOp;
+                pCtx->eflags.Bits.u1RF = 0;
+                break;
+            }
+
+            case HMTPRINSTR_WRITE_REG:
+            case HMTPRINSTR_WRITE_IMM:
+            {
+                if (pPatch->enmType == HMTPRINSTR_WRITE_REG)
+                {
+                    uint8_t idxReg = pPatch->uSrcOperand;
+                    AssertStmt(idxReg < RT_ELEMENTS(pCtx->aGRegs), idxReg = RT_ELEMENTS(pCtx->aGRegs) - 1);
+                    u8Tpr = pCtx->aGRegs[idxReg].u8;
+                }
+                else
+                    u8Tpr = (uint8_t)pPatch->uSrcOperand;
+
+                int rc2 = APICSetTpr(pVCpu, u8Tpr);
+                AssertRC(rc2);
+                pCtx->rip += pPatch->cbOp;
+                pCtx->eflags.Bits.u1RF = 0;
+                ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR
+                                                         | HM_CHANGED_GUEST_RIP
+                                                         | HM_CHANGED_GUEST_RFLAGS);
+                break;
+            }
+
+            default:
+            {
+                AssertMsgFailed(("Unexpected patch type %d\n", pPatch->enmType));
+                pVCpu->hm.s.u32HMError = pPatch->enmType;
+                return VERR_SVM_UNEXPECTED_PATCH_TYPE;
+            }
+        }
+    }
+
+    return fPatchFound ? VINF_SUCCESS : VERR_NOT_FOUND;
+}
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+/**
+ * Notification callback for when a \#VMEXIT happens outside SVM R0 code (e.g.
+ * in IEM).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pCtx            Pointer to the guest-CPU context.
+ *
+ * @sa      hmR0SvmVmRunCacheVmcb.
+ */
+VMM_INT_DECL(void) HMNotifySvmNstGstVmexit(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        /*
+         * Restore fields as our own code might look at the VMCB controls as part
+         * of the #VMEXIT handling in IEM. Otherwise, strictly speaking we don't need to
+         * restore these fields because currently none of them are written back to memory
+         * by a physical CPU on #VMEXIT.
+         */
+        PSVMVMCBCTRL pVmcbNstGstCtrl = &pCtx->hwvirt.svm.Vmcb.ctrl;
+        pVmcbNstGstCtrl->u16InterceptRdCRx                 = pVmcbNstGstCache->u16InterceptRdCRx;
+        pVmcbNstGstCtrl->u16InterceptWrCRx                 = pVmcbNstGstCache->u16InterceptWrCRx;
+        pVmcbNstGstCtrl->u16InterceptRdDRx                 = pVmcbNstGstCache->u16InterceptRdDRx;
+        pVmcbNstGstCtrl->u16InterceptWrDRx                 = pVmcbNstGstCache->u16InterceptWrDRx;
+        pVmcbNstGstCtrl->u16PauseFilterThreshold           = pVmcbNstGstCache->u16PauseFilterThreshold;
+        pVmcbNstGstCtrl->u16PauseFilterCount               = pVmcbNstGstCache->u16PauseFilterCount;
+        pVmcbNstGstCtrl->u32InterceptXcpt                  = pVmcbNstGstCache->u32InterceptXcpt;
+        pVmcbNstGstCtrl->u64InterceptCtrl                  = pVmcbNstGstCache->u64InterceptCtrl;
+        pVmcbNstGstCtrl->u64TSCOffset                      = pVmcbNstGstCache->u64TSCOffset;
+        pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking          = pVmcbNstGstCache->fVIntrMasking;
+        pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVmcbNstGstCache->fNestedPaging;
+        pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt               = pVmcbNstGstCache->fLbrVirt;
+        pVmcbNstGstCache->fCacheValid = false;
+    }
+
+    /*
+     * Transitions to ring-3 flag a full CPU-state change except if we transition to ring-3
+     * in response to a physical CPU interrupt as no changes to the guest-CPU state are
+     * expected (see VINF_EM_RAW_INTERRUPT handling in hmR0SvmExitToRing3).
+     *
+     * However, with nested-guests, the state -can- change on trips to ring-3 for we might
+     * try to inject a nested-guest physical interrupt and cause a SVM_EXIT_INTR #VMEXIT for
+     * the nested-guest from ring-3. Import the complete state here as we will be swapping
+     * to the guest VMCB after the #VMEXIT.
+     */
+    CPUMImportGuestStateOnDemand(pVCpu, CPUMCTX_EXTRN_ALL);
+    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_ALL);
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+}
+#endif
+
+/**
+ * Checks if the Virtual GIF (Global Interrupt Flag) feature is supported and
+ * enabled for the VM.
+ *
+ * @returns @c true if VGIF is enabled, @c false otherwise.
+ * @param   pVM         The cross context VM structure.
+ *
+ * @remarks This value returned by this functions is expected by the callers not
+ *          to change throughout the lifetime of the VM.
+ */
+VMM_INT_DECL(bool) HMIsSvmVGifActive(PCVMCC pVM)
+{
+#ifdef IN_RING0
+    bool const fVGif    = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
+#else
+    bool const fVGif    = RT_BOOL(pVM->hm.s.ForR3.svm.fFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
+#endif
+    return fVGif && pVM->hm.s.svm.fVGif;
+}
+
+
+/**
+ * Interface used by IEM to handle patched TPR accesses.
+ *
+ * @returns VBox status code
+ * @retval  VINF_SUCCESS if hypercall was handled, RIP + RFLAGS all dealt with.
+ * @retval  VERR_NOT_FOUND if hypercall was _not_ handled.
+ * @retval  VERR_SVM_UNEXPECTED_PATCH_TYPE on IPE.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(int) HMHCMaybeMovTprSvmHypercall(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (pVM->hm.s.fTprPatchingAllowed)
+    {
+        int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
+        if (RT_SUCCESS(rc))
+            return VINF_SUCCESS;
+        return rc;
+    }
+    return VERR_NOT_FOUND;
+}
+
+
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+/**
+ * Checks if the current AMD CPU is subject to erratum 170 "In SVM mode,
+ * incorrect code bytes may be fetched after a world-switch".
+ *
+ * @param   pu32Family      Where to store the CPU family (can be NULL).
+ * @param   pu32Model       Where to store the CPU model (can be NULL).
+ * @param   pu32Stepping    Where to store the CPU stepping (can be NULL).
+ * @returns true if the erratum applies, false otherwise.
+ */
+VMM_INT_DECL(int) HMIsSubjectToSvmErratum170(uint32_t *pu32Family, uint32_t *pu32Model, uint32_t *pu32Stepping)
+{
+    /*
+     * Erratum 170 which requires a forced TLB flush for each world switch:
+     * See AMD spec. "Revision Guide for AMD NPT Family 0Fh Processors".
+     *
+     * All BH-G1/2 and DH-G1/2 models include a fix:
+     * Athlon X2:   0x6b 1/2
+     *              0x68 1/2
+     * Athlon 64:   0x7f 1
+     *              0x6f 2
+     * Sempron:     0x7f 1/2
+     *              0x6f 2
+     *              0x6c 2
+     *              0x7c 2
+     * Turion 64:   0x68 2
+     */
+    uint32_t u32Dummy;
+    uint32_t u32Version, u32Family, u32Model, u32Stepping, u32BaseFamily;
+    ASMCpuId(1, &u32Version, &u32Dummy, &u32Dummy, &u32Dummy);
+    u32BaseFamily = (u32Version >> 8) & 0xf;
+    u32Family     = u32BaseFamily + (u32BaseFamily == 0xf ? ((u32Version >> 20) & 0x7f) : 0);
+    u32Model      = ((u32Version >> 4) & 0xf);
+    u32Model      = u32Model | ((u32BaseFamily == 0xf ? (u32Version >> 16) & 0x0f : 0) << 4);
+    u32Stepping   = u32Version & 0xf;
+
+    bool fErratumApplies = false;
+    if (   u32Family == 0xf
+        && !((u32Model == 0x68 || u32Model == 0x6b || u32Model == 0x7f) && u32Stepping >= 1)
+        && !((u32Model == 0x6f || u32Model == 0x6c || u32Model == 0x7c) && u32Stepping >= 2))
+        fErratumApplies = true;
+
+    if (pu32Family)
+        *pu32Family   = u32Family;
+    if (pu32Model)
+        *pu32Model    = u32Model;
+    if (pu32Stepping)
+        *pu32Stepping = u32Stepping;
+
+    return fErratumApplies;
+}
+#endif
+
+
+/**
+ * Converts an SVM event type to a TRPM event type.
+ *
+ * @returns The TRPM event type.
+ * @retval  TRPM_32BIT_HACK if the specified type of event isn't among the set
+ *          of recognized trap types.
+ *
+ * @param   pEvent       Pointer to the SVM event.
+ * @param   uVector      The vector associated with the event.
+ */
+VMM_INT_DECL(TRPMEVENT) HMSvmEventToTrpmEventType(PCSVMEVENT pEvent, uint8_t uVector)
+{
+    uint8_t const uType = pEvent->n.u3Type;
+    switch (uType)
+    {
+        case SVM_EVENT_EXTERNAL_IRQ:    return TRPM_HARDWARE_INT;
+        case SVM_EVENT_SOFTWARE_INT:    return TRPM_SOFTWARE_INT;
+        case SVM_EVENT_NMI:             return TRPM_TRAP;
+        case SVM_EVENT_EXCEPTION:
+        {
+            if (   uVector == X86_XCPT_BP
+                || uVector == X86_XCPT_OF)
+                return TRPM_SOFTWARE_INT;
+            return TRPM_TRAP;
+        }
+        default:
+            break;
+    }
+    AssertMsgFailed(("HMSvmEventToTrpmEvent: Invalid pending-event type %#x\n", uType));
+    return TRPM_32BIT_HACK;
+}
+
+
+/**
+ * Gets the SVM nested-guest control intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu64Intercepts  Where to store the control intercepts. Only updated when
+ *                          @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmCtrlIntercepts(PCVMCPU pVCpu, uint64_t *pu64Intercepts)
+{
+    Assert(pu64Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu64Intercepts = pVmcbNstGstCache->u64InterceptCtrl;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest CRx-read intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu16Intercepts  Where to store the CRx-read intercepts. Only updated
+ *                          when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmReadCRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts)
+{
+    Assert(pu16Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu16Intercepts = pVmcbNstGstCache->u16InterceptRdCRx;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest CRx-write intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu16Intercepts  Where to store the CRx-write intercepts. Only updated
+ *                          when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmWriteCRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts)
+{
+    Assert(pu16Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu16Intercepts = pVmcbNstGstCache->u16InterceptWrCRx;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest DRx-read intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu16Intercepts  Where to store the DRx-read intercepts. Only updated
+ *                          when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmReadDRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts)
+{
+    Assert(pu16Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu16Intercepts = pVmcbNstGstCache->u16InterceptRdDRx;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest DRx-write intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu16Intercepts  Where to store the DRx-write intercepts. Only updated
+ *                          when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmWriteDRxIntercepts(PCVMCPU pVCpu, uint16_t *pu16Intercepts)
+{
+    Assert(pu16Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu16Intercepts = pVmcbNstGstCache->u16InterceptWrDRx;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest exception intercepts if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu32Intercepts  Where to store the exception intercepts. Only updated
+ *                          when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmXcptIntercepts(PCVMCPU pVCpu, uint32_t *pu32Intercepts)
+{
+    Assert(pu32Intercepts);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu32Intercepts = pVmcbNstGstCache->u32InterceptXcpt;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Checks if the nested-guest VMCB has virtual-interrupts masking enabled.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pfVIntrMasking  Where to store the virtual-interrupt masking bit.
+ *                          Updated only when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmVirtIntrMasking(PCVMCPU pVCpu, bool *pfVIntrMasking)
+{
+    Assert(pfVIntrMasking);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pfVIntrMasking = pVmcbNstGstCache->fVIntrMasking;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Gets the SVM nested-guest nested-paging bit if cached by HM.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling EMT.
+ * @param   pfNestedPaging      Where to store the nested-paging bit. Updated only
+ *                              when @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmNestedPaging(PCVMCPU pVCpu, bool *pfNestedPaging)
+{
+    Assert(pfNestedPaging);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pfNestedPaging = pVmcbNstGstCache->fNestedPaging;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Returns the nested-guest VMCB pause-filter count.
+ *
+ * @returns @c true on success, @c false otherwise.
+ * @param   pVCpu                   The cross context virtual CPU structure of the
+ *                                  calling EMT.
+ * @param   pu16PauseFilterCount    Where to store the pause-filter count. Only
+ *                                  updated @c true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmPauseFilterCount(PCVMCPU pVCpu, uint16_t *pu16PauseFilterCount)
+{
+    Assert(pu16PauseFilterCount);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu16PauseFilterCount = pVmcbNstGstCache->u16PauseFilterCount;
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Returns the SVM nested-guest TSC offset if cached by HM.
+ *
+ * @returns The TSC offset after applying any nested-guest TSC offset.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   pu64TscOffset   Where to store the TSC offset. Only updated when @c
+ *                          true is returned.
+ */
+VMM_INT_DECL(bool) HMGetGuestSvmTscOffset(PCVMCPU pVCpu, uint64_t *pu64TscOffset)
+{
+    Assert(pu64TscOffset);
+    PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
+    if (pVmcbNstGstCache->fCacheValid)
+    {
+        *pu64TscOffset = pVmcbNstGstCache->u64TSCOffset;
+        return true;
+    }
+    return false;
+}
+
diff --git a/src/VBox/VMM/VMMAll/HMVMXAll.cpp b/src/VBox/VMM/VMMAll/HMVMXAll.cpp
new file mode 100644
index 00000000..e574fe27
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/HMVMXAll.cpp
@@ -0,0 +1,1346 @@
+/* $Id: HMVMXAll.cpp $ */
+/** @file
+ * HM VMX (VT-x) - All contexts.
+ */
+
+/*
+ * Copyright (C) 2018-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_HM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include "HMInternal.h"
+#include <VBox/vmm/hmvmxinline.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/pdmapi.h>
+#include <iprt/errcore.h>
+
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+# include <iprt/asm-amd64-x86.h> /* ASMCpuId_EAX */
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#define VMXV_DIAG_DESC(a_Def, a_Desc)      #a_Def " - " #a_Desc
+/** VMX virtual-instructions and VM-exit diagnostics. */
+static const char * const g_apszVmxVDiagDesc[] =
+{
+    /* Internal processing errors. */
+    VMXV_DIAG_DESC(kVmxVDiag_None                             , "None"                      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_1                            , "Ipe_1"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_2                            , "Ipe_2"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_3                            , "Ipe_3"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_4                            , "Ipe_4"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_5                            , "Ipe_5"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_6                            , "Ipe_6"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_7                            , "Ipe_7"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_8                            , "Ipe_8"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_9                            , "Ipe_9"                     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_10                           , "Ipe_10"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_11                           , "Ipe_11"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_12                           , "Ipe_12"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_13                           , "Ipe_13"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_14                           , "Ipe_14"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_15                           , "Ipe_15"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Ipe_16                           , "Ipe_16"                    ),
+    /* VMXON. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_A20M                       , "A20M"                      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cpl                        , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr0Fixed0                  , "Cr0Fixed0"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr0Fixed1                  , "Cr0Fixed1"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr4Fixed0                  , "Cr4Fixed0"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Cr4Fixed1                  , "Cr4Fixed1"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Intercept                  , "Intercept"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_LongModeCS                 , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_MsrFeatCtl                 , "MsrFeatCtl"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrAbnormal                , "PtrAbnormal"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrAlign                   , "PtrAlign"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrMap                     , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrReadPhys                , "PtrReadPhys"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_PtrWidth                   , "PtrWidth"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_RealOrV86Mode              , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_ShadowVmcs                 , "ShadowVmcs"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_VmxAlreadyRoot             , "VmxAlreadyRoot"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_Vmxe                       , "Vmxe"                      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_VmcsRevId                  , "VmcsRevId"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxon_VmxRootCpl                 , "VmxRootCpl"                ),
+    /* VMXOFF. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_Cpl                       , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_Intercept                 , "Intercept"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_LongModeCS                , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_RealOrV86Mode             , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_Vmxe                      , "Vmxe"                      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmxoff_VmxRoot                   , "VmxRoot"                   ),
+    /* VMPTRLD. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrAbnormal              , "PtrAbnormal"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrAlign                 , "PtrAlign"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrMap                   , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrReadPhys              , "PtrReadPhys"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrVmxon                 , "PtrVmxon"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_PtrWidth                 , "PtrWidth"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_RevPtrReadPhys           , "RevPtrReadPhys"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_ShadowVmcs               , "ShadowVmcs"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_VmcsRevId                , "VmcsRevId"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrld_VmxRoot                  , "VmxRoot"                   ),
+    /* VMPTRST. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_PtrMap                   , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmptrst_VmxRoot                  , "VmxRoot"                   ),
+    /* VMCLEAR. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrAbnormal              , "PtrAbnormal"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrAlign                 , "PtrAlign"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrMap                   , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrReadPhys              , "PtrReadPhys"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrVmxon                 , "PtrVmxon"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_PtrWidth                 , "PtrWidth"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmclear_VmxRoot                  , "VmxRoot"                   ),
+    /* VMWRITE. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_FieldInvalid             , "FieldInvalid"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_FieldRo                  , "FieldRo"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_LinkPtrInvalid           , "LinkPtrInvalid"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_PtrInvalid               , "PtrInvalid"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_PtrMap                   , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmwrite_VmxRoot                  , "VmxRoot"                   ),
+    /* VMREAD. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_Cpl                       , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_FieldInvalid              , "FieldInvalid"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_LinkPtrInvalid            , "LinkPtrInvalid"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_LongModeCS                , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_PtrInvalid                , "PtrInvalid"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_PtrMap                    , "PtrMap"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_RealOrV86Mode             , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmread_VmxRoot                   , "VmxRoot"                   ),
+    /* INVVPID. */
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_DescRsvd                 , "DescRsvd"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_TypeInvalid              , "TypeInvalid"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type0InvalidAddr         , "Type0InvalidAddr"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type0InvalidVpid         , "Type0InvalidVpid"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type1InvalidVpid         , "Type1InvalidVpid"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_Type3InvalidVpid         , "Type3InvalidVpid"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invvpid_VmxRoot                  , "VmxRoot"                   ),
+    /* INVEPT. */
+    VMXV_DIAG_DESC(kVmxVDiag_Invept_Cpl                       , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invept_DescRsvd                  , "DescRsvd"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invept_EptpInvalid               , "EptpInvalid"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invept_LongModeCS                , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invept_RealOrV86Mode             , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invept_TypeInvalid               , "TypeInvalid"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Invept_VmxRoot                   , "VmxRoot"                   ),
+    /* VMLAUNCH/VMRESUME. */
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrApicAccess           , "AddrApicAccess"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrApicAccessEqVirtApic , "AddrApicAccessEqVirtApic"  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrApicAccessHandlerReg , "AddrApicAccessHandlerReg"  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrEntryMsrLoad         , "AddrEntryMsrLoad"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrExitMsrLoad          , "AddrExitMsrLoad"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrExitMsrStore         , "AddrExitMsrStore"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrIoBitmapA            , "AddrIoBitmapA"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrIoBitmapB            , "AddrIoBitmapB"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrMsrBitmap            , "AddrMsrBitmap"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVirtApicPage         , "AddrVirtApicPage"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVmcsLinkPtr          , "AddrVmcsLinkPtr"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVmreadBitmap         , "AddrVmreadBitmap"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_AddrVmwriteBitmap        , "AddrVmwriteBitmap"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ApicRegVirt              , "ApicRegVirt"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_BlocKMovSS               , "BlockMovSS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_Cpl                      , "Cpl"                       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_Cr3TargetCount           , "Cr3TargetCount"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryCtlsAllowed1        , "EntryCtlsAllowed1"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryCtlsDisallowed0     , "EntryCtlsDisallowed0"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryInstrLen            , "EntryInstrLen"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryInstrLenZero        , "EntryInstrLenZero"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryIntInfoErrCodePe    , "EntryIntInfoErrCodePe"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryIntInfoErrCodeVec   , "EntryIntInfoErrCodeVec"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryIntInfoTypeVecRsvd  , "EntryIntInfoTypeVecRsvd"   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EntryXcptErrCodeRsvd     , "EntryXcptErrCodeRsvd"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EptpAccessDirty          , "EptpAccessDirty"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EptpPageWalkLength       , "EptpPageWalkLength"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EptpMemType              , "EptpMemType"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_EptpRsvd                 , "EptpRsvd"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ExitCtlsAllowed1         , "ExitCtlsAllowed1"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ExitCtlsDisallowed0      , "ExitCtlsDisallowed0"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateHlt         , "GuestActStateHlt"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateRsvd        , "GuestActStateRsvd"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateShutdown    , "GuestActStateShutdown"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateSsDpl       , "GuestActStateSsDpl"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestActStateStiMovSs    , "GuestActStateStiMovSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr0Fixed0           , "GuestCr0Fixed0"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr0Fixed1           , "GuestCr0Fixed1"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr0PgPe             , "GuestCr0PgPe"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr3                 , "GuestCr3"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr4Fixed0           , "GuestCr4Fixed0"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestCr4Fixed1           , "GuestCr4Fixed1"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestDebugCtl            , "GuestDebugCtl"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestDr7                 , "GuestDr7"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestEferMsr             , "GuestEferMsr"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestEferMsrRsvd         , "GuestEferMsrRsvd"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestGdtrBase            , "GuestGdtrBase"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestGdtrLimit           , "GuestGdtrLimit"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIdtrBase            , "GuestIdtrBase"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIdtrLimit           , "GuestIdtrLimit"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateEnclave     , "GuestIntStateEnclave"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateExtInt      , "GuestIntStateExtInt"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateNmi         , "GuestIntStateNmi"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateRFlagsSti   , "GuestIntStateRFlagsSti"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateRsvd        , "GuestIntStateRsvd"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateSmi         , "GuestIntStateSmi"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateStiMovSs    , "GuestIntStateStiMovSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestIntStateVirtNmi     , "GuestIntStateVirtNmi"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPae                 , "GuestPae"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPatMsr              , "GuestPatMsr"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPcide               , "GuestPcide"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPdpte               , "GuestPdpteRsvd"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptBsNoTf    , "GuestPndDbgXcptBsNoTf"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptBsTf      , "GuestPndDbgXcptBsTf"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptRsvd      , "GuestPndDbgXcptRsvd"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestPndDbgXcptRtm       , "GuestPndDbgXcptRtm"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRip                 , "GuestRip"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRipRsvd             , "GuestRipRsvd"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRFlagsIf            , "GuestRFlagsIf"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRFlagsRsvd          , "GuestRFlagsRsvd"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestRFlagsVm            , "GuestRFlagsVm"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDefBig     , "GuestSegAttrCsDefBig"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDplEqSs    , "GuestSegAttrCsDplEqSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDplLtSs    , "GuestSegAttrCsDplLtSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsDplZero    , "GuestSegAttrCsDplZero"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsType       , "GuestSegAttrCsType"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrCsTypeRead   , "GuestSegAttrCsTypeRead"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeCs   , "GuestSegAttrDescTypeCs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeDs   , "GuestSegAttrDescTypeDs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeEs   , "GuestSegAttrDescTypeEs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeFs   , "GuestSegAttrDescTypeFs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeGs   , "GuestSegAttrDescTypeGs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDescTypeSs   , "GuestSegAttrDescTypeSs"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplCs     , "GuestSegAttrDplRplCs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplDs     , "GuestSegAttrDplRplDs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplEs     , "GuestSegAttrDplRplEs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplFs     , "GuestSegAttrDplRplFs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplGs     , "GuestSegAttrDplRplGs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrDplRplSs     , "GuestSegAttrDplRplSs"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranCs       , "GuestSegAttrGranCs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranDs       , "GuestSegAttrGranDs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranEs       , "GuestSegAttrGranEs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranFs       , "GuestSegAttrGranFs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranGs       , "GuestSegAttrGranGs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrGranSs       , "GuestSegAttrGranSs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrDescType , "GuestSegAttrLdtrDescType"  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrGran     , "GuestSegAttrLdtrGran"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrPresent  , "GuestSegAttrLdtrPresent"   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrRsvd     , "GuestSegAttrLdtrRsvd"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrLdtrType     , "GuestSegAttrLdtrType"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentCs    , "GuestSegAttrPresentCs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentDs    , "GuestSegAttrPresentDs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentEs    , "GuestSegAttrPresentEs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentFs    , "GuestSegAttrPresentFs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentGs    , "GuestSegAttrPresentGs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrPresentSs    , "GuestSegAttrPresentSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdCs       , "GuestSegAttrRsvdCs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdDs       , "GuestSegAttrRsvdDs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdEs       , "GuestSegAttrRsvdEs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdFs       , "GuestSegAttrRsvdFs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdGs       , "GuestSegAttrRsvdGs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrRsvdSs       , "GuestSegAttrRsvdSs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrSsDplEqRpl   , "GuestSegAttrSsDplEqRpl"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrSsDplZero    , "GuestSegAttrSsDplZero "    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrSsType       , "GuestSegAttrSsType"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrDescType   , "GuestSegAttrTrDescType"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrGran       , "GuestSegAttrTrGran"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrPresent    , "GuestSegAttrTrPresent"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrRsvd       , "GuestSegAttrTrRsvd"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrType       , "GuestSegAttrTrType"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTrUnusable   , "GuestSegAttrTrUnusable"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccCs    , "GuestSegAttrTypeAccCs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccDs    , "GuestSegAttrTypeAccDs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccEs    , "GuestSegAttrTypeAccEs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccFs    , "GuestSegAttrTypeAccFs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccGs    , "GuestSegAttrTypeAccGs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrTypeAccSs    , "GuestSegAttrTypeAccSs"     ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Cs        , "GuestSegAttrV86Cs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Ds        , "GuestSegAttrV86Ds"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Es        , "GuestSegAttrV86Es"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Fs        , "GuestSegAttrV86Fs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Gs        , "GuestSegAttrV86Gs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegAttrV86Ss        , "GuestSegAttrV86Ss"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseCs           , "GuestSegBaseCs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseDs           , "GuestSegBaseDs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseEs           , "GuestSegBaseEs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseFs           , "GuestSegBaseFs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseGs           , "GuestSegBaseGs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseLdtr         , "GuestSegBaseLdtr"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseSs           , "GuestSegBaseSs"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseTr           , "GuestSegBaseTr"            ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Cs        , "GuestSegBaseV86Cs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Ds        , "GuestSegBaseV86Ds"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Es        , "GuestSegBaseV86Es"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Fs        , "GuestSegBaseV86Fs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Gs        , "GuestSegBaseV86Gs"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegBaseV86Ss        , "GuestSegBaseV86Ss"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Cs       , "GuestSegLimitV86Cs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Ds       , "GuestSegLimitV86Ds"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Es       , "GuestSegLimitV86Es"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Fs       , "GuestSegLimitV86Fs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Gs       , "GuestSegLimitV86Gs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegLimitV86Ss       , "GuestSegLimitV86Ss"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegSelCsSsRpl       , "GuestSegSelCsSsRpl"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegSelLdtr          , "GuestSegSelLdtr"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSegSelTr            , "GuestSegSelTr"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_GuestSysenterEspEip      , "GuestSysenterEspEip"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrCurVmcs       , "VmcsLinkPtrCurVmcs"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrReadPhys      , "VmcsLinkPtrReadPhys"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrRevId         , "VmcsLinkPtrRevId"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLinkPtrShadow        , "VmcsLinkPtrShadow"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr0Fixed0            , "HostCr0Fixed0"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr0Fixed1            , "HostCr0Fixed1"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr3                  , "HostCr3"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Fixed0            , "HostCr4Fixed0"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Fixed1            , "HostCr4Fixed1"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Pae               , "HostCr4Pae"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCr4Pcide             , "HostCr4Pcide"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostCsTr                 , "HostCsTr"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostEferMsr              , "HostEferMsr"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostEferMsrRsvd          , "HostEferMsrRsvd"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostGuestLongMode        , "HostGuestLongMode"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostGuestLongModeNoCpu   , "HostGuestLongModeNoCpu"    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostLongMode             , "HostLongMode"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostPatMsr               , "HostPatMsr"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostRip                  , "HostRip"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostRipRsvd              , "HostRipRsvd"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSel                  , "HostSel"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSegBase              , "HostSegBase"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSs                   , "HostSs"                    ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_HostSysenterEspEip       , "HostSysenterEspEip"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_IoBitmapAPtrReadPhys     , "IoBitmapAPtrReadPhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_IoBitmapBPtrReadPhys     , "IoBitmapBPtrReadPhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_LongModeCS               , "LongModeCS"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrBitmapPtrReadPhys     , "MsrBitmapPtrReadPhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoad                  , "MsrLoad"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadCount             , "MsrLoadCount"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadPtrReadPhys       , "MsrLoadPtrReadPhys"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadRing3             , "MsrLoadRing3"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_MsrLoadRsvd              , "MsrLoadRsvd"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_NmiWindowExit            , "NmiWindowExit"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PinCtlsAllowed1          , "PinCtlsAllowed1"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PinCtlsDisallowed0       , "PinCtlsDisallowed0"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtlsAllowed1         , "ProcCtlsAllowed1"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtlsDisallowed0      , "ProcCtlsDisallowed0"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtls2Allowed1        , "ProcCtls2Allowed1"         ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_ProcCtls2Disallowed0     , "ProcCtls2Disallowed0"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PtrInvalid               , "PtrInvalid"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_PtrShadowVmcs            , "PtrShadowVmcs"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_RealOrV86Mode            , "RealOrV86Mode"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_SavePreemptTimer         , "SavePreemptTimer"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_TprThresholdRsvd         , "TprThresholdRsvd"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_TprThresholdVTpr         , "TprThresholdVTpr"          ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtApicPagePtrReadPhys  , "VirtApicPageReadPhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtIntDelivery          , "VirtIntDelivery"           ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtNmi                  , "VirtNmi"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtX2ApicTprShadow      , "VirtX2ApicTprShadow"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VirtX2ApicVirtApic       , "VirtX2ApicVirtApic"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsClear                , "VmcsClear"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmcsLaunch               , "VmcsLaunch"                ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmreadBitmapPtrReadPhys  , "VmreadBitmapPtrReadPhys"   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmwriteBitmapPtrReadPhys , "VmwriteBitmapPtrReadPhys"  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_VmxRoot                  , "VmxRoot"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmentry_Vpid                     , "Vpid"                      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_HostPdpte                 , "HostPdpte"                 ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoad                   , "MsrLoad"                   ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadCount              , "MsrLoadCount"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadPtrReadPhys        , "MsrLoadPtrReadPhys"        ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadRing3              , "MsrLoadRing3"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrLoadRsvd               , "MsrLoadRsvd"               ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStore                  , "MsrStore"                  ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStoreCount             , "MsrStoreCount"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStorePtrReadPhys       , "MsrStorePtrReadPhys"       ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStorePtrWritePhys      , "MsrStorePtrWritePhys"      ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStoreRing3             , "MsrStoreRing3"             ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_MsrStoreRsvd              , "MsrStoreRsvd"              ),
+    VMXV_DIAG_DESC(kVmxVDiag_Vmexit_VirtApicPagePtrWritePhys  , "VirtApicPagePtrWritePhys"  )
+    /* kVmxVDiag_End */
+};
+AssertCompile(RT_ELEMENTS(g_apszVmxVDiagDesc) == kVmxVDiag_End);
+#undef VMXV_DIAG_DESC
+
+
+/**
+ * Gets the descriptive name of a VMX instruction/VM-exit diagnostic code.
+ *
+ * @returns The descriptive string.
+ * @param   enmDiag    The VMX diagnostic.
+ */
+VMM_INT_DECL(const char *) HMGetVmxDiagDesc(VMXVDIAG enmDiag)
+{
+    if (RT_LIKELY((unsigned)enmDiag < RT_ELEMENTS(g_apszVmxVDiagDesc)))
+        return g_apszVmxVDiagDesc[enmDiag];
+    return "Unknown/invalid";
+}
+
+
+/**
+ * Checks if a code selector (CS) is suitable for execution using hardware-assisted
+ * VMX when unrestricted execution isn't available.
+ *
+ * @returns true if selector is suitable for VMX, otherwise
+ *        false.
+ * @param   pSel        Pointer to the selector to check (CS).
+ * @param   uStackDpl   The CPL, aka the DPL of the stack segment.
+ */
+static bool hmVmxIsCodeSelectorOk(PCCPUMSELREG pSel, unsigned uStackDpl)
+{
+    /*
+     * Segment must be an accessed code segment, it must be present and it must
+     * be usable.
+     * Note! These are all standard requirements and if CS holds anything else
+     *       we've got buggy code somewhere!
+     */
+    AssertCompile(X86DESCATTR_TYPE == 0xf);
+    AssertMsgReturn(   (pSel->Attr.u & (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_CODE | X86DESCATTR_DT | X86DESCATTR_P | X86DESCATTR_UNUSABLE))
+                    ==                 (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_CODE | X86DESCATTR_DT | X86DESCATTR_P),
+                    ("%#x\n", pSel->Attr.u),
+                    false);
+
+    /*
+     * For conforming segments, CS.DPL must be <= SS.DPL, while CS.DPL must equal
+     * SS.DPL for non-confroming segments.
+     * Note! This is also a hard requirement like above.
+     */
+    AssertMsgReturn(  pSel->Attr.n.u4Type & X86_SEL_TYPE_CONF
+                    ? pSel->Attr.n.u2Dpl <= uStackDpl
+                    : pSel->Attr.n.u2Dpl == uStackDpl,
+                    ("u4Type=%#x u2Dpl=%u uStackDpl=%u\n", pSel->Attr.n.u4Type, pSel->Attr.n.u2Dpl, uStackDpl),
+                    false);
+
+    /*
+     * The following two requirements are VT-x specific:
+     *  - G bit must be set if any high limit bits are set.
+     *  - G bit must be clear if any low limit bits are clear.
+     */
+    if (   ((pSel->u32Limit & 0xfff00000) == 0x00000000 ||  pSel->Attr.n.u1Granularity)
+        && ((pSel->u32Limit & 0x00000fff) == 0x00000fff || !pSel->Attr.n.u1Granularity))
+        return true;
+    return false;
+}
+
+
+/**
+ * Checks if a data selector (DS/ES/FS/GS) is suitable for execution using
+ * hardware-assisted VMX when unrestricted execution isn't available.
+ *
+ * @returns true if selector is suitable for VMX, otherwise
+ *        false.
+ * @param   pSel        Pointer to the selector to check
+ *                      (DS/ES/FS/GS).
+ */
+static bool hmVmxIsDataSelectorOk(PCCPUMSELREG pSel)
+{
+    /*
+     * Unusable segments are OK.  These days they should be marked as such, as
+     * but as an alternative we for old saved states and AMD<->VT-x migration
+     * we also treat segments with all the attributes cleared as unusable.
+     */
+    if (pSel->Attr.n.u1Unusable || !pSel->Attr.u)
+        return true;
+
+    /** @todo tighten these checks. Will require CPUM load adjusting. */
+
+    /* Segment must be accessed. */
+    if (pSel->Attr.u & X86_SEL_TYPE_ACCESSED)
+    {
+        /* Code segments must also be readable. */
+        if (  !(pSel->Attr.u & X86_SEL_TYPE_CODE)
+            || (pSel->Attr.u & X86_SEL_TYPE_READ))
+        {
+            /* The S bit must be set. */
+            if (pSel->Attr.n.u1DescType)
+            {
+                /* Except for conforming segments, DPL >= RPL. */
+                if (   pSel->Attr.n.u2Dpl  >= (pSel->Sel & X86_SEL_RPL)
+                    || pSel->Attr.n.u4Type >= X86_SEL_TYPE_ER_ACC)
+                {
+                    /* Segment must be present. */
+                    if (pSel->Attr.n.u1Present)
+                    {
+                        /*
+                         * The following two requirements are VT-x specific:
+                         *   - G bit must be set if any high limit bits are set.
+                         *   - G bit must be clear if any low limit bits are clear.
+                         */
+                        if (   ((pSel->u32Limit & 0xfff00000) == 0x00000000 ||  pSel->Attr.n.u1Granularity)
+                            && ((pSel->u32Limit & 0x00000fff) == 0x00000fff || !pSel->Attr.n.u1Granularity))
+                            return true;
+                    }
+                }
+            }
+        }
+    }
+
+    return false;
+}
+
+
+/**
+ * Checks if the stack selector (SS) is suitable for execution using
+ * hardware-assisted VMX when unrestricted execution isn't available.
+ *
+ * @returns true if selector is suitable for VMX, otherwise
+ *        false.
+ * @param   pSel        Pointer to the selector to check (SS).
+ */
+static bool hmVmxIsStackSelectorOk(PCCPUMSELREG pSel)
+{
+    /*
+     * Unusable segments are OK.  These days they should be marked as such, as
+     * but as an alternative we for old saved states and AMD<->VT-x migration
+     * we also treat segments with all the attributes cleared as unusable.
+     */
+    /** @todo r=bird: actually all zeroes isn't gonna cut it... SS.DPL == CPL. */
+    if (pSel->Attr.n.u1Unusable || !pSel->Attr.u)
+        return true;
+
+    /*
+     * Segment must be an accessed writable segment, it must be present.
+     * Note! These are all standard requirements and if SS holds anything else
+     *       we've got buggy code somewhere!
+     */
+    AssertCompile(X86DESCATTR_TYPE == 0xf);
+    AssertMsgReturn(   (pSel->Attr.u & (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_WRITE | X86DESCATTR_DT | X86DESCATTR_P | X86_SEL_TYPE_CODE))
+                    ==                 (X86_SEL_TYPE_ACCESSED | X86_SEL_TYPE_WRITE | X86DESCATTR_DT | X86DESCATTR_P),
+                    ("%#x\n", pSel->Attr.u), false);
+
+    /*
+     * DPL must equal RPL. But in real mode or soon after enabling protected
+     * mode, it might not be.
+     */
+    if (pSel->Attr.n.u2Dpl == (pSel->Sel & X86_SEL_RPL))
+    {
+        /*
+         * The following two requirements are VT-x specific:
+         *   - G bit must be set if any high limit bits are set.
+         *   - G bit must be clear if any low limit bits are clear.
+         */
+        if (   ((pSel->u32Limit & 0xfff00000) == 0x00000000 ||  pSel->Attr.n.u1Granularity)
+            && ((pSel->u32Limit & 0x00000fff) == 0x00000fff || !pSel->Attr.n.u1Granularity))
+            return true;
+    }
+    return false;
+}
+
+
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+/**
+ * Checks if the CPU is subject to the "VMX-Preemption Timer Does Not Count Down at
+ * the Rate Specified" erratum.
+ *
+ * Errata names and related steppings:
+ *      - BA86   - D0.
+ *      - AAX65  - C2.
+ *      - AAU65  - C2, K0.
+ *      - AAO95  - B1.
+ *      - AAT59  - C2.
+ *      - AAK139 - D0.
+ *      - AAM126 - C0, C1, D0.
+ *      - AAN92  - B1.
+ *      - AAJ124 - C0, D0.
+ *      - AAP86  - B1.
+ *
+ * Steppings: B1, C0, C1, C2, D0, K0.
+ *
+ * @returns @c true if subject to it, @c false if not.
+ */
+VMM_INT_DECL(bool) HMIsSubjectToVmxPreemptTimerErratum(void)
+{
+    uint32_t u = ASMCpuId_EAX(1);
+    u &= ~(RT_BIT_32(14) | RT_BIT_32(15) | RT_BIT_32(28) | RT_BIT_32(29) | RT_BIT_32(30) | RT_BIT_32(31));
+    if (   u == 0x000206E6 /* 323344.pdf - BA86   - D0 - Xeon Processor 7500 Series */
+        || u == 0x00020652 /* 323056.pdf - AAX65  - C2 - Xeon Processor L3406 */
+                           /* 322814.pdf - AAT59  - C2 - CoreTM i7-600, i5-500, i5-400 and i3-300 Mobile Processor Series */
+                           /* 322911.pdf - AAU65  - C2 - CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */
+        || u == 0x00020655 /* 322911.pdf - AAU65  - K0 - CoreTM i5-600, i3-500 Desktop Processor Series and Intel Pentium Processor G6950 */
+        || u == 0x000106E5 /* 322373.pdf - AAO95  - B1 - Xeon Processor 3400 Series */
+                           /* 322166.pdf - AAN92  - B1 - CoreTM i7-800 and i5-700 Desktop Processor Series */
+                           /* 320767.pdf - AAP86  - B1 - Core i7-900 Mobile Processor Extreme Edition Series, Intel Core i7-800 and i7-700 Mobile Processor Series */
+        || u == 0x000106A0 /* 321333.pdf - AAM126 - C0 - Xeon Processor 3500 Series Specification */
+        || u == 0x000106A1 /* 321333.pdf - AAM126 - C1 - Xeon Processor 3500 Series Specification */
+        || u == 0x000106A4 /* 320836.pdf - AAJ124 - C0 - Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */
+        || u == 0x000106A5 /* 321333.pdf - AAM126 - D0 - Xeon Processor 3500 Series Specification */
+                           /* 321324.pdf - AAK139 - D0 - Xeon Processor 5500 Series Specification */
+                           /* 320836.pdf - AAJ124 - D0 - Core i7-900 Desktop Processor Extreme Edition Series and Intel Core i7-900 Desktop Processor Series */
+        || u == 0x000306A8 /* ?????????? - ?????? - ?? - Xeon E3-1220 v2 */
+        )
+        return true;
+    return false;
+}
+#endif
+
+
+/**
+ * Checks if the guest is in a suitable state for hardware-assisted VMX execution.
+ *
+ * @returns @c true if it is suitable, @c false otherwise.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pCtx    Pointer to the guest CPU context.
+ *
+ * @remarks @a pCtx can be a partial context and thus may not be necessarily the
+ *          same as pVCpu->cpum.GstCtx! Thus don't eliminate the @a pCtx parameter.
+ *          Secondly, if additional checks are added that require more of the CPU
+ *          state, make sure REM (which supplies a partial state) is updated.
+ */
+VMM_INT_DECL(bool) HMCanExecuteVmxGuest(PVMCC pVM, PVMCPUCC pVCpu, PCCPUMCTX pCtx)
+{
+    Assert(HMIsEnabled(pVM));
+    bool const fUnrestrictedGuest = CTX_EXPR(pVM->hm.s.vmx.fUnrestrictedGuestCfg, pVM->hmr0.s.vmx.fUnrestrictedGuest, RT_NOTHING);
+    Assert(   ( fUnrestrictedGuest && !pVM->hm.s.vmx.pRealModeTSS)
+           || (!fUnrestrictedGuest && pVM->hm.s.vmx.pRealModeTSS));
+
+    pVCpu->hm.s.fActive = false;
+
+    bool const fSupportsRealMode = fUnrestrictedGuest || PDMVmmDevHeapIsEnabled(pVM);
+    if (!fUnrestrictedGuest)
+    {
+        /*
+         * The VMM device heap is a requirement for emulating real mode or protected mode without paging with the unrestricted
+         * guest execution feature is missing (VT-x only).
+         */
+        if (fSupportsRealMode)
+        {
+            if (CPUMIsGuestInRealModeEx(pCtx))
+            {
+                /*
+                 * In V86 mode (VT-x or not), the CPU enforces real-mode compatible selector
+                 * bases, limits, and attributes, i.e. limit must be 64K, base must be selector * 16,
+                 * and attributes must be 0x9b for code and 0x93 for code segments.
+                 * If this is not true, we cannot execute real mode as V86 and have to fall
+                 * back to emulation.
+                 */
+                if (   pCtx->cs.Sel != (pCtx->cs.u64Base >> 4)
+                    || pCtx->ds.Sel != (pCtx->ds.u64Base >> 4)
+                    || pCtx->es.Sel != (pCtx->es.u64Base >> 4)
+                    || pCtx->ss.Sel != (pCtx->ss.u64Base >> 4)
+                    || pCtx->fs.Sel != (pCtx->fs.u64Base >> 4)
+                    || pCtx->gs.Sel != (pCtx->gs.u64Base >> 4))
+                {
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRmSelBase);
+                    return false;
+                }
+                if (   (pCtx->cs.u32Limit != 0xffff)
+                    || (pCtx->ds.u32Limit != 0xffff)
+                    || (pCtx->es.u32Limit != 0xffff)
+                    || (pCtx->ss.u32Limit != 0xffff)
+                    || (pCtx->fs.u32Limit != 0xffff)
+                    || (pCtx->gs.u32Limit != 0xffff))
+                {
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRmSelLimit);
+                    return false;
+                }
+                if (   (pCtx->cs.Attr.u != 0x9b)
+                    || (pCtx->ds.Attr.u != 0x93)
+                    || (pCtx->es.Attr.u != 0x93)
+                    || (pCtx->ss.Attr.u != 0x93)
+                    || (pCtx->fs.Attr.u != 0x93)
+                    || (pCtx->gs.Attr.u != 0x93))
+                {
+                    STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRmSelAttr);
+                    return false;
+                }
+                STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckRmOk);
+            }
+            else
+            {
+                /*
+                 * Verify the requirements for executing code in protected mode. VT-x can't
+                 * handle the CPU state right after a switch from real to protected mode
+                 * (all sorts of RPL & DPL assumptions).
+                 */
+                PCVMXVMCSINFOSHARED pVmcsInfo = hmGetVmxActiveVmcsInfoShared(pVCpu);
+                if (pVmcsInfo->fWasInRealMode)
+                {
+                    if (!CPUMIsGuestInV86ModeEx(pCtx))
+                    {
+                        /* The guest switched to protected mode, check if the state is suitable for VT-x. */
+                        if ((pCtx->cs.Sel & X86_SEL_RPL) != (pCtx->ss.Sel & X86_SEL_RPL))
+                        {
+                            STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadRpl);
+                            return false;
+                        }
+                        if (   !hmVmxIsCodeSelectorOk(&pCtx->cs, pCtx->ss.Attr.n.u2Dpl)
+                            || !hmVmxIsDataSelectorOk(&pCtx->ds)
+                            || !hmVmxIsDataSelectorOk(&pCtx->es)
+                            || !hmVmxIsDataSelectorOk(&pCtx->fs)
+                            || !hmVmxIsDataSelectorOk(&pCtx->gs)
+                            || !hmVmxIsStackSelectorOk(&pCtx->ss))
+                        {
+                            STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadSel);
+                            return false;
+                        }
+                    }
+                    else
+                    {
+                        /* The guest switched to V86 mode, check if the state is suitable for VT-x. */
+                        if (   pCtx->cs.Sel != (pCtx->cs.u64Base >> 4)
+                            || pCtx->ds.Sel != (pCtx->ds.u64Base >> 4)
+                            || pCtx->es.Sel != (pCtx->es.u64Base >> 4)
+                            || pCtx->ss.Sel != (pCtx->ss.u64Base >> 4)
+                            || pCtx->fs.Sel != (pCtx->fs.u64Base >> 4)
+                            || pCtx->gs.Sel != (pCtx->gs.u64Base >> 4))
+                        {
+                            STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadV86SelBase);
+                            return false;
+                        }
+                        if (   pCtx->cs.u32Limit != 0xffff
+                            || pCtx->ds.u32Limit != 0xffff
+                            || pCtx->es.u32Limit != 0xffff
+                            || pCtx->ss.u32Limit != 0xffff
+                            || pCtx->fs.u32Limit != 0xffff
+                            || pCtx->gs.u32Limit != 0xffff)
+                        {
+                            STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadV86SelLimit);
+                            return false;
+                        }
+                        if (   pCtx->cs.Attr.u != 0xf3
+                            || pCtx->ds.Attr.u != 0xf3
+                            || pCtx->es.Attr.u != 0xf3
+                            || pCtx->ss.Attr.u != 0xf3
+                            || pCtx->fs.Attr.u != 0xf3
+                            || pCtx->gs.Attr.u != 0xf3)
+                        {
+                            STAM_COUNTER_INC(&pVCpu->hm.s.StatVmxCheckBadV86SelAttr);
+                            return false;
+                        }
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (!CPUMIsGuestInLongModeEx(pCtx))
+            {
+                if (/* Requires a fake PD for real *and* protected mode without paging - stored in the VMM device heap: */
+                       !CTX_EXPR(pVM->hm.s.fNestedPagingCfg, pVM->hmr0.s.fNestedPaging, RT_NOTHING)
+                    /* Requires a fake TSS for real mode - stored in the VMM device heap: */
+                    || CPUMIsGuestInRealModeEx(pCtx))
+                    return false;
+
+                /* Too early for VT-x; Solaris guests will fail with a guru meditation otherwise; same for XP. */
+                if (pCtx->idtr.pIdt == 0 || pCtx->idtr.cbIdt == 0 || pCtx->tr.Sel == 0)
+                    return false;
+
+                /*
+                 * The guest is about to complete the switch to protected mode. Wait a bit longer.
+                 * Windows XP; switch to protected mode; all selectors are marked not present
+                 * in the hidden registers (possible recompiler bug; see load_seg_vm).
+                 */
+                /** @todo Is this supposed recompiler bug still relevant with IEM? */
+                if (pCtx->cs.Attr.n.u1Present == 0)
+                    return false;
+                if (pCtx->ss.Attr.n.u1Present == 0)
+                    return false;
+
+                /*
+                 * Windows XP: possible same as above, but new recompiler requires new
+                 * heuristics? VT-x doesn't seem to like something about the guest state and
+                 * this stuff avoids it.
+                 */
+                /** @todo This check is actually wrong, it doesn't take the direction of the
+                 *        stack segment into account. But, it does the job for now. */
+                if (pCtx->rsp >= pCtx->ss.u32Limit)
+                    return false;
+            }
+        }
+    }
+
+    if (pVM->hm.s.vmx.fEnabled)
+    {
+        /* If bit N is set in cr0_fixed0, then it must be set in the guest's cr0. */
+        uint32_t uCr0Mask = (uint32_t)CTX_EXPR(pVM->hm.s.ForR3.vmx.Msrs.u64Cr0Fixed0, g_HmMsrs.u.vmx.u64Cr0Fixed0, RT_NOTHING);
+
+        /* We ignore the NE bit here on purpose; see HMR0.cpp for details. */
+        uCr0Mask &= ~X86_CR0_NE;
+
+        if (fSupportsRealMode)
+        {
+            /* We ignore the PE & PG bits here on purpose; we emulate real and protected mode without paging. */
+            uCr0Mask &= ~(X86_CR0_PG | X86_CR0_PE);
+        }
+        else
+        {
+            /* We support protected mode without paging using identity mapping. */
+            uCr0Mask &= ~X86_CR0_PG;
+        }
+        if ((pCtx->cr0 & uCr0Mask) != uCr0Mask)
+            return false;
+
+        /* If bit N is cleared in cr0_fixed1, then it must be zero in the guest's cr0. */
+        uCr0Mask = (uint32_t)~CTX_EXPR(pVM->hm.s.ForR3.vmx.Msrs.u64Cr0Fixed1, g_HmMsrs.u.vmx.u64Cr0Fixed1, RT_NOTHING);
+        if ((pCtx->cr0 & uCr0Mask) != 0)
+            return false;
+
+        /* If bit N is set in cr4_fixed0, then it must be set in the guest's cr4. */
+        uCr0Mask  = (uint32_t)CTX_EXPR(pVM->hm.s.ForR3.vmx.Msrs.u64Cr4Fixed0, g_HmMsrs.u.vmx.u64Cr4Fixed0, RT_NOTHING);
+        uCr0Mask &= ~X86_CR4_VMXE;
+        if ((pCtx->cr4 & uCr0Mask) != uCr0Mask)
+            return false;
+
+        /* If bit N is cleared in cr4_fixed1, then it must be zero in the guest's cr4. */
+        uCr0Mask = (uint32_t)~CTX_EXPR(pVM->hm.s.ForR3.vmx.Msrs.u64Cr4Fixed1, g_HmMsrs.u.vmx.u64Cr4Fixed1, RT_NOTHING);
+        if ((pCtx->cr4 & uCr0Mask) != 0)
+            return false;
+
+        pVCpu->hm.s.fActive = true;
+        return true;
+    }
+
+    return false;
+}
+
+
+/**
+ * Dumps the virtual VMCS state to the release log.
+ *
+ * This is a purely a convenience function to output to the release log because
+ * cpumR3InfoVmxVmcs dumps only to the debug console and isn't always easy to use in
+ * case of a crash.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(void) HMDumpHwvirtVmxState(PVMCPU pVCpu)
+{
+    /* The string width of -4 used in the macros below to cover 'LDTR', 'GDTR', 'IDTR. */
+#define HMVMX_DUMP_HOST_XDTR(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        LogRel(("  %s%-4s                       = {base=%016RX64}\n", \
+            (a_pszPrefix), (a_SegName), (a_pVmcs)->u64Host##a_Seg##Base.u)); \
+    } while (0)
+#define HMVMX_DUMP_HOST_FS_GS_TR(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        LogRel(("  %s%-4s                       = {%04x base=%016RX64}\n", \
+                (a_pszPrefix), (a_SegName), (a_pVmcs)->Host##a_Seg, (a_pVmcs)->u64Host##a_Seg##Base.u)); \
+    } while (0)
+#define HMVMX_DUMP_GUEST_SEGREG(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        LogRel(("  %s%-4s                       = {%04x base=%016RX64 limit=%08x flags=%04x}\n", \
+                (a_pszPrefix), (a_SegName), (a_pVmcs)->Guest##a_Seg, (a_pVmcs)->u64Guest##a_Seg##Base.u, \
+                (a_pVmcs)->u32Guest##a_Seg##Limit, (a_pVmcs)->u32Guest##a_Seg##Attr)); \
+    } while (0)
+#define HMVMX_DUMP_GUEST_XDTR(a_pVmcs, a_Seg, a_SegName, a_pszPrefix) \
+    do { \
+        LogRel(("  %s%-4s                       = {base=%016RX64 limit=%08x}\n", \
+                (a_pszPrefix), (a_SegName), (a_pVmcs)->u64Guest##a_Seg##Base.u, (a_pVmcs)->u32Guest##a_Seg##Limit)); \
+    } while (0)
+
+    PCCPUMCTX  const pCtx  = &pVCpu->cpum.GstCtx;
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    if (!pVmcs)
+    {
+        LogRel(("Virtual VMCS not allocated\n"));
+        return;
+    }
+    LogRel(("GCPhysVmxon                = %#RGp\n",     pCtx->hwvirt.vmx.GCPhysVmxon));
+    LogRel(("GCPhysVmcs                 = %#RGp\n",     pCtx->hwvirt.vmx.GCPhysVmcs));
+    LogRel(("GCPhysShadowVmcs           = %#RGp\n",     pCtx->hwvirt.vmx.GCPhysShadowVmcs));
+    LogRel(("enmDiag                    = %u (%s)\n",   pCtx->hwvirt.vmx.enmDiag, HMGetVmxDiagDesc(pCtx->hwvirt.vmx.enmDiag)));
+    LogRel(("uDiagAux                   = %#RX64\n",    pCtx->hwvirt.vmx.uDiagAux));
+    LogRel(("enmAbort                   = %u (%s)\n",   pCtx->hwvirt.vmx.enmAbort, VMXGetAbortDesc(pCtx->hwvirt.vmx.enmAbort)));
+    LogRel(("uAbortAux                  = %u (%#x)\n",  pCtx->hwvirt.vmx.uAbortAux, pCtx->hwvirt.vmx.uAbortAux));
+    LogRel(("fInVmxRootMode             = %RTbool\n",   pCtx->hwvirt.vmx.fInVmxRootMode));
+    LogRel(("fInVmxNonRootMode          = %RTbool\n",   pCtx->hwvirt.vmx.fInVmxNonRootMode));
+    LogRel(("fInterceptEvents           = %RTbool\n",   pCtx->hwvirt.vmx.fInterceptEvents));
+    LogRel(("fNmiUnblockingIret         = %RTbool\n",   pCtx->hwvirt.vmx.fNmiUnblockingIret));
+    LogRel(("uFirstPauseLoopTick        = %RX64\n",     pCtx->hwvirt.vmx.uFirstPauseLoopTick));
+    LogRel(("uPrevPauseTick             = %RX64\n",     pCtx->hwvirt.vmx.uPrevPauseTick));
+    LogRel(("uEntryTick                 = %RX64\n",     pCtx->hwvirt.vmx.uEntryTick));
+    LogRel(("offVirtApicWrite           = %#RX16\n",    pCtx->hwvirt.vmx.offVirtApicWrite));
+    LogRel(("fVirtNmiBlocking           = %RTbool\n",   pCtx->hwvirt.vmx.fVirtNmiBlocking));
+    LogRel(("VMCS cache:\n"));
+
+    const char *pszPrefix = "  ";
+    /* Header. */
+    {
+        LogRel(("%sHeader:\n", pszPrefix));
+        LogRel(("  %sVMCS revision id           = %#RX32\n",      pszPrefix, pVmcs->u32VmcsRevId));
+        LogRel(("  %sVMX-abort id               = %#RX32 (%s)\n", pszPrefix, pVmcs->enmVmxAbort, VMXGetAbortDesc(pVmcs->enmVmxAbort)));
+        LogRel(("  %sVMCS state                 = %#x (%s)\n",    pszPrefix, pVmcs->fVmcsState, VMXGetVmcsStateDesc(pVmcs->fVmcsState)));
+    }
+
+    /* Control fields. */
+    {
+        /* 16-bit. */
+        LogRel(("%sControl:\n", pszPrefix));
+        LogRel(("  %sVPID                       = %#RX16\n",   pszPrefix, pVmcs->u16Vpid));
+        LogRel(("  %sPosted intr notify vector  = %#RX16\n",   pszPrefix, pVmcs->u16PostIntNotifyVector));
+        LogRel(("  %sEPTP index                 = %#RX16\n",   pszPrefix, pVmcs->u16EptpIndex));
+
+        /* 32-bit. */
+        LogRel(("  %sPin ctls                   = %#RX32\n",   pszPrefix, pVmcs->u32PinCtls));
+        LogRel(("  %sProcessor ctls             = %#RX32\n",   pszPrefix, pVmcs->u32ProcCtls));
+        LogRel(("  %sSecondary processor ctls   = %#RX32\n",   pszPrefix, pVmcs->u32ProcCtls2));
+        LogRel(("  %sVM-exit ctls               = %#RX32\n",   pszPrefix, pVmcs->u32ExitCtls));
+        LogRel(("  %sVM-entry ctls              = %#RX32\n",   pszPrefix, pVmcs->u32EntryCtls));
+        LogRel(("  %sException bitmap           = %#RX32\n",   pszPrefix, pVmcs->u32XcptBitmap));
+        LogRel(("  %sPage-fault mask            = %#RX32\n",   pszPrefix, pVmcs->u32XcptPFMask));
+        LogRel(("  %sPage-fault match           = %#RX32\n",   pszPrefix, pVmcs->u32XcptPFMatch));
+        LogRel(("  %sCR3-target count           = %RU32\n",    pszPrefix, pVmcs->u32Cr3TargetCount));
+        LogRel(("  %sVM-exit MSR store count    = %RU32\n",    pszPrefix, pVmcs->u32ExitMsrStoreCount));
+        LogRel(("  %sVM-exit MSR load count     = %RU32\n",    pszPrefix, pVmcs->u32ExitMsrLoadCount));
+        LogRel(("  %sVM-entry MSR load count    = %RU32\n",    pszPrefix, pVmcs->u32EntryMsrLoadCount));
+        LogRel(("  %sVM-entry interruption info = %#RX32\n",   pszPrefix, pVmcs->u32EntryIntInfo));
+        {
+            uint32_t const fInfo = pVmcs->u32EntryIntInfo;
+            uint8_t  const uType = VMX_ENTRY_INT_INFO_TYPE(fInfo);
+            LogRel(("    %sValid                      = %RTbool\n",  pszPrefix, VMX_ENTRY_INT_INFO_IS_VALID(fInfo)));
+            LogRel(("    %sType                       = %#x (%s)\n", pszPrefix, uType, VMXGetEntryIntInfoTypeDesc(uType)));
+            LogRel(("    %sVector                     = %#x\n",      pszPrefix, VMX_ENTRY_INT_INFO_VECTOR(fInfo)));
+            LogRel(("    %sNMI-unblocking-IRET        = %RTbool\n",  pszPrefix, VMX_ENTRY_INT_INFO_IS_NMI_UNBLOCK_IRET(fInfo)));
+            LogRel(("    %sError-code valid           = %RTbool\n",  pszPrefix, VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(fInfo)));
+        }
+        LogRel(("  %sVM-entry xcpt error-code   = %#RX32\n",   pszPrefix, pVmcs->u32EntryXcptErrCode));
+        LogRel(("  %sVM-entry instr length      = %u byte(s)\n", pszPrefix, pVmcs->u32EntryInstrLen));
+        LogRel(("  %sTPR threshold              = %#RX32\n",   pszPrefix, pVmcs->u32TprThreshold));
+        LogRel(("  %sPLE gap                    = %#RX32\n",   pszPrefix, pVmcs->u32PleGap));
+        LogRel(("  %sPLE window                 = %#RX32\n",   pszPrefix, pVmcs->u32PleWindow));
+
+        /* 64-bit. */
+        LogRel(("  %sIO-bitmap A addr           = %#RX64\n",   pszPrefix, pVmcs->u64AddrIoBitmapA.u));
+        LogRel(("  %sIO-bitmap B addr           = %#RX64\n",   pszPrefix, pVmcs->u64AddrIoBitmapB.u));
+        LogRel(("  %sMSR-bitmap addr            = %#RX64\n",   pszPrefix, pVmcs->u64AddrMsrBitmap.u));
+        LogRel(("  %sVM-exit MSR store addr     = %#RX64\n",   pszPrefix, pVmcs->u64AddrExitMsrStore.u));
+        LogRel(("  %sVM-exit MSR load addr      = %#RX64\n",   pszPrefix, pVmcs->u64AddrExitMsrLoad.u));
+        LogRel(("  %sVM-entry MSR load addr     = %#RX64\n",   pszPrefix, pVmcs->u64AddrEntryMsrLoad.u));
+        LogRel(("  %sExecutive VMCS ptr         = %#RX64\n",   pszPrefix, pVmcs->u64ExecVmcsPtr.u));
+        LogRel(("  %sPML addr                   = %#RX64\n",   pszPrefix, pVmcs->u64AddrPml.u));
+        LogRel(("  %sTSC offset                 = %#RX64\n",   pszPrefix, pVmcs->u64TscOffset.u));
+        LogRel(("  %sVirtual-APIC addr          = %#RX64\n",   pszPrefix, pVmcs->u64AddrVirtApic.u));
+        LogRel(("  %sAPIC-access addr           = %#RX64\n",   pszPrefix, pVmcs->u64AddrApicAccess.u));
+        LogRel(("  %sPosted-intr desc addr      = %#RX64\n",   pszPrefix, pVmcs->u64AddrPostedIntDesc.u));
+        LogRel(("  %sVM-functions control       = %#RX64\n",   pszPrefix, pVmcs->u64VmFuncCtls.u));
+        LogRel(("  %sEPTP ptr                   = %#RX64\n",   pszPrefix, pVmcs->u64EptPtr.u));
+        LogRel(("  %sEOI-exit bitmap 0          = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap0.u));
+        LogRel(("  %sEOI-exit bitmap 1          = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap1.u));
+        LogRel(("  %sEOI-exit bitmap 2          = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap2.u));
+        LogRel(("  %sEOI-exit bitmap 3          = %#RX64\n",   pszPrefix, pVmcs->u64EoiExitBitmap3.u));
+        LogRel(("  %sEPTP-list addr             = %#RX64\n",   pszPrefix, pVmcs->u64AddrEptpList.u));
+        LogRel(("  %sVMREAD-bitmap addr         = %#RX64\n",   pszPrefix, pVmcs->u64AddrVmreadBitmap.u));
+        LogRel(("  %sVMWRITE-bitmap addr        = %#RX64\n",   pszPrefix, pVmcs->u64AddrVmwriteBitmap.u));
+        LogRel(("  %sVirt-Xcpt info addr        = %#RX64\n",   pszPrefix, pVmcs->u64AddrXcptVeInfo.u));
+        LogRel(("  %sXSS-exiting bitmap         = %#RX64\n",   pszPrefix, pVmcs->u64XssExitBitmap.u));
+        LogRel(("  %sENCLS-exiting bitmap       = %#RX64\n",   pszPrefix, pVmcs->u64EnclsExitBitmap.u));
+        LogRel(("  %sSPP table pointer          = %#RX64\n",   pszPrefix, pVmcs->u64SppTablePtr.u));
+        LogRel(("  %sTSC multiplier             = %#RX64\n",   pszPrefix, pVmcs->u64TscMultiplier.u));
+        LogRel(("  %sENCLV-exiting bitmap       = %#RX64\n",   pszPrefix, pVmcs->u64EnclvExitBitmap.u));
+
+        /* Natural width. */
+        LogRel(("  %sCR0 guest/host mask        = %#RX64\n",   pszPrefix, pVmcs->u64Cr0Mask.u));
+        LogRel(("  %sCR4 guest/host mask        = %#RX64\n",   pszPrefix, pVmcs->u64Cr4Mask.u));
+        LogRel(("  %sCR0 read shadow            = %#RX64\n",   pszPrefix, pVmcs->u64Cr0ReadShadow.u));
+        LogRel(("  %sCR4 read shadow            = %#RX64\n",   pszPrefix, pVmcs->u64Cr4ReadShadow.u));
+        LogRel(("  %sCR3-target 0               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target0.u));
+        LogRel(("  %sCR3-target 1               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target1.u));
+        LogRel(("  %sCR3-target 2               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target2.u));
+        LogRel(("  %sCR3-target 3               = %#RX64\n",   pszPrefix, pVmcs->u64Cr3Target3.u));
+    }
+
+    /* Guest state. */
+    {
+        LogRel(("%sGuest state:\n", pszPrefix));
+
+        /* 16-bit. */
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Cs,   "cs",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Ss,   "ss",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Es,   "es",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Ds,   "ds",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Fs,   "fs",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Gs,   "gs",   pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Ldtr, "ldtr", pszPrefix);
+        HMVMX_DUMP_GUEST_SEGREG(pVmcs, Tr,   "tr",   pszPrefix);
+        HMVMX_DUMP_GUEST_XDTR(  pVmcs, Gdtr, "gdtr", pszPrefix);
+        HMVMX_DUMP_GUEST_XDTR(  pVmcs, Idtr, "idtr", pszPrefix);
+        LogRel(("  %sInterrupt status           = %#RX16\n",   pszPrefix, pVmcs->u16GuestIntStatus));
+        LogRel(("  %sPML index                  = %#RX16\n",   pszPrefix, pVmcs->u16PmlIndex));
+
+        /* 32-bit. */
+        LogRel(("  %sInterruptibility state     = %#RX32\n",   pszPrefix, pVmcs->u32GuestIntrState));
+        LogRel(("  %sActivity state             = %#RX32\n",   pszPrefix, pVmcs->u32GuestActivityState));
+        LogRel(("  %sSMBASE                     = %#RX32\n",   pszPrefix, pVmcs->u32GuestSmBase));
+        LogRel(("  %sSysEnter CS                = %#RX32\n",   pszPrefix, pVmcs->u32GuestSysenterCS));
+        LogRel(("  %sVMX-preemption timer value = %#RX32\n",   pszPrefix, pVmcs->u32PreemptTimer));
+
+        /* 64-bit. */
+        LogRel(("  %sVMCS link ptr              = %#RX64\n",   pszPrefix, pVmcs->u64VmcsLinkPtr.u));
+        LogRel(("  %sDBGCTL                     = %#RX64\n",   pszPrefix, pVmcs->u64GuestDebugCtlMsr.u));
+        LogRel(("  %sPAT                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestPatMsr.u));
+        LogRel(("  %sEFER                       = %#RX64\n",   pszPrefix, pVmcs->u64GuestEferMsr.u));
+        LogRel(("  %sPERFGLOBALCTRL             = %#RX64\n",   pszPrefix, pVmcs->u64GuestPerfGlobalCtlMsr.u));
+        LogRel(("  %sPDPTE 0                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte0.u));
+        LogRel(("  %sPDPTE 1                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte1.u));
+        LogRel(("  %sPDPTE 2                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte2.u));
+        LogRel(("  %sPDPTE 3                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestPdpte3.u));
+        LogRel(("  %sBNDCFGS                    = %#RX64\n",   pszPrefix, pVmcs->u64GuestBndcfgsMsr.u));
+        LogRel(("  %sRTIT_CTL                   = %#RX64\n",   pszPrefix, pVmcs->u64GuestRtitCtlMsr.u));
+
+        /* Natural width. */
+        LogRel(("  %scr0                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestCr0.u));
+        LogRel(("  %scr3                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestCr3.u));
+        LogRel(("  %scr4                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestCr4.u));
+        LogRel(("  %sdr7                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestDr7.u));
+        LogRel(("  %srsp                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestRsp.u));
+        LogRel(("  %srip                        = %#RX64\n",   pszPrefix, pVmcs->u64GuestRip.u));
+        LogRel(("  %srflags                     = %#RX64\n",   pszPrefix, pVmcs->u64GuestRFlags.u));
+        LogRel(("  %sPending debug xcpts        = %#RX64\n",   pszPrefix, pVmcs->u64GuestPendingDbgXcpts.u));
+        LogRel(("  %sSysEnter ESP               = %#RX64\n",   pszPrefix, pVmcs->u64GuestSysenterEsp.u));
+        LogRel(("  %sSysEnter EIP               = %#RX64\n",   pszPrefix, pVmcs->u64GuestSysenterEip.u));
+    }
+
+    /* Host state. */
+    {
+        LogRel(("%sHost state:\n", pszPrefix));
+
+        /* 16-bit. */
+        LogRel(("  %scs                         = %#RX16\n",   pszPrefix, pVmcs->HostCs));
+        LogRel(("  %sss                         = %#RX16\n",   pszPrefix, pVmcs->HostSs));
+        LogRel(("  %sds                         = %#RX16\n",   pszPrefix, pVmcs->HostDs));
+        LogRel(("  %ses                         = %#RX16\n",   pszPrefix, pVmcs->HostEs));
+        HMVMX_DUMP_HOST_FS_GS_TR(pVmcs, Fs, "fs", pszPrefix);
+        HMVMX_DUMP_HOST_FS_GS_TR(pVmcs, Gs, "gs", pszPrefix);
+        HMVMX_DUMP_HOST_FS_GS_TR(pVmcs, Tr, "tr", pszPrefix);
+        HMVMX_DUMP_HOST_XDTR(pVmcs, Gdtr, "gdtr", pszPrefix);
+        HMVMX_DUMP_HOST_XDTR(pVmcs, Idtr, "idtr", pszPrefix);
+
+        /* 32-bit. */
+        LogRel(("  %sSysEnter CS                = %#RX32\n",   pszPrefix, pVmcs->u32HostSysenterCs));
+
+        /* 64-bit. */
+        LogRel(("  %sEFER                       = %#RX64\n",   pszPrefix, pVmcs->u64HostEferMsr.u));
+        LogRel(("  %sPAT                        = %#RX64\n",   pszPrefix, pVmcs->u64HostPatMsr.u));
+        LogRel(("  %sPERFGLOBALCTRL             = %#RX64\n",   pszPrefix, pVmcs->u64HostPerfGlobalCtlMsr.u));
+
+        /* Natural width. */
+        LogRel(("  %scr0                        = %#RX64\n",   pszPrefix, pVmcs->u64HostCr0.u));
+        LogRel(("  %scr3                        = %#RX64\n",   pszPrefix, pVmcs->u64HostCr3.u));
+        LogRel(("  %scr4                        = %#RX64\n",   pszPrefix, pVmcs->u64HostCr4.u));
+        LogRel(("  %sSysEnter ESP               = %#RX64\n",   pszPrefix, pVmcs->u64HostSysenterEsp.u));
+        LogRel(("  %sSysEnter EIP               = %#RX64\n",   pszPrefix, pVmcs->u64HostSysenterEip.u));
+        LogRel(("  %srsp                        = %#RX64\n",   pszPrefix, pVmcs->u64HostRsp.u));
+        LogRel(("  %srip                        = %#RX64\n",   pszPrefix, pVmcs->u64HostRip.u));
+    }
+
+    /* Read-only fields. */
+    {
+        LogRel(("%sRead-only data fields:\n", pszPrefix));
+
+        /* 16-bit (none currently). */
+
+        /* 32-bit. */
+        uint32_t const uExitReason = pVmcs->u32RoExitReason;
+        LogRel(("  %sExit reason                = %u (%s)\n",  pszPrefix, uExitReason, HMGetVmxExitName(uExitReason)));
+        LogRel(("  %sExit qualification         = %#RX64\n",   pszPrefix, pVmcs->u64RoExitQual.u));
+        LogRel(("  %sVM-instruction error       = %#RX32\n",   pszPrefix, pVmcs->u32RoVmInstrError));
+        LogRel(("  %sVM-exit intr info          = %#RX32\n",   pszPrefix, pVmcs->u32RoExitIntInfo));
+        {
+            uint32_t const fInfo = pVmcs->u32RoExitIntInfo;
+            uint8_t  const uType = VMX_EXIT_INT_INFO_TYPE(fInfo);
+            LogRel(("    %sValid                      = %RTbool\n",  pszPrefix, VMX_EXIT_INT_INFO_IS_VALID(fInfo)));
+            LogRel(("    %sType                       = %#x (%s)\n", pszPrefix, uType, VMXGetExitIntInfoTypeDesc(uType)));
+            LogRel(("    %sVector                     = %#x\n",      pszPrefix, VMX_EXIT_INT_INFO_VECTOR(fInfo)));
+            LogRel(("    %sNMI-unblocking-IRET        = %RTbool\n",  pszPrefix, VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(fInfo)));
+            LogRel(("    %sError-code valid           = %RTbool\n",  pszPrefix, VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(fInfo)));
+        }
+        LogRel(("  %sVM-exit intr error-code    = %#RX32\n",   pszPrefix, pVmcs->u32RoExitIntErrCode));
+        LogRel(("  %sIDT-vectoring info         = %#RX32\n",   pszPrefix, pVmcs->u32RoIdtVectoringInfo));
+        {
+            uint32_t const fInfo = pVmcs->u32RoIdtVectoringInfo;
+            uint8_t  const uType = VMX_IDT_VECTORING_INFO_TYPE(fInfo);
+            LogRel(("    %sValid                      = %RTbool\n",  pszPrefix, VMX_IDT_VECTORING_INFO_IS_VALID(fInfo)));
+            LogRel(("    %sType                       = %#x (%s)\n", pszPrefix, uType, VMXGetIdtVectoringInfoTypeDesc(uType)));
+            LogRel(("    %sVector                     = %#x\n",      pszPrefix, VMX_IDT_VECTORING_INFO_VECTOR(fInfo)));
+            LogRel(("    %sError-code valid           = %RTbool\n",  pszPrefix, VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(fInfo)));
+        }
+        LogRel(("  %sIDT-vectoring error-code   = %#RX32\n",   pszPrefix, pVmcs->u32RoIdtVectoringErrCode));
+        LogRel(("  %sVM-exit instruction length = %u bytes\n", pszPrefix, pVmcs->u32RoExitInstrLen));
+        LogRel(("  %sVM-exit instruction info   = %#RX64\n",   pszPrefix, pVmcs->u32RoExitInstrInfo));
+
+        /* 64-bit. */
+        LogRel(("  %sGuest-physical addr        = %#RX64\n",   pszPrefix, pVmcs->u64RoGuestPhysAddr.u));
+
+        /* Natural width. */
+        LogRel(("  %sI/O RCX                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRcx.u));
+        LogRel(("  %sI/O RSI                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRsi.u));
+        LogRel(("  %sI/O RDI                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRdi.u));
+        LogRel(("  %sI/O RIP                    = %#RX64\n",   pszPrefix, pVmcs->u64RoIoRip.u));
+        LogRel(("  %sGuest-linear addr          = %#RX64\n",   pszPrefix, pVmcs->u64RoGuestLinearAddr.u));
+    }
+
+#undef HMVMX_DUMP_HOST_XDTR
+#undef HMVMX_DUMP_HOST_FS_GS_TR
+#undef HMVMX_DUMP_GUEST_SEGREG
+#undef HMVMX_DUMP_GUEST_XDTR
+}
+
+
+/**
+ * Gets the active (in use) VMCS info. object for the specified VCPU.
+ *
+ * This is either the guest or nested-guest VMCS info. and need not necessarily
+ * pertain to the "current" VMCS (in the VMX definition of the term). For instance,
+ * if the VM-entry failed due to an invalid-guest state, we may have "cleared" the
+ * current VMCS while returning to ring-3. However, the VMCS info. object for that
+ * VMCS would still be active and returned here so that we could dump the VMCS
+ * fields to ring-3 for diagnostics. This function is thus only used to
+ * distinguish between the nested-guest or guest VMCS.
+ *
+ * @returns The active VMCS information.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @thread  EMT.
+ * @remarks This function may be called with preemption or interrupts disabled!
+ */
+VMM_INT_DECL(PVMXVMCSINFOSHARED) hmGetVmxActiveVmcsInfoShared(PVMCPUCC pVCpu)
+{
+#ifdef IN_RING0
+    if (!pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs)
+#else
+    if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcsCopyForRing3)
+#endif
+        return &pVCpu->hm.s.vmx.VmcsInfo;
+    return &pVCpu->hm.s.vmx.VmcsInfoNstGst;
+}
+
+
+/**
+ * Converts a VMX event type into an appropriate TRPM event type.
+ *
+ * @returns TRPM event.
+ * @param   uIntInfo    The VMX event.
+ */
+VMM_INT_DECL(TRPMEVENT) HMVmxEventTypeToTrpmEventType(uint32_t uIntInfo)
+{
+    Assert(VMX_IDT_VECTORING_INFO_IS_VALID(uIntInfo));
+
+    TRPMEVENT enmTrapType;
+    uint8_t const uType   = VMX_IDT_VECTORING_INFO_TYPE(uIntInfo);
+    uint8_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(uIntInfo);
+
+    switch (uType)
+    {
+        case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
+           enmTrapType = TRPM_HARDWARE_INT;
+           break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_NMI:
+        case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
+            enmTrapType = TRPM_TRAP;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:  /* INT1 (ICEBP). */
+            Assert(uVector == X86_XCPT_DB); NOREF(uVector);
+            enmTrapType = TRPM_SOFTWARE_INT;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:       /* INT3 (#BP) and INTO (#OF) */
+            Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF); NOREF(uVector);
+            enmTrapType = TRPM_SOFTWARE_INT;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
+            enmTrapType = TRPM_SOFTWARE_INT;
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid trap type %#x\n", uType));
+            enmTrapType = TRPM_32BIT_HACK;
+            break;
+    }
+
+    return enmTrapType;
+}
+
+
+/**
+ * Converts a TRPM event type into an appropriate VMX event type.
+ *
+ * @returns VMX event type mask.
+ * @param   uVector         The event vector.
+ * @param   enmTrpmEvent    The TRPM event.
+ * @param   fIcebp          Whether the \#DB vector is caused by an INT1/ICEBP
+ *                          instruction.
+ */
+VMM_INT_DECL(uint32_t) HMTrpmEventTypeToVmxEventType(uint8_t uVector, TRPMEVENT enmTrpmEvent, bool fIcebp)
+{
+    uint32_t uIntInfoType = 0;
+    if (enmTrpmEvent == TRPM_TRAP)
+    {
+        Assert(!fIcebp);
+        switch (uVector)
+        {
+            case X86_XCPT_NMI:
+                uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_NMI << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                break;
+
+            case X86_XCPT_BP:
+            case X86_XCPT_OF:
+                uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                break;
+
+            case X86_XCPT_PF:
+            case X86_XCPT_DF:
+            case X86_XCPT_TS:
+            case X86_XCPT_NP:
+            case X86_XCPT_SS:
+            case X86_XCPT_GP:
+            case X86_XCPT_AC:
+                uIntInfoType |= VMX_IDT_VECTORING_INFO_ERROR_CODE_VALID;
+                RT_FALL_THRU();
+            default:
+                uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                break;
+        }
+    }
+    else if (enmTrpmEvent == TRPM_HARDWARE_INT)
+    {
+        Assert(!fIcebp);
+        uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_EXT_INT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+    }
+    else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
+    {
+        switch (uVector)
+        {
+            case X86_XCPT_BP:
+            case X86_XCPT_OF:
+                uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                break;
+
+            case X86_XCPT_DB:
+            {
+                if (fIcebp)
+                {
+                    uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                    break;
+                }
+                RT_FALL_THRU();
+            }
+            default:
+                uIntInfoType |= (VMX_IDT_VECTORING_INFO_TYPE_SW_INT << VMX_IDT_VECTORING_INFO_TYPE_SHIFT);
+                break;
+        }
+    }
+    else
+        AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
+    return uIntInfoType;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Notification callback for when a VM-exit happens outside VMX R0 code (e.g. in
+ * IEM).
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks Can be called from ring-0 as well as ring-3.
+ */
+VMM_INT_DECL(void) HMNotifyVmxNstGstVmexit(PVMCPU pVCpu)
+{
+    LogFlowFunc(("\n"));
+
+    /*
+     * Transitions to ring-3 flag a full CPU-state change except if we transition to ring-3
+     * in response to a physical CPU interrupt as no changes to the guest-CPU state are
+     * expected (see VINF_EM_RAW_INTERRUPT handling in hmR0VmxExitToRing3).
+     *
+     * However, with nested-guests, the state -can- change on trips to ring-3 for we might
+     * try to inject a nested-guest physical interrupt and cause a VMX_EXIT_EXT_INT VM-exit
+     * for the nested-guest from ring-3.
+     *
+     * Signalling reload of just the guest-CPU state that changed with the VM-exit is -not-
+     * sufficient since HM also needs to reload state related to VM-entry/VM-exit controls
+     * etc. So signal reloading of the entire state. It does not seem worth making this any
+     * more fine grained at the moment.
+     */
+    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_ALL);
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+    /*
+     * Make sure we need to merge the guest VMCS controls with the nested-guest
+     * VMCS controls on the next nested-guest VM-entry.
+     */
+    pVCpu->hm.s.vmx.fMergedNstGstCtls = false;
+
+    /*
+     * Flush the TLB before entering the outer guest execution (mainly required since the
+     * APIC-access guest-physical address would have changed and probably more things in
+     * the future).
+     */
+    pVCpu->hm.s.vmx.fSwitchedNstGstFlushTlb = true;
+
+    /** @todo Handle releasing of the page-mapping lock later. */
+#if 0
+    if (pVCpu->hm.s.vmx.fVirtApicPageLocked)
+    {
+        PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->hm.s.vmx.PgMapLockVirtApic);
+        pVCpu->hm.s.vmx.fVirtApicPageLocked = false;
+    }
+#endif
+}
+
+
+/**
+ * Notification callback for when the nested hypervisor's current VMCS is loaded or
+ * changed outside VMX R0 code (e.g. in IEM).
+ *
+ * This need -not- be called for modifications to the nested hypervisor's current
+ * VMCS when the guest is in VMX non-root mode as VMCS shadowing is not applicable
+ * there.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks Can be called from ring-0 as well as ring-3.
+ */
+VMM_INT_DECL(void) HMNotifyVmxNstGstCurrentVmcsChanged(PVMCPU pVCpu)
+{
+    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+    ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, CPUMCTX_EXTRN_HWVIRT);
+
+    /*
+     * Make sure we need to copy the nested hypervisor's current VMCS into the shadow VMCS
+     * on the next guest VM-entry.
+     */
+    pVCpu->hm.s.vmx.fCopiedNstGstToShadowVmcs = false;
+}
+
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAll.cpp b/src/VBox/VMM/VMMAll/IEMAll.cpp
new file mode 100644
index 00000000..106c51c5
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAll.cpp
@@ -0,0 +1,11599 @@
+/* $Id: IEMAll.cpp $ */
+/** @file
+ * IEM - Interpreted Execution Manager - All Contexts.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/** @page pg_iem    IEM - Interpreted Execution Manager
+ *
+ * The interpreted exeuction manager (IEM) is for executing short guest code
+ * sequences that are causing too many exits / virtualization traps.  It will
+ * also be used to interpret single instructions, thus replacing the selective
+ * interpreters in EM and IOM.
+ *
+ * Design goals:
+ *      - Relatively small footprint, although we favour speed and correctness
+ *        over size.
+ *      - Reasonably fast.
+ *      - Correctly handle lock prefixed instructions.
+ *      - Complete instruction set - eventually.
+ *      - Refactorable into a recompiler, maybe.
+ *      - Replace EMInterpret*.
+ *
+ * Using the existing disassembler has been considered, however this is thought
+ * to conflict with speed as the disassembler chews things a bit too much while
+ * leaving us with a somewhat complicated state to interpret afterwards.
+ *
+ *
+ * The current code is very much work in progress. You've been warned!
+ *
+ *
+ * @section sec_iem_fpu_instr   FPU Instructions
+ *
+ * On x86 and AMD64 hosts, the FPU instructions are implemented by executing the
+ * same or equivalent instructions on the host FPU.  To make life easy, we also
+ * let the FPU prioritize the unmasked exceptions for us.  This however, only
+ * works reliably when CR0.NE is set, i.e. when using \#MF instead the IRQ 13
+ * for FPU exception delivery, because with CR0.NE=0 there is a window where we
+ * can trigger spurious FPU exceptions.
+ *
+ * The guest FPU state is not loaded into the host CPU and kept there till we
+ * leave IEM because the calling conventions have declared an all year open
+ * season on much of the FPU state.  For instance an innocent looking call to
+ * memcpy might end up using a whole bunch of XMM or MM registers if the
+ * particular implementation finds it worthwhile.
+ *
+ *
+ * @section sec_iem_logging     Logging
+ *
+ * The IEM code uses the \"IEM\" log group for the main logging. The different
+ * logging levels/flags are generally used for the following purposes:
+ *      - Level 1  (Log)  : Errors, exceptions, interrupts and such major events.
+ *      - Flow  (LogFlow) : Basic enter/exit IEM state info.
+ *      - Level 2  (Log2) : ?
+ *      - Level 3  (Log3) : More detailed enter/exit IEM state info.
+ *      - Level 4  (Log4) : Decoding mnemonics w/ EIP.
+ *      - Level 5  (Log5) : Decoding details.
+ *      - Level 6  (Log6) : Enables/disables the lockstep comparison with REM.
+ *      - Level 7  (Log7) : iret++ execution logging.
+ *      - Level 8  (Log8) : Memory writes.
+ *      - Level 9  (Log9) : Memory reads.
+ *      - Level 10 (Log10): TLBs.
+ *      - Level 11 (Log11): Unmasked FPU exceptions.
+ *
+ * The SVM (AMD-V) and VMX (VT-x) code has the following assignments:
+ *      - Level 1  (Log)  : Errors and other major events.
+ *      - Flow (LogFlow)  : Misc flow stuff (cleanup?)
+ *      - Level 2  (Log2) : VM exits.
+ */
+
+/* Disabled warning C4505: 'iemRaisePageFaultJmp' : unreferenced local function has been removed */
+#ifdef _MSC_VER
+# pragma warning(disable:4505)
+#endif
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP   LOG_GROUP_IEM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/gim.h>
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+# include <VBox/vmm/em.h>
+# include <VBox/vmm/hm_svm.h>
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+# include <VBox/vmm/hmvmxinline.h>
+#endif
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/dbgftrace.h>
+#include "IEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <iprt/asm-math.h>
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+# include <iprt/asm-amd64-x86.h>
+#elif defined(RT_ARCH_ARM64) || defined(RT_ARCH_ARM32)
+# include <iprt/asm-arm.h>
+#endif
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/x86.h>
+
+#include "IEMInline.h"
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * CPU exception classes.
+ */
+typedef enum IEMXCPTCLASS
+{
+    IEMXCPTCLASS_BENIGN,
+    IEMXCPTCLASS_CONTRIBUTORY,
+    IEMXCPTCLASS_PAGE_FAULT,
+    IEMXCPTCLASS_DOUBLE_FAULT
+} IEMXCPTCLASS;
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#if defined(IEM_LOG_MEMORY_WRITES)
+/** What IEM just wrote. */
+uint8_t g_abIemWrote[256];
+/** How much IEM just wrote. */
+size_t g_cbIemWrote;
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static VBOXSTRICTRC    iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
+                                                 uint8_t uXcpt, uint16_t uErrorCode) RT_NOEXCEPT;
+
+
+/**
+ * Slow path of iemInitDecoder() and iemInitExec() that checks what kind of
+ * breakpoints are enabled.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ */
+void iemInitPendingBreakpointsSlow(PVMCPUCC pVCpu)
+{
+    /*
+     * Process guest breakpoints.
+     */
+#define PROCESS_ONE_BP(a_fDr7, a_iBp) do { \
+        if (a_fDr7 & X86_DR7_L_G(a_iBp)) \
+        { \
+            switch (X86_DR7_GET_RW(a_fDr7, a_iBp)) \
+            { \
+                case X86_DR7_RW_EO: \
+                    pVCpu->iem.s.fPendingInstructionBreakpoints = true; \
+                    break; \
+                case X86_DR7_RW_WO: \
+                case X86_DR7_RW_RW: \
+                    pVCpu->iem.s.fPendingDataBreakpoints = true; \
+                    break; \
+                case X86_DR7_RW_IO: \
+                    pVCpu->iem.s.fPendingIoBreakpoints = true; \
+                    break; \
+            } \
+        } \
+    } while (0)
+    uint32_t const fGstDr7 = (uint32_t)pVCpu->cpum.GstCtx.dr[7];
+    if (fGstDr7 & X86_DR7_ENABLED_MASK)
+    {
+        PROCESS_ONE_BP(fGstDr7, 0);
+        PROCESS_ONE_BP(fGstDr7, 1);
+        PROCESS_ONE_BP(fGstDr7, 2);
+        PROCESS_ONE_BP(fGstDr7, 3);
+    }
+
+    /*
+     * Process hypervisor breakpoints.
+     */
+    uint32_t const fHyperDr7 = DBGFBpGetDR7(pVCpu->CTX_SUFF(pVM));
+    if (fHyperDr7 & X86_DR7_ENABLED_MASK)
+    {
+        PROCESS_ONE_BP(fHyperDr7, 0);
+        PROCESS_ONE_BP(fHyperDr7, 1);
+        PROCESS_ONE_BP(fHyperDr7, 2);
+        PROCESS_ONE_BP(fHyperDr7, 3);
+    }
+}
+
+
+/**
+ * Initializes the decoder state.
+ *
+ * iemReInitDecoder is mostly a copy of this function.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   fBypassHandlers     Whether to bypass access handlers.
+ * @param   fDisregardLock      Whether to disregard the LOCK prefix.
+ */
+DECLINLINE(void) iemInitDecoder(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock)
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
+
+    pVCpu->iem.s.uCpl               = CPUMGetGuestCPL(pVCpu);
+    IEMMODE enmMode = iemCalcCpuMode(pVCpu);
+    pVCpu->iem.s.enmCpuMode         = enmMode;
+    pVCpu->iem.s.enmDefAddrMode     = enmMode;  /** @todo check if this is correct... */
+    pVCpu->iem.s.enmEffAddrMode     = enmMode;
+    if (enmMode != IEMMODE_64BIT)
+    {
+        pVCpu->iem.s.enmDefOpSize   = enmMode;  /** @todo check if this is correct... */
+        pVCpu->iem.s.enmEffOpSize   = enmMode;
+    }
+    else
+    {
+        pVCpu->iem.s.enmDefOpSize   = IEMMODE_32BIT;
+        pVCpu->iem.s.enmEffOpSize   = IEMMODE_32BIT;
+    }
+    pVCpu->iem.s.fPrefixes          = 0;
+    pVCpu->iem.s.uRexReg            = 0;
+    pVCpu->iem.s.uRexB              = 0;
+    pVCpu->iem.s.uRexIndex          = 0;
+    pVCpu->iem.s.idxPrefix          = 0;
+    pVCpu->iem.s.uVex3rdReg         = 0;
+    pVCpu->iem.s.uVexLength         = 0;
+    pVCpu->iem.s.fEvexStuff         = 0;
+    pVCpu->iem.s.iEffSeg            = X86_SREG_DS;
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf         = NULL;
+    pVCpu->iem.s.offInstrNextByte   = 0;
+    pVCpu->iem.s.offCurInstrStart   = 0;
+# ifdef VBOX_STRICT
+    pVCpu->iem.s.cbInstrBuf         = UINT16_MAX;
+    pVCpu->iem.s.cbInstrBufTotal    = UINT16_MAX;
+    pVCpu->iem.s.uInstrBufPc        = UINT64_C(0xc0ffc0ffcff0c0ff);
+# endif
+#else
+    pVCpu->iem.s.offOpcode          = 0;
+    pVCpu->iem.s.cbOpcode           = 0;
+#endif
+    pVCpu->iem.s.offModRm           = 0;
+    pVCpu->iem.s.cActiveMappings    = 0;
+    pVCpu->iem.s.iNextMapping       = 0;
+    pVCpu->iem.s.rcPassUp           = VINF_SUCCESS;
+    pVCpu->iem.s.fBypassHandlers    = fBypassHandlers;
+    pVCpu->iem.s.fDisregardLock     = fDisregardLock;
+    pVCpu->iem.s.fPendingInstructionBreakpoints = false;
+    pVCpu->iem.s.fPendingDataBreakpoints        = false;
+    pVCpu->iem.s.fPendingIoBreakpoints          = false;
+    if (RT_LIKELY(   !(pVCpu->cpum.GstCtx.dr[7] & X86_DR7_ENABLED_MASK)
+                  && pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledHwBreakpoints == 0))
+    { /* likely */ }
+    else
+        iemInitPendingBreakpointsSlow(pVCpu);
+
+#ifdef DBGFTRACE_ENABLED
+    switch (enmMode)
+    {
+        case IEMMODE_64BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
+            break;
+        case IEMMODE_32BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
+            break;
+        case IEMMODE_16BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
+            break;
+    }
+#endif
+}
+
+
+/**
+ * Reinitializes the decoder state 2nd+ loop of IEMExecLots.
+ *
+ * This is mostly a copy of iemInitDecoder.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ */
+DECLINLINE(void) iemReInitDecoder(PVMCPUCC pVCpu)
+{
+    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.fs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.gs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.tr));
+
+    pVCpu->iem.s.uCpl               = CPUMGetGuestCPL(pVCpu);   /** @todo this should be updated during execution! */
+    IEMMODE enmMode = iemCalcCpuMode(pVCpu);
+    pVCpu->iem.s.enmCpuMode         = enmMode;                  /** @todo this should be updated during execution! */
+    pVCpu->iem.s.enmDefAddrMode     = enmMode;  /** @todo check if this is correct... */
+    pVCpu->iem.s.enmEffAddrMode     = enmMode;
+    if (enmMode != IEMMODE_64BIT)
+    {
+        pVCpu->iem.s.enmDefOpSize   = enmMode;  /** @todo check if this is correct... */
+        pVCpu->iem.s.enmEffOpSize   = enmMode;
+    }
+    else
+    {
+        pVCpu->iem.s.enmDefOpSize   = IEMMODE_32BIT;
+        pVCpu->iem.s.enmEffOpSize   = IEMMODE_32BIT;
+    }
+    pVCpu->iem.s.fPrefixes          = 0;
+    pVCpu->iem.s.uRexReg            = 0;
+    pVCpu->iem.s.uRexB              = 0;
+    pVCpu->iem.s.uRexIndex          = 0;
+    pVCpu->iem.s.idxPrefix          = 0;
+    pVCpu->iem.s.uVex3rdReg         = 0;
+    pVCpu->iem.s.uVexLength         = 0;
+    pVCpu->iem.s.fEvexStuff         = 0;
+    pVCpu->iem.s.iEffSeg            = X86_SREG_DS;
+#ifdef IEM_WITH_CODE_TLB
+    if (pVCpu->iem.s.pbInstrBuf)
+    {
+        uint64_t off = (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+                        ? pVCpu->cpum.GstCtx.rip
+                        : pVCpu->cpum.GstCtx.eip + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base)
+                     - pVCpu->iem.s.uInstrBufPc;
+        if (off < pVCpu->iem.s.cbInstrBufTotal)
+        {
+            pVCpu->iem.s.offInstrNextByte = (uint32_t)off;
+            pVCpu->iem.s.offCurInstrStart = (uint16_t)off;
+            if ((uint16_t)off + 15 <= pVCpu->iem.s.cbInstrBufTotal)
+                pVCpu->iem.s.cbInstrBuf = (uint16_t)off + 15;
+            else
+                pVCpu->iem.s.cbInstrBuf = pVCpu->iem.s.cbInstrBufTotal;
+        }
+        else
+        {
+            pVCpu->iem.s.pbInstrBuf       = NULL;
+            pVCpu->iem.s.offInstrNextByte = 0;
+            pVCpu->iem.s.offCurInstrStart = 0;
+            pVCpu->iem.s.cbInstrBuf       = 0;
+            pVCpu->iem.s.cbInstrBufTotal  = 0;
+        }
+    }
+    else
+    {
+        pVCpu->iem.s.offInstrNextByte = 0;
+        pVCpu->iem.s.offCurInstrStart = 0;
+        pVCpu->iem.s.cbInstrBuf       = 0;
+        pVCpu->iem.s.cbInstrBufTotal  = 0;
+    }
+#else
+    pVCpu->iem.s.cbOpcode           = 0;
+    pVCpu->iem.s.offOpcode          = 0;
+#endif
+    pVCpu->iem.s.offModRm           = 0;
+    Assert(pVCpu->iem.s.cActiveMappings == 0);
+    pVCpu->iem.s.iNextMapping       = 0;
+    Assert(pVCpu->iem.s.rcPassUp   == VINF_SUCCESS);
+    Assert(pVCpu->iem.s.fBypassHandlers == false);
+
+#ifdef DBGFTRACE_ENABLED
+    switch (enmMode)
+    {
+        case IEMMODE_64BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I64/%u %08llx", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.rip);
+            break;
+        case IEMMODE_32BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I32/%u %04x:%08x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
+            break;
+        case IEMMODE_16BIT:
+            RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "I16/%u %04x:%04x", pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip);
+            break;
+    }
+#endif
+}
+
+
+
+/**
+ * Prefetch opcodes the first time when starting executing.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   fBypassHandlers     Whether to bypass access handlers.
+ * @param   fDisregardLock      Whether to disregard LOCK prefixes.
+ *
+ * @todo    Combine fDisregardLock and fBypassHandlers into a flag parameter and
+ *          store them as such.
+ */
+static VBOXSTRICTRC iemInitDecoderAndPrefetchOpcodes(PVMCPUCC pVCpu, bool fBypassHandlers, bool fDisregardLock) RT_NOEXCEPT
+{
+    iemInitDecoder(pVCpu, fBypassHandlers, fDisregardLock);
+
+#ifndef IEM_WITH_CODE_TLB
+    /*
+     * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
+     *
+     * First translate CS:rIP to a physical address.
+     *
+     * Note! The iemOpcodeFetchMoreBytes code depends on this here code to fetch
+     *       all relevant bytes from the first page, as it ASSUMES it's only ever
+     *       called for dealing with CS.LIM, page crossing and instructions that
+     *       are too long.
+     */
+    uint32_t    cbToTryRead;
+    RTGCPTR     GCPtrPC;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        cbToTryRead = GUEST_PAGE_SIZE;
+        GCPtrPC     = pVCpu->cpum.GstCtx.rip;
+        if (IEM_IS_CANONICAL(GCPtrPC))
+            cbToTryRead = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    else
+    {
+        uint32_t GCPtrPC32 = pVCpu->cpum.GstCtx.eip;
+        AssertMsg(!(GCPtrPC32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT, ("%04x:%RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+        if (GCPtrPC32 <= pVCpu->cpum.GstCtx.cs.u32Limit)
+            cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrPC32 + 1;
+        else
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        if (cbToTryRead) { /* likely */ }
+        else /* overflowed */
+        {
+            Assert(GCPtrPC32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
+            cbToTryRead = UINT32_MAX;
+        }
+        GCPtrPC = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrPC32;
+        Assert(GCPtrPC <= UINT32_MAX);
+    }
+
+    PGMPTWALK Walk;
+    int rc = PGMGstGetPage(pVCpu, GCPtrPC, &Walk);
+    if (RT_SUCCESS(rc))
+        Assert(Walk.fSucceeded); /* probable. */
+    else
+    {
+        Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - rc=%Rrc\n", GCPtrPC, rc));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        if (Walk.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
+# endif
+        return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, rc);
+    }
+    if ((Walk.fEffective & X86_PTE_US) || pVCpu->iem.s.uCpl != 3) { /* likely */ }
+    else
+    {
+        Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - supervisor page\n", GCPtrPC));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        if (Walk.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+# endif
+        return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+    }
+    if (!(Walk.fEffective & X86_PTE_PAE_NX) || !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE)) { /* likely */ }
+    else
+    {
+        Log(("iemInitDecoderAndPrefetchOpcodes: %RGv - NX\n", GCPtrPC));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        if (Walk.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+# endif
+        return iemRaisePageFault(pVCpu, GCPtrPC, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+    }
+    RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
+    /** @todo Check reserved bits and such stuff. PGM is better at doing
+     *        that, so do it when implementing the guest virtual address
+     *        TLB... */
+
+    /*
+     * Read the bytes at this address.
+     */
+    uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrPC & GUEST_PAGE_OFFSET_MASK);
+    if (cbToTryRead > cbLeftOnPage)
+        cbToTryRead = cbLeftOnPage;
+    if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode))
+        cbToTryRead = sizeof(pVCpu->iem.s.abOpcode);
+
+    if (!pVCpu->iem.s.fBypassHandlers)
+    {
+        VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, pVCpu->iem.s.abOpcode, cbToTryRead, PGMACCESSORIGIN_IEM);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* likely */ }
+        else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+        {
+            Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        }
+        else
+        {
+            Log((RT_SUCCESS(rcStrict)
+                 ? "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                 : "iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                 GCPtrPC, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            return rcStrict;
+        }
+    }
+    else
+    {
+        rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pVCpu->iem.s.abOpcode, GCPhys, cbToTryRead);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            Log(("iemInitDecoderAndPrefetchOpcodes: %RGv/%RGp LB %#x - read error - rc=%Rrc (!!)\n",
+                 GCPtrPC, GCPhys, rc, cbToTryRead));
+            return rc;
+        }
+    }
+    pVCpu->iem.s.cbOpcode = cbToTryRead;
+#endif /* !IEM_WITH_CODE_TLB */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Invalidates the IEM TLBs.
+ *
+ * This is called internally as well as by PGM when moving GC mappings.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling
+ *                      thread.
+ */
+VMM_INT_DECL(void) IEMTlbInvalidateAll(PVMCPUCC pVCpu)
+{
+#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
+    Log10(("IEMTlbInvalidateAll\n"));
+# ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.cbInstrBufTotal = 0;
+    pVCpu->iem.s.CodeTlb.uTlbRevision += IEMTLB_REVISION_INCR;
+    if (pVCpu->iem.s.CodeTlb.uTlbRevision != 0)
+    { /* very likely */ }
+    else
+    {
+        pVCpu->iem.s.CodeTlb.uTlbRevision = IEMTLB_REVISION_INCR;
+        unsigned i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
+        while (i-- > 0)
+            pVCpu->iem.s.CodeTlb.aEntries[i].uTag = 0;
+    }
+# endif
+
+# ifdef IEM_WITH_DATA_TLB
+    pVCpu->iem.s.DataTlb.uTlbRevision += IEMTLB_REVISION_INCR;
+    if (pVCpu->iem.s.DataTlb.uTlbRevision != 0)
+    { /* very likely */ }
+    else
+    {
+        pVCpu->iem.s.DataTlb.uTlbRevision = IEMTLB_REVISION_INCR;
+        unsigned i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
+        while (i-- > 0)
+            pVCpu->iem.s.DataTlb.aEntries[i].uTag = 0;
+    }
+# endif
+#else
+    RT_NOREF(pVCpu);
+#endif
+}
+
+
+/**
+ * Invalidates a page in the TLBs.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling
+ *                      thread.
+ * @param   GCPtr       The address of the page to invalidate
+ * @thread EMT(pVCpu)
+ */
+VMM_INT_DECL(void) IEMTlbInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtr)
+{
+#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
+    Log10(("IEMTlbInvalidatePage: GCPtr=%RGv\n", GCPtr));
+    GCPtr = IEMTLB_CALC_TAG_NO_REV(GCPtr);
+    Assert(!(GCPtr >> (48 - X86_PAGE_SHIFT)));
+    uintptr_t const idx = IEMTLB_TAG_TO_INDEX(GCPtr);
+
+# ifdef IEM_WITH_CODE_TLB
+    if (pVCpu->iem.s.CodeTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.CodeTlb.uTlbRevision))
+    {
+        pVCpu->iem.s.CodeTlb.aEntries[idx].uTag = 0;
+        if (GCPtr == IEMTLB_CALC_TAG_NO_REV(pVCpu->iem.s.uInstrBufPc))
+            pVCpu->iem.s.cbInstrBufTotal = 0;
+    }
+# endif
+
+# ifdef IEM_WITH_DATA_TLB
+    if (pVCpu->iem.s.DataTlb.aEntries[idx].uTag == (GCPtr | pVCpu->iem.s.DataTlb.uTlbRevision))
+        pVCpu->iem.s.DataTlb.aEntries[idx].uTag = 0;
+# endif
+#else
+    NOREF(pVCpu); NOREF(GCPtr);
+#endif
+}
+
+
+#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
+/**
+ * Invalid both TLBs slow fashion following a rollover.
+ *
+ * Worker for IEMTlbInvalidateAllPhysical,
+ * IEMTlbInvalidateAllPhysicalAllCpus, iemOpcodeFetchBytesJmp, iemMemMap,
+ * iemMemMapJmp and others.
+ *
+ * @thread EMT(pVCpu)
+ */
+static void IEMTlbInvalidateAllPhysicalSlow(PVMCPUCC pVCpu)
+{
+    Log10(("IEMTlbInvalidateAllPhysicalSlow\n"));
+    ASMAtomicWriteU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
+    ASMAtomicWriteU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, IEMTLB_PHYS_REV_INCR * 2);
+
+    unsigned i;
+# ifdef IEM_WITH_CODE_TLB
+    i = RT_ELEMENTS(pVCpu->iem.s.CodeTlb.aEntries);
+    while (i-- > 0)
+    {
+        pVCpu->iem.s.CodeTlb.aEntries[i].pbMappingR3       = NULL;
+        pVCpu->iem.s.CodeTlb.aEntries[i].fFlagsAndPhysRev &= ~(  IEMTLBE_F_PG_NO_WRITE   | IEMTLBE_F_PG_NO_READ
+                                                               | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PHYS_REV);
+    }
+# endif
+# ifdef IEM_WITH_DATA_TLB
+    i = RT_ELEMENTS(pVCpu->iem.s.DataTlb.aEntries);
+    while (i-- > 0)
+    {
+        pVCpu->iem.s.DataTlb.aEntries[i].pbMappingR3       = NULL;
+        pVCpu->iem.s.DataTlb.aEntries[i].fFlagsAndPhysRev &= ~(  IEMTLBE_F_PG_NO_WRITE   | IEMTLBE_F_PG_NO_READ
+                                                               | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PHYS_REV);
+    }
+# endif
+
+}
+#endif
+
+
+/**
+ * Invalidates the host physical aspects of the IEM TLBs.
+ *
+ * This is called internally as well as by PGM when moving GC mappings.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling
+ *                      thread.
+ * @note    Currently not used.
+ */
+VMM_INT_DECL(void) IEMTlbInvalidateAllPhysical(PVMCPUCC pVCpu)
+{
+#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
+    /* Note! This probably won't end up looking exactly like this, but it give an idea... */
+    Log10(("IEMTlbInvalidateAllPhysical\n"));
+
+# ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.cbInstrBufTotal = 0;
+# endif
+    uint64_t uTlbPhysRev = pVCpu->iem.s.CodeTlb.uTlbPhysRev + IEMTLB_PHYS_REV_INCR;
+    if (RT_LIKELY(uTlbPhysRev > IEMTLB_PHYS_REV_INCR * 2))
+    {
+        pVCpu->iem.s.CodeTlb.uTlbPhysRev = uTlbPhysRev;
+        pVCpu->iem.s.DataTlb.uTlbPhysRev = uTlbPhysRev;
+    }
+    else
+        IEMTlbInvalidateAllPhysicalSlow(pVCpu);
+#else
+    NOREF(pVCpu);
+#endif
+}
+
+
+/**
+ * Invalidates the host physical aspects of the IEM TLBs.
+ *
+ * This is called internally as well as by PGM when moving GC mappings.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpuCaller The ID of the calling EMT if available to the caller,
+ *                      otherwise NIL_VMCPUID.
+ *
+ * @remarks Caller holds the PGM lock.
+ */
+VMM_INT_DECL(void) IEMTlbInvalidateAllPhysicalAllCpus(PVMCC pVM, VMCPUID idCpuCaller)
+{
+#if defined(IEM_WITH_CODE_TLB) || defined(IEM_WITH_DATA_TLB)
+    PVMCPUCC const pVCpuCaller = idCpuCaller >= pVM->cCpus ? VMMGetCpu(pVM) : VMMGetCpuById(pVM, idCpuCaller);
+    if (pVCpuCaller)
+        VMCPU_ASSERT_EMT(pVCpuCaller);
+    Log10(("IEMTlbInvalidateAllPhysicalAllCpus\n"));
+
+    VMCC_FOR_EACH_VMCPU(pVM)
+    {
+# ifdef IEM_WITH_CODE_TLB
+        if (pVCpuCaller == pVCpu)
+            pVCpu->iem.s.cbInstrBufTotal = 0;
+# endif
+
+        uint64_t const uTlbPhysRevPrev = ASMAtomicUoReadU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev);
+        uint64_t       uTlbPhysRevNew  = uTlbPhysRevPrev + IEMTLB_PHYS_REV_INCR;
+        if (RT_LIKELY(uTlbPhysRevNew > IEMTLB_PHYS_REV_INCR * 2))
+        { /* likely */}
+        else if (pVCpuCaller == pVCpu)
+            uTlbPhysRevNew = IEMTLB_PHYS_REV_INCR;
+        else
+        {
+            IEMTlbInvalidateAllPhysicalSlow(pVCpu);
+            continue;
+        }
+        ASMAtomicCmpXchgU64(&pVCpu->iem.s.CodeTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
+        ASMAtomicCmpXchgU64(&pVCpu->iem.s.DataTlb.uTlbPhysRev, uTlbPhysRevNew, uTlbPhysRevPrev);
+    }
+    VMCC_FOR_EACH_VMCPU_END(pVM);
+
+#else
+    RT_NOREF(pVM, idCpuCaller);
+#endif
+}
+
+#ifdef IEM_WITH_CODE_TLB
+
+/**
+ * Tries to fetches @a cbDst opcode bytes, raise the appropriate exception on
+ * failure and jumps.
+ *
+ * We end up here for a number of reasons:
+ *      - pbInstrBuf isn't yet initialized.
+ *      - Advancing beyond the buffer boundrary (e.g. cross page).
+ *      - Advancing beyond the CS segment limit.
+ *      - Fetching from non-mappable page (e.g. MMIO).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   pvDst               Where to return the bytes.
+ * @param   cbDst               Number of bytes to read.
+ *
+ * @todo    Make cbDst = 0 a way of initializing pbInstrBuf?
+ */
+void iemOpcodeFetchBytesJmp(PVMCPUCC pVCpu, size_t cbDst, void *pvDst) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# ifdef IN_RING3
+    for (;;)
+    {
+        Assert(cbDst <= 8);
+        uint32_t offBuf = pVCpu->iem.s.offInstrNextByte;
+
+        /*
+         * We might have a partial buffer match, deal with that first to make the
+         * rest simpler.  This is the first part of the cross page/buffer case.
+         */
+        if (pVCpu->iem.s.pbInstrBuf != NULL)
+        {
+            if (offBuf < pVCpu->iem.s.cbInstrBuf)
+            {
+                Assert(offBuf + cbDst > pVCpu->iem.s.cbInstrBuf);
+                uint32_t const cbCopy = pVCpu->iem.s.cbInstrBuf - pVCpu->iem.s.offInstrNextByte;
+                memcpy(pvDst, &pVCpu->iem.s.pbInstrBuf[offBuf], cbCopy);
+
+                cbDst  -= cbCopy;
+                pvDst   = (uint8_t *)pvDst + cbCopy;
+                offBuf += cbCopy;
+                pVCpu->iem.s.offInstrNextByte += offBuf;
+            }
+        }
+
+        /*
+         * Check segment limit, figuring how much we're allowed to access at this point.
+         *
+         * We will fault immediately if RIP is past the segment limit / in non-canonical
+         * territory.  If we do continue, there are one or more bytes to read before we
+         * end up in trouble and we need to do that first before faulting.
+         */
+        RTGCPTR  GCPtrFirst;
+        uint32_t cbMaxRead;
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        {
+            GCPtrFirst = pVCpu->cpum.GstCtx.rip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
+            if (RT_LIKELY(IEM_IS_CANONICAL(GCPtrFirst)))
+            { /* likely */ }
+            else
+                iemRaiseGeneralProtectionFault0Jmp(pVCpu);
+            cbMaxRead = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
+        }
+        else
+        {
+            GCPtrFirst = pVCpu->cpum.GstCtx.eip + (offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart);
+            /* Assert(!(GCPtrFirst & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT); - this is allowed */
+            if (RT_LIKELY((uint32_t)GCPtrFirst <= pVCpu->cpum.GstCtx.cs.u32Limit))
+            { /* likely */ }
+            else /** @todo For CPUs older than the 386, we should not necessarily generate \#GP here but wrap around! */
+                iemRaiseSelectorBoundsJmp(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+            cbMaxRead = pVCpu->cpum.GstCtx.cs.u32Limit - (uint32_t)GCPtrFirst + 1;
+            if (cbMaxRead != 0)
+            { /* likely */ }
+            else
+            {
+                /* Overflowed because address is 0 and limit is max. */
+                Assert(GCPtrFirst == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
+                cbMaxRead = X86_PAGE_SIZE;
+            }
+            GCPtrFirst = (uint32_t)GCPtrFirst + (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base;
+            uint32_t cbMaxRead2 = X86_PAGE_SIZE - ((uint32_t)GCPtrFirst & X86_PAGE_OFFSET_MASK);
+            if (cbMaxRead2 < cbMaxRead)
+                cbMaxRead = cbMaxRead2;
+            /** @todo testcase: unreal modes, both huge 16-bit and 32-bit. */
+        }
+
+        /*
+         * Get the TLB entry for this piece of code.
+         */
+        uint64_t const     uTag  = IEMTLB_CALC_TAG(    &pVCpu->iem.s.CodeTlb, GCPtrFirst);
+        PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.CodeTlb, uTag);
+        if (pTlbe->uTag == uTag)
+        {
+            /* likely when executing lots of code, otherwise unlikely */
+# ifdef VBOX_WITH_STATISTICS
+            pVCpu->iem.s.CodeTlb.cTlbHits++;
+# endif
+        }
+        else
+        {
+            pVCpu->iem.s.CodeTlb.cTlbMisses++;
+            PGMPTWALK Walk;
+            int rc = PGMGstGetPage(pVCpu, GCPtrFirst, &Walk);
+            if (RT_FAILURE(rc))
+            {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+                /** @todo Nested VMX: Need to handle EPT violation/misconfig here?  */
+                Assert(!(Walk.fFailed & PGM_WALKFAIL_EPT));
+#endif
+                Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrFirst, rc));
+                iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, rc);
+            }
+
+            AssertCompile(IEMTLBE_F_PT_NO_EXEC == 1);
+            Assert(Walk.fSucceeded);
+            pTlbe->uTag             = uTag;
+            pTlbe->fFlagsAndPhysRev = (~Walk.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A))
+                                    | (Walk.fEffective >> X86_PTE_PAE_BIT_NX);
+            pTlbe->GCPhys           = Walk.GCPhys;
+            pTlbe->pbMappingR3      = NULL;
+        }
+
+        /*
+         * Check TLB page table level access flags.
+         */
+        if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_EXEC))
+        {
+            if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER) && pVCpu->iem.s.uCpl == 3)
+            {
+                Log(("iemOpcodeFetchBytesJmp: %RGv - supervisor page\n", GCPtrFirst));
+                iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+            }
+            if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_EXEC) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
+            {
+                Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrFirst));
+                iemRaisePageFaultJmp(pVCpu, GCPtrFirst, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+            }
+        }
+
+        /*
+         * Look up the physical page info if necessary.
+         */
+        if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
+        { /* not necessary */ }
+        else
+        {
+            AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE     == IEMTLBE_F_PG_NO_WRITE);
+            AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ      == IEMTLBE_F_PG_NO_READ);
+            AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
+            AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED   == IEMTLBE_F_PG_UNASSIGNED);
+            if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
+            { /* likely */ }
+            else
+                IEMTlbInvalidateAllPhysicalSlow(pVCpu);
+            pTlbe->fFlagsAndPhysRev &= ~(  IEMTLBE_F_PHYS_REV
+                                         | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_UNASSIGNED);
+            int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.CodeTlb.uTlbPhysRev,
+                                                &pTlbe->pbMappingR3, &pTlbe->fFlagsAndPhysRev);
+            AssertRCStmt(rc, IEM_DO_LONGJMP(pVCpu, rc));
+        }
+
+# if defined(IN_RING3) || defined(IN_RING0) /** @todo fixme */
+        /*
+         * Try do a direct read using the pbMappingR3 pointer.
+         */
+        if (    (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ))
+             == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
+        {
+            uint32_t const offPg = (GCPtrFirst & X86_PAGE_OFFSET_MASK);
+            pVCpu->iem.s.cbInstrBufTotal = offPg + cbMaxRead;
+            if (offBuf == (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart)
+            {
+                pVCpu->iem.s.cbInstrBuf       = offPg + RT_MIN(15, cbMaxRead);
+                pVCpu->iem.s.offCurInstrStart = (int16_t)offPg;
+            }
+            else
+            {
+                uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
+                if (cbInstr + (uint32_t)cbDst <= 15)
+                {
+                    pVCpu->iem.s.cbInstrBuf       = offPg + RT_MIN(cbMaxRead + cbInstr, 15) - cbInstr;
+                    pVCpu->iem.s.offCurInstrStart = (int16_t)(offPg - cbInstr);
+                }
+                else
+                {
+                    Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0)\n",
+                         pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, cbInstr, cbDst));
+                    iemRaiseGeneralProtectionFault0Jmp(pVCpu);
+                }
+            }
+            if (cbDst <= cbMaxRead)
+            {
+                pVCpu->iem.s.offInstrNextByte = offPg + (uint32_t)cbDst;
+                pVCpu->iem.s.uInstrBufPc      = GCPtrFirst & ~(RTGCPTR)X86_PAGE_OFFSET_MASK;
+                pVCpu->iem.s.pbInstrBuf       = pTlbe->pbMappingR3;
+                memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbDst);
+                return;
+            }
+            pVCpu->iem.s.pbInstrBuf = NULL;
+
+            memcpy(pvDst, &pTlbe->pbMappingR3[offPg], cbMaxRead);
+            pVCpu->iem.s.offInstrNextByte = offPg + cbMaxRead;
+        }
+# else
+#  error "refactor as needed"
+        /*
+         * If there is no special read handling, so we can read a bit more and
+         * put it in the prefetch buffer.
+         */
+        if (   cbDst < cbMaxRead
+            && (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PG_NO_READ)) == pVCpu->iem.s.CodeTlb.uTlbPhysRev)
+        {
+            VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys,
+                                                &pVCpu->iem.s.abOpcode[0], cbToTryRead, PGMACCESSORIGIN_IEM);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+            { /* likely */ }
+            else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+            {
+                Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                     GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+                rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                AssertStmt(rcStrict == VINF_SUCCESS, IEM_DO_LONGJMP(pVCpu, VBOXSTRICRC_VAL(rcStrict)));
+            }
+            else
+            {
+                Log((RT_SUCCESS(rcStrict)
+                     ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                     : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                     GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+                IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+            }
+        }
+# endif
+        /*
+         * Special read handling, so only read exactly what's needed.
+         * This is a highly unlikely scenario.
+         */
+        else
+        {
+            pVCpu->iem.s.CodeTlb.cTlbSlowReadPath++;
+
+            /* Check instruction length. */
+            uint32_t const cbInstr = offBuf - (uint32_t)(int32_t)pVCpu->iem.s.offCurInstrStart;
+            if (RT_LIKELY(cbInstr + cbDst <= 15))
+            { /* likely */ }
+            else
+            {
+                Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0) [slow]\n",
+                     pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, cbInstr, cbDst));
+                iemRaiseGeneralProtectionFault0Jmp(pVCpu);
+            }
+
+            /* Do the reading. */
+            uint32_t const cbToRead = RT_MIN((uint32_t)cbDst, cbMaxRead);
+            VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK),
+                                                pvDst, cbToRead, PGMACCESSORIGIN_IEM);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+            { /* likely */ }
+            else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+            {
+                Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                     GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
+                rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                AssertStmt(rcStrict == VINF_SUCCESS, IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict)));
+            }
+            else
+            {
+                Log((RT_SUCCESS(rcStrict)
+                     ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                     : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                     GCPtrFirst, pTlbe->GCPhys + (GCPtrFirst & X86_PAGE_OFFSET_MASK), VBOXSTRICTRC_VAL(rcStrict), cbToRead));
+                IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+            }
+            pVCpu->iem.s.offInstrNextByte = offBuf + cbToRead;
+            if (cbToRead == cbDst)
+                return;
+        }
+
+        /*
+         * More to read, loop.
+         */
+        cbDst -= cbMaxRead;
+        pvDst  = (uint8_t *)pvDst + cbMaxRead;
+    }
+# else  /* !IN_RING3 */
+    RT_NOREF(pvDst, cbDst);
+    if (pvDst || cbDst)
+        IEM_DO_LONGJMP(pVCpu, VERR_INTERNAL_ERROR);
+# endif /* !IN_RING3 */
+}
+
+#else
+
+/**
+ * Try fetch at least @a cbMin bytes more opcodes, raise the appropriate
+ * exception if it fails.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   cbMin               The minimum number of bytes relative offOpcode
+ *                              that must be read.
+ */
+VBOXSTRICTRC iemOpcodeFetchMoreBytes(PVMCPUCC pVCpu, size_t cbMin) RT_NOEXCEPT
+{
+    /*
+     * What we're doing here is very similar to iemMemMap/iemMemBounceBufferMap.
+     *
+     * First translate CS:rIP to a physical address.
+     */
+    uint8_t const   cbOpcode  = pVCpu->iem.s.cbOpcode;
+    uint8_t const   offOpcode = pVCpu->iem.s.offOpcode;
+    uint8_t const   cbLeft    = cbOpcode - offOpcode;
+    Assert(cbLeft < cbMin);
+    Assert(cbOpcode <= sizeof(pVCpu->iem.s.abOpcode));
+
+    uint32_t        cbToTryRead;
+    RTGCPTR         GCPtrNext;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        GCPtrNext   = pVCpu->cpum.GstCtx.rip + cbOpcode;
+        if (!IEM_IS_CANONICAL(GCPtrNext))
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        cbToTryRead = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
+    }
+    else
+    {
+        uint32_t GCPtrNext32 = pVCpu->cpum.GstCtx.eip;
+        /* Assert(!(GCPtrNext32 & ~(uint32_t)UINT16_MAX) || pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT); - this is allowed */
+        GCPtrNext32 += cbOpcode;
+        if (GCPtrNext32 > pVCpu->cpum.GstCtx.cs.u32Limit)
+            /** @todo For CPUs older than the 386, we should not generate \#GP here but wrap around! */
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        cbToTryRead = pVCpu->cpum.GstCtx.cs.u32Limit - GCPtrNext32 + 1;
+        if (!cbToTryRead) /* overflowed */
+        {
+            Assert(GCPtrNext32 == 0); Assert(pVCpu->cpum.GstCtx.cs.u32Limit == UINT32_MAX);
+            cbToTryRead = UINT32_MAX;
+            /** @todo check out wrapping around the code segment.  */
+        }
+        if (cbToTryRead < cbMin - cbLeft)
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        GCPtrNext = (uint32_t)pVCpu->cpum.GstCtx.cs.u64Base + GCPtrNext32;
+
+        uint32_t cbLeftOnPage = GUEST_PAGE_SIZE - (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
+        if (cbToTryRead > cbLeftOnPage)
+            cbToTryRead = cbLeftOnPage;
+    }
+
+    /* Restrict to opcode buffer space.
+
+       We're making ASSUMPTIONS here based on work done previously in
+       iemInitDecoderAndPrefetchOpcodes, where bytes from the first page will
+       be fetched in case of an instruction crossing two pages. */
+    if (cbToTryRead > sizeof(pVCpu->iem.s.abOpcode) - cbOpcode)
+        cbToTryRead = sizeof(pVCpu->iem.s.abOpcode) - cbOpcode;
+    if (RT_LIKELY(cbToTryRead + cbLeft >= cbMin))
+    { /* likely */ }
+    else
+    {
+        Log(("iemOpcodeFetchMoreBytes: %04x:%08RX64 LB %#x + %#zx -> #GP(0)\n",
+             pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip, offOpcode, cbMin));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    PGMPTWALK Walk;
+    int rc = PGMGstGetPage(pVCpu, GCPtrNext, &Walk);
+    if (RT_FAILURE(rc))
+    {
+        Log(("iemOpcodeFetchMoreBytes: %RGv - rc=%Rrc\n", GCPtrNext, rc));
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        if (Walk.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
+#endif
+        return iemRaisePageFault(pVCpu, GCPtrNext, 1, IEM_ACCESS_INSTRUCTION, rc);
+    }
+    if (!(Walk.fEffective & X86_PTE_US) && pVCpu->iem.s.uCpl == 3)
+    {
+        Log(("iemOpcodeFetchMoreBytes: %RGv - supervisor page\n", GCPtrNext));
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        if (Walk.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+#endif
+        return iemRaisePageFault(pVCpu, GCPtrNext, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+    }
+    if ((Walk.fEffective & X86_PTE_PAE_NX) && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE))
+    {
+        Log(("iemOpcodeFetchMoreBytes: %RGv - NX\n", GCPtrNext));
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        if (Walk.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, IEM_ACCESS_INSTRUCTION, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+#endif
+        return iemRaisePageFault(pVCpu, GCPtrNext, 1, IEM_ACCESS_INSTRUCTION, VERR_ACCESS_DENIED);
+    }
+    RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrNext & GUEST_PAGE_OFFSET_MASK);
+    Log5(("GCPtrNext=%RGv GCPhys=%RGp cbOpcodes=%#x\n",  GCPtrNext,  GCPhys,  cbOpcode));
+    /** @todo Check reserved bits and such stuff. PGM is better at doing
+     *        that, so do it when implementing the guest virtual address
+     *        TLB... */
+
+    /*
+     * Read the bytes at this address.
+     *
+     * We read all unpatched bytes in iemInitDecoderAndPrefetchOpcodes already,
+     * and since PATM should only patch the start of an instruction there
+     * should be no need to check again here.
+     */
+    if (!pVCpu->iem.s.fBypassHandlers)
+    {
+        VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhys, &pVCpu->iem.s.abOpcode[cbOpcode],
+                                            cbToTryRead, PGMACCESSORIGIN_IEM);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* likely */ }
+        else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+        {
+            Log(("iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status -  rcStrict=%Rrc\n",
+                 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        }
+        else
+        {
+            Log((RT_SUCCESS(rcStrict)
+                 ? "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read status - rcStrict=%Rrc\n"
+                 : "iemOpcodeFetchMoreBytes: %RGv/%RGp LB %#x - read error - rcStrict=%Rrc (!!)\n",
+                 GCPtrNext, GCPhys, VBOXSTRICTRC_VAL(rcStrict), cbToTryRead));
+            return rcStrict;
+        }
+    }
+    else
+    {
+        rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.abOpcode[cbOpcode], GCPhys, cbToTryRead);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            Log(("iemOpcodeFetchMoreBytes: %RGv - read error - rc=%Rrc (!!)\n", GCPtrNext, rc));
+            return rc;
+        }
+    }
+    pVCpu->iem.s.cbOpcode = cbOpcode + cbToTryRead;
+    Log5(("%.*Rhxs\n", pVCpu->iem.s.cbOpcode, pVCpu->iem.s.abOpcode));
+
+    return VINF_SUCCESS;
+}
+
+#endif /* !IEM_WITH_CODE_TLB */
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU8 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   pb                  Where to return the opcode byte.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU8Slow(PVMCPUCC pVCpu, uint8_t *pb) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pb = pVCpu->iem.s.abOpcode[offOpcode];
+        pVCpu->iem.s.offOpcode = offOpcode + 1;
+    }
+    else
+        *pb = 0;
+    return rcStrict;
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU8Jmp doesn't like, longjmp on error.
+ *
+ * @returns The opcode byte.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+uint8_t iemOpcodeGetNextU8SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint8_t u8;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u8), &u8);
+    return u8;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 1);
+    if (rcStrict == VINF_SUCCESS)
+        return pVCpu->iem.s.abOpcode[pVCpu->iem.s.offOpcode++];
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS8SxU16 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16                Where to return the opcode dword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextS8SxU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
+{
+    uint8_t      u8;
+    VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
+    if (rcStrict == VINF_SUCCESS)
+        *pu16 = (int8_t)u8;
+    return rcStrict;
+}
+
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS8SxU32 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode dword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextS8SxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
+{
+    uint8_t      u8;
+    VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
+    if (rcStrict == VINF_SUCCESS)
+        *pu32 = (int8_t)u8;
+    return rcStrict;
+}
+
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS8SxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode qword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextS8SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
+{
+    uint8_t      u8;
+    VBOXSTRICTRC rcStrict = iemOpcodeGetNextU8Slow(pVCpu, &u8);
+    if (rcStrict == VINF_SUCCESS)
+        *pu64 = (int8_t)u8;
+    return rcStrict;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16                Where to return the opcode word.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU16Slow(PVMCPUCC pVCpu, uint16_t *pu16) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu16 = *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu16 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+# endif
+        pVCpu->iem.s.offOpcode = offOpcode + 2;
+    }
+    else
+        *pu16 = 0;
+    return rcStrict;
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16Jmp doesn't like, longjmp on error
+ *
+ * @returns The opcode word.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+uint16_t iemOpcodeGetNextU16SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint16_t u16;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u16), &u16);
+    return u16;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        pVCpu->iem.s.offOpcode += 2;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint16_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+#  endif
+    }
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16ZxU32 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode double word.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU16ZxU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu32 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+        pVCpu->iem.s.offOpcode = offOpcode + 2;
+    }
+    else
+        *pu32 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU16ZxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode quad word.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU16ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 2);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu64 = RT_MAKE_U16(pVCpu->iem.s.abOpcode[offOpcode], pVCpu->iem.s.abOpcode[offOpcode + 1]);
+        pVCpu->iem.s.offOpcode = offOpcode + 2;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU32 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32                Where to return the opcode dword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU32Slow(PVMCPUCC pVCpu, uint32_t *pu32) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu32 = *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu32 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3]);
+# endif
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+    }
+    else
+        *pu32 = 0;
+    return rcStrict;
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU32Jmp doesn't like, longjmp on error.
+ *
+ * @returns The opcode dword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+uint32_t iemOpcodeGetNextU32SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint32_t u32;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u32), &u32);
+    return u32;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint32_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 3]);
+#  endif
+    }
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU32ZxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode dword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU32ZxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu64 = RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3]);
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextS32SxU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode qword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextS32SxU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 4);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        *pu64 = (int32_t)RT_MAKE_U32_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                             pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                             pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                             pVCpu->iem.s.abOpcode[offOpcode + 3]);
+        pVCpu->iem.s.offOpcode = offOpcode + 4;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+#endif /* !IEM_WITH_SETJMP */
+
+#ifndef IEM_WITH_SETJMP
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU64 doesn't like.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64                Where to return the opcode qword.
+ */
+VBOXSTRICTRC iemOpcodeGetNextU64Slow(PVMCPUCC pVCpu, uint64_t *pu64) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+# ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        *pu64 = *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+# else
+        *pu64 = RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 3],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 4],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 5],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 6],
+                                    pVCpu->iem.s.abOpcode[offOpcode + 7]);
+# endif
+        pVCpu->iem.s.offOpcode = offOpcode + 8;
+    }
+    else
+        *pu64 = 0;
+    return rcStrict;
+}
+
+#else  /* IEM_WITH_SETJMP */
+
+/**
+ * Deals with the problematic cases that iemOpcodeGetNextU64Jmp doesn't like, longjmp on error.
+ *
+ * @returns The opcode qword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+uint64_t iemOpcodeGetNextU64SlowJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# ifdef IEM_WITH_CODE_TLB
+    uint64_t u64;
+    iemOpcodeFetchBytesJmp(pVCpu, sizeof(u64), &u64);
+    return u64;
+# else
+    VBOXSTRICTRC rcStrict = iemOpcodeFetchMoreBytes(pVCpu, 8);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        uint8_t offOpcode = pVCpu->iem.s.offOpcode;
+        pVCpu->iem.s.offOpcode = offOpcode + 8;
+#  ifdef IEM_USE_UNALIGNED_DATA_ACCESS
+        return *(uint64_t const *)&pVCpu->iem.s.abOpcode[offOpcode];
+#  else
+        return RT_MAKE_U64_FROM_U8(pVCpu->iem.s.abOpcode[offOpcode],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 1],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 2],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 3],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 4],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 5],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 6],
+                                   pVCpu->iem.s.abOpcode[offOpcode + 7]);
+#  endif
+    }
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+# endif
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+
+
+/** @name  Misc Worker Functions.
+ * @{
+ */
+
+/**
+ * Gets the exception class for the specified exception vector.
+ *
+ * @returns The class of the specified exception.
+ * @param   uVector       The exception vector.
+ */
+static IEMXCPTCLASS iemGetXcptClass(uint8_t uVector) RT_NOEXCEPT
+{
+    Assert(uVector <= X86_XCPT_LAST);
+    switch (uVector)
+    {
+        case X86_XCPT_DE:
+        case X86_XCPT_TS:
+        case X86_XCPT_NP:
+        case X86_XCPT_SS:
+        case X86_XCPT_GP:
+        case X86_XCPT_SX:   /* AMD only */
+            return IEMXCPTCLASS_CONTRIBUTORY;
+
+        case X86_XCPT_PF:
+        case X86_XCPT_VE:   /* Intel only */
+            return IEMXCPTCLASS_PAGE_FAULT;
+
+        case X86_XCPT_DF:
+            return IEMXCPTCLASS_DOUBLE_FAULT;
+    }
+    return IEMXCPTCLASS_BENIGN;
+}
+
+
+/**
+ * Evaluates how to handle an exception caused during delivery of another event
+ * (exception / interrupt).
+ *
+ * @returns How to handle the recursive exception.
+ * @param   pVCpu               The cross context virtual CPU structure of the
+ *                              calling thread.
+ * @param   fPrevFlags          The flags of the previous event.
+ * @param   uPrevVector         The vector of the previous event.
+ * @param   fCurFlags           The flags of the current exception.
+ * @param   uCurVector          The vector of the current exception.
+ * @param   pfXcptRaiseInfo     Where to store additional information about the
+ *                              exception condition. Optional.
+ */
+VMM_INT_DECL(IEMXCPTRAISE) IEMEvaluateRecursiveXcpt(PVMCPUCC pVCpu, uint32_t fPrevFlags, uint8_t uPrevVector, uint32_t fCurFlags,
+                                                    uint8_t uCurVector, PIEMXCPTRAISEINFO pfXcptRaiseInfo)
+{
+    /*
+     * Only CPU exceptions can be raised while delivering other events, software interrupt
+     * (INTn/INT3/INTO/ICEBP) generated exceptions cannot occur as the current (second) exception.
+     */
+    AssertReturn(fCurFlags & IEM_XCPT_FLAGS_T_CPU_XCPT, IEMXCPTRAISE_INVALID);
+    Assert(pVCpu); RT_NOREF(pVCpu);
+    Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x\n", uPrevVector, uCurVector));
+
+    IEMXCPTRAISE     enmRaise   = IEMXCPTRAISE_CURRENT_XCPT;
+    IEMXCPTRAISEINFO fRaiseInfo = IEMXCPTRAISEINFO_NONE;
+    if (fPrevFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+    {
+        IEMXCPTCLASS enmPrevXcptClass = iemGetXcptClass(uPrevVector);
+        if (enmPrevXcptClass != IEMXCPTCLASS_BENIGN)
+        {
+            IEMXCPTCLASS enmCurXcptClass = iemGetXcptClass(uCurVector);
+            if (   enmPrevXcptClass == IEMXCPTCLASS_PAGE_FAULT
+                && (   enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT
+                    || enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY))
+            {
+                enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
+                fRaiseInfo = enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT ? IEMXCPTRAISEINFO_PF_PF
+                                                                        : IEMXCPTRAISEINFO_PF_CONTRIBUTORY_XCPT;
+                Log2(("IEMEvaluateRecursiveXcpt: Vectoring page fault. uPrevVector=%#x uCurVector=%#x uCr2=%#RX64\n", uPrevVector,
+                      uCurVector, pVCpu->cpum.GstCtx.cr2));
+            }
+            else if (   enmPrevXcptClass == IEMXCPTCLASS_CONTRIBUTORY
+                     && enmCurXcptClass  == IEMXCPTCLASS_CONTRIBUTORY)
+            {
+                enmRaise = IEMXCPTRAISE_DOUBLE_FAULT;
+                Log2(("IEMEvaluateRecursiveXcpt: uPrevVector=%#x uCurVector=%#x -> #DF\n", uPrevVector, uCurVector));
+            }
+            else if (   enmPrevXcptClass == IEMXCPTCLASS_DOUBLE_FAULT
+                     && (   enmCurXcptClass == IEMXCPTCLASS_CONTRIBUTORY
+                         || enmCurXcptClass == IEMXCPTCLASS_PAGE_FAULT))
+            {
+                enmRaise = IEMXCPTRAISE_TRIPLE_FAULT;
+                Log2(("IEMEvaluateRecursiveXcpt: #DF handler raised a %#x exception -> triple fault\n", uCurVector));
+            }
+        }
+        else
+        {
+            if (uPrevVector == X86_XCPT_NMI)
+            {
+                fRaiseInfo = IEMXCPTRAISEINFO_NMI_XCPT;
+                if (uCurVector == X86_XCPT_PF)
+                {
+                    fRaiseInfo |= IEMXCPTRAISEINFO_NMI_PF;
+                    Log2(("IEMEvaluateRecursiveXcpt: NMI delivery caused a page fault\n"));
+                }
+            }
+            else if (   uPrevVector == X86_XCPT_AC
+                     && uCurVector  == X86_XCPT_AC)
+            {
+                enmRaise   = IEMXCPTRAISE_CPU_HANG;
+                fRaiseInfo = IEMXCPTRAISEINFO_AC_AC;
+                Log2(("IEMEvaluateRecursiveXcpt: Recursive #AC - Bad guest\n"));
+            }
+        }
+    }
+    else if (fPrevFlags & IEM_XCPT_FLAGS_T_EXT_INT)
+    {
+        fRaiseInfo = IEMXCPTRAISEINFO_EXT_INT_XCPT;
+        if (uCurVector == X86_XCPT_PF)
+            fRaiseInfo |= IEMXCPTRAISEINFO_EXT_INT_PF;
+    }
+    else
+    {
+        Assert(fPrevFlags & IEM_XCPT_FLAGS_T_SOFT_INT);
+        fRaiseInfo = IEMXCPTRAISEINFO_SOFT_INT_XCPT;
+    }
+
+    if (pfXcptRaiseInfo)
+        *pfXcptRaiseInfo = fRaiseInfo;
+    return enmRaise;
+}
+
+
+/**
+ * Enters the CPU shutdown state initiated by a triple fault or other
+ * unrecoverable conditions.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling thread.
+ */
+static VBOXSTRICTRC iemInitiateCpuShutdown(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        IEM_VMX_VMEXIT_TRIPLE_FAULT_RET(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SHUTDOWN))
+    {
+        Log2(("shutdown: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SHUTDOWN, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    RT_NOREF(pVCpu);
+    return VINF_EM_TRIPLE_FAULT;
+}
+
+
+/**
+ * Validates a new SS segment.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling thread.
+ * @param   NewSS           The new SS selctor.
+ * @param   uCpl            The CPL to load the stack for.
+ * @param   pDesc           Where to return the descriptor.
+ */
+static VBOXSTRICTRC iemMiscValidateNewSS(PVMCPUCC pVCpu, RTSEL NewSS, uint8_t uCpl, PIEMSELDESC pDesc) RT_NOEXCEPT
+{
+    /* Null selectors are not allowed (we're not called for dispatching
+       interrupts with SS=0 in long mode). */
+    if (!(NewSS & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - null selector -> #TS(0)\n", NewSS));
+        return iemRaiseTaskSwitchFault0(pVCpu);
+    }
+
+    /** @todo testcase: check that the TSS.ssX RPL is checked.  Also check when. */
+    if ((NewSS & X86_SEL_RPL) != uCpl)
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - RPL and CPL (%d) differs -> #TS\n", NewSS, uCpl));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
+    }
+
+    /*
+     * Read the descriptor.
+     */
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, pDesc, NewSS, X86_XCPT_TS);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Perform the descriptor validation documented for LSS, POP SS and MOV SS.
+     */
+    if (!pDesc->Legacy.Gen.u1DescType)
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - system selector (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
+    }
+
+    if (    (pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
+        || !(pDesc->Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - code or read only (%#x) -> #TS\n", NewSS, pDesc->Legacy.Gen.u4Type));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
+    }
+    if (pDesc->Legacy.Gen.u2Dpl != uCpl)
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - DPL (%d) and CPL (%d) differs -> #TS\n", NewSS, pDesc->Legacy.Gen.u2Dpl, uCpl));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, NewSS);
+    }
+
+    /* Is it there? */
+    /** @todo testcase: Is this checked before the canonical / limit check below? */
+    if (!pDesc->Legacy.Gen.u1Present)
+    {
+        Log(("iemMiscValidateNewSSandRsp: %#x - segment not present -> #NP\n", NewSS));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, NewSS);
+    }
+
+    return VINF_SUCCESS;
+}
+
+/** @} */
+
+
+/** @name  Raising Exceptions.
+ *
+ * @{
+ */
+
+
+/**
+ * Loads the specified stack far pointer from the TSS.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   uCpl            The CPL to load the stack for.
+ * @param   pSelSS          Where to return the new stack segment.
+ * @param   puEsp           Where to return the new stack pointer.
+ */
+static VBOXSTRICTRC iemRaiseLoadStackFromTss32Or16(PVMCPUCC pVCpu, uint8_t uCpl, PRTSEL pSelSS, uint32_t *puEsp) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict;
+    Assert(uCpl < 4);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
+    switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
+    {
+        /*
+         * 16-bit TSS (X86TSS16).
+         */
+        case X86_SEL_TYPE_SYS_286_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
+        case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+        {
+            uint32_t off = uCpl * 4 + 2;
+            if (off + 4 <= pVCpu->cpum.GstCtx.tr.u32Limit)
+            {
+                /** @todo check actual access pattern here. */
+                uint32_t u32Tmp = 0; /* gcc maybe... */
+                rcStrict = iemMemFetchSysU32(pVCpu, &u32Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    *puEsp  = RT_LOWORD(u32Tmp);
+                    *pSelSS = RT_HIWORD(u32Tmp);
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+            {
+                Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
+                rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
+            }
+            break;
+        }
+
+        /*
+         * 32-bit TSS (X86TSS32).
+         */
+        case X86_SEL_TYPE_SYS_386_TSS_AVAIL: AssertFailed(); RT_FALL_THRU();
+        case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+        {
+            uint32_t off = uCpl * 8 + 4;
+            if (off + 7 <= pVCpu->cpum.GstCtx.tr.u32Limit)
+            {
+/** @todo check actual access pattern here. */
+                uint64_t u64Tmp;
+                rcStrict = iemMemFetchSysU64(pVCpu, &u64Tmp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    *puEsp  = u64Tmp & UINT32_MAX;
+                    *pSelSS = (RTSEL)(u64Tmp >> 32);
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+            {
+                Log(("LoadStackFromTss32Or16: out of bounds! uCpl=%d, u32Limit=%#x TSS16\n", uCpl, pVCpu->cpum.GstCtx.tr.u32Limit));
+                rcStrict = iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
+            }
+            break;
+        }
+
+        default:
+            AssertFailed();
+            rcStrict = VERR_IEM_IPE_4;
+            break;
+    }
+
+    *puEsp  = 0; /* make gcc happy */
+    *pSelSS = 0; /* make gcc happy */
+    return rcStrict;
+}
+
+
+/**
+ * Loads the specified stack pointer from the 64-bit TSS.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   uCpl            The CPL to load the stack for.
+ * @param   uIst            The interrupt stack table index, 0 if to use uCpl.
+ * @param   puRsp           Where to return the new stack pointer.
+ */
+static VBOXSTRICTRC iemRaiseLoadStackFromTss64(PVMCPUCC pVCpu, uint8_t uCpl, uint8_t uIst, uint64_t *puRsp) RT_NOEXCEPT
+{
+    Assert(uCpl < 4);
+    Assert(uIst < 8);
+    *puRsp  = 0; /* make gcc happy */
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
+    AssertReturn(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY, VERR_IEM_IPE_5);
+
+    uint32_t off;
+    if (uIst)
+        off = (uIst - 1) * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, ist1);
+    else
+        off = uCpl * sizeof(uint64_t) + RT_UOFFSETOF(X86TSS64, rsp0);
+    if (off + sizeof(uint64_t) > pVCpu->cpum.GstCtx.tr.u32Limit)
+    {
+        Log(("iemRaiseLoadStackFromTss64: out of bounds! uCpl=%d uIst=%d, u32Limit=%#x\n", uCpl, uIst, pVCpu->cpum.GstCtx.tr.u32Limit));
+        return iemRaiseTaskSwitchFaultCurrentTSS(pVCpu);
+    }
+
+    return iemMemFetchSysU64(pVCpu, puRsp, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + off);
+}
+
+
+/**
+ * Adjust the CPU state according to the exception being raised.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   u8Vector        The exception that has been raised.
+ */
+DECLINLINE(void) iemRaiseXcptAdjustState(PVMCPUCC pVCpu, uint8_t u8Vector)
+{
+    switch (u8Vector)
+    {
+        case X86_XCPT_DB:
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
+            pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
+            break;
+        /** @todo Read the AMD and Intel exception reference... */
+    }
+}
+
+
+/**
+ * Implements exceptions and interrupts for real mode.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr         The number of bytes to offset rIP by in the return
+ *                          address.
+ * @param   u8Vector        The interrupt / exception vector number.
+ * @param   fFlags          The flags.
+ * @param   uErr            The error value if IEM_XCPT_FLAGS_ERR is set.
+ * @param   uCr2            The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
+ */
+static VBOXSTRICTRC
+iemRaiseXcptOrIntInRealMode(PVMCPUCC      pVCpu,
+                            uint8_t     cbInstr,
+                            uint8_t     u8Vector,
+                            uint32_t    fFlags,
+                            uint16_t    uErr,
+                            uint64_t    uCr2) RT_NOEXCEPT
+{
+    NOREF(uErr); NOREF(uCr2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    /*
+     * Read the IDT entry.
+     */
+    if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(4) * u8Vector + 3)
+    {
+        Log(("RaiseXcptOrIntInRealMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
+        return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+    RTFAR16 Idte;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, (uint32_t *)&Idte, UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(4) * u8Vector);
+    if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
+    {
+        Log(("iemRaiseXcptOrIntInRealMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * Push the stack frame.
+     */
+    uint16_t *pu16Frame;
+    uint64_t  uNewRsp;
+    rcStrict = iemMemStackPushBeginSpecial(pVCpu, 6, 3, (void **)&pu16Frame, &uNewRsp);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
+#if IEM_CFG_TARGET_CPU == IEMTARGETCPU_DYNAMIC
+    AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
+    if (pVCpu->iem.s.uTargetCpu <= IEMTARGETCPU_186)
+        fEfl |= UINT16_C(0xf000);
+#endif
+    pu16Frame[2] = (uint16_t)fEfl;
+    pu16Frame[1] = (uint16_t)pVCpu->cpum.GstCtx.cs.Sel;
+    pu16Frame[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
+    rcStrict = iemMemStackPushCommitSpecial(pVCpu, pu16Frame, uNewRsp);
+    if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
+        return rcStrict;
+
+    /*
+     * Load the vector address into cs:ip and make exception specific state
+     * adjustments.
+     */
+    pVCpu->cpum.GstCtx.cs.Sel           = Idte.sel;
+    pVCpu->cpum.GstCtx.cs.ValidSel      = Idte.sel;
+    pVCpu->cpum.GstCtx.cs.fFlags        = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.u64Base       = (uint32_t)Idte.sel << 4;
+    /** @todo do we load attribs and limit as well? Should we check against limit like far jump? */
+    pVCpu->cpum.GstCtx.rip              = Idte.off;
+    fEfl &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_AC);
+    IEMMISC_SET_EFL(pVCpu, fEfl);
+
+    /** @todo do we actually do this in real mode? */
+    if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+        iemRaiseXcptAdjustState(pVCpu, u8Vector);
+
+    return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a NULL data selector into when coming from V8086 mode.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   pSReg           Pointer to the segment register.
+ */
+DECLINLINE(void) iemHlpLoadNullDataSelectorOnV86Xcpt(PVMCPUCC pVCpu, PCPUMSELREG pSReg)
+{
+    pSReg->Sel      = 0;
+    pSReg->ValidSel = 0;
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
+    {
+        /* VT-x (Intel 3960x) doesn't change the base and limit, clears and sets the following attributes */
+        pSReg->Attr.u &= X86DESCATTR_DT | X86DESCATTR_TYPE | X86DESCATTR_DPL | X86DESCATTR_G | X86DESCATTR_D;
+        pSReg->Attr.u |= X86DESCATTR_UNUSABLE;
+    }
+    else
+    {
+        pSReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+        /** @todo check this on AMD-V */
+        pSReg->u64Base  = 0;
+        pSReg->u32Limit = 0;
+    }
+}
+
+
+/**
+ * Loads a segment selector during a task switch in V8086 mode.
+ *
+ * @param   pSReg           Pointer to the segment register.
+ * @param   uSel            The selector value to load.
+ */
+DECLINLINE(void) iemHlpLoadSelectorInV86Mode(PCPUMSELREG pSReg, uint16_t uSel)
+{
+    /* See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
+    pSReg->Sel      = uSel;
+    pSReg->ValidSel = uSel;
+    pSReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+    pSReg->u64Base  = uSel << 4;
+    pSReg->u32Limit = 0xffff;
+    pSReg->Attr.u   = 0xf3;
+}
+
+
+/**
+ * Loads a segment selector during a task switch in protected mode.
+ *
+ * In this task switch scenario, we would throw \#TS exceptions rather than
+ * \#GPs.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   pSReg           Pointer to the segment register.
+ * @param   uSel            The new selector value.
+ *
+ * @remarks This does _not_ handle CS or SS.
+ * @remarks This expects pVCpu->iem.s.uCpl to be up to date.
+ */
+static VBOXSTRICTRC iemHlpTaskSwitchLoadDataSelectorInProtMode(PVMCPUCC pVCpu, PCPUMSELREG pSReg, uint16_t uSel) RT_NOEXCEPT
+{
+    Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+
+    /* Null data selector. */
+    if (!(uSel & X86_SEL_MASK_OFF_RPL))
+    {
+        iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, uSel);
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+        return VINF_SUCCESS;
+    }
+
+    /* Fetch the descriptor. */
+    IEMSELDESC Desc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_TS);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: failed to fetch selector. uSel=%u rc=%Rrc\n", uSel,
+             VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /* Must be a data segment or readable code segment. */
+    if (   !Desc.Legacy.Gen.u1DescType
+        || (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
+    {
+        Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: invalid segment type. uSel=%u Desc.u4Type=%#x\n", uSel,
+             Desc.Legacy.Gen.u4Type));
+        return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /* Check privileges for data segments and non-conforming code segments. */
+    if (   (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+        != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+    {
+        /* The RPL and the new CPL must be less than or equal to the DPL. */
+        if (   (unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
+            || (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl))
+        {
+            Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Invalid priv. uSel=%u uSel.RPL=%u DPL=%u CPL=%u\n",
+                 uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+        }
+    }
+
+    /* Is it there? */
+    if (!Desc.Legacy.Gen.u1Present)
+    {
+        Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: Segment not present. uSel=%u\n", uSel));
+        return iemRaiseSelectorNotPresentWithErr(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /* The base and limit. */
+    uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
+    uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
+
+    /*
+     * Ok, everything checked out fine. Now set the accessed bit before
+     * committing the result into the registers.
+     */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    /* Commit */
+    pSReg->Sel      = uSel;
+    pSReg->Attr.u   = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pSReg->u32Limit = cbLimit;
+    pSReg->u64Base  = u64Base;  /** @todo testcase/investigate: seen claims that the upper half of the base remains unchanged... */
+    pSReg->ValidSel = uSel;
+    pSReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
+        pSReg->Attr.u &= ~X86DESCATTR_UNUSABLE;
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs a task switch.
+ *
+ * If the task switch is the result of a JMP, CALL or IRET instruction, the
+ * caller is responsible for performing the necessary checks (like DPL, TSS
+ * present etc.) which are specific to JMP/CALL/IRET. See Intel Instruction
+ * reference for JMP, CALL, IRET.
+ *
+ * If the task switch is the due to a software interrupt or hardware exception,
+ * the caller is responsible for validating the TSS selector and descriptor. See
+ * Intel Instruction reference for INT n.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   enmTaskSwitch   The cause of the task switch.
+ * @param   uNextEip        The EIP effective after the task switch.
+ * @param   fFlags          The flags, see IEM_XCPT_FLAGS_XXX.
+ * @param   uErr            The error value if IEM_XCPT_FLAGS_ERR is set.
+ * @param   uCr2            The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
+ * @param   SelTSS          The TSS selector of the new task.
+ * @param   pNewDescTSS     Pointer to the new TSS descriptor.
+ */
+VBOXSTRICTRC
+iemTaskSwitch(PVMCPUCC        pVCpu,
+              IEMTASKSWITCH   enmTaskSwitch,
+              uint32_t        uNextEip,
+              uint32_t        fFlags,
+              uint16_t        uErr,
+              uint64_t        uCr2,
+              RTSEL           SelTSS,
+              PIEMSELDESC     pNewDescTSS) RT_NOEXCEPT
+{
+    Assert(!IEM_IS_REAL_MODE(pVCpu));
+    Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    uint32_t const uNewTSSType = pNewDescTSS->Legacy.Gate.u4Type;
+    Assert(   uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_AVAIL
+           || uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
+           || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
+           || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
+
+    bool const fIsNewTSS386 = (   uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_AVAIL
+                               || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
+
+    Log(("iemTaskSwitch: enmTaskSwitch=%u NewTSS=%#x fIsNewTSS386=%RTbool EIP=%#RX32 uNextEip=%#RX32\n", enmTaskSwitch, SelTSS,
+         fIsNewTSS386, pVCpu->cpum.GstCtx.eip, uNextEip));
+
+    /* Update CR2 in case it's a page-fault. */
+    /** @todo This should probably be done much earlier in IEM/PGM. See
+     *        @bugref{5653#c49}. */
+    if (fFlags & IEM_XCPT_FLAGS_CR2)
+        pVCpu->cpum.GstCtx.cr2 = uCr2;
+
+    /*
+     * Check the new TSS limit. See Intel spec. 6.15 "Exception and Interrupt Reference"
+     * subsection "Interrupt 10 - Invalid TSS Exception (#TS)".
+     */
+    uint32_t const uNewTSSLimit    = pNewDescTSS->Legacy.Gen.u16LimitLow | (pNewDescTSS->Legacy.Gen.u4LimitHigh << 16);
+    uint32_t const uNewTSSLimitMin = fIsNewTSS386 ? X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN : X86_SEL_TYPE_SYS_286_TSS_LIMIT_MIN;
+    if (uNewTSSLimit < uNewTSSLimitMin)
+    {
+        Log(("iemTaskSwitch: Invalid new TSS limit. enmTaskSwitch=%u uNewTSSLimit=%#x uNewTSSLimitMin=%#x -> #TS\n",
+             enmTaskSwitch, uNewTSSLimit, uNewTSSLimitMin));
+        return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /*
+     * Task switches in VMX non-root mode always cause task switches.
+     * The new TSS must have been read and validated (DPL, limits etc.) before a
+     * task-switch VM-exit commences.
+     *
+     * See Intel spec. 25.4.2 "Treatment of Task Switches".
+     */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        Log(("iemTaskSwitch: Guest intercept (source=%u, sel=%#x) -> VM-exit.\n", enmTaskSwitch, SelTSS));
+        IEM_VMX_VMEXIT_TASK_SWITCH_RET(pVCpu, enmTaskSwitch, SelTSS, uNextEip - pVCpu->cpum.GstCtx.eip);
+    }
+
+    /*
+     * The SVM nested-guest intercept for task-switch takes priority over all exceptions
+     * after validating the incoming (new) TSS, see AMD spec. 15.14.1 "Task Switch Intercept".
+     */
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TASK_SWITCH))
+    {
+        uint32_t const uExitInfo1 = SelTSS;
+        uint32_t       uExitInfo2 = uErr;
+        switch (enmTaskSwitch)
+        {
+            case IEMTASKSWITCH_JUMP: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_JUMP; break;
+            case IEMTASKSWITCH_IRET: uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_IRET; break;
+            default: break;
+        }
+        if (fFlags & IEM_XCPT_FLAGS_ERR)
+            uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_HAS_ERROR_CODE;
+        if (pVCpu->cpum.GstCtx.eflags.Bits.u1RF)
+            uExitInfo2 |= SVM_EXIT2_TASK_SWITCH_EFLAGS_RF;
+
+        Log(("iemTaskSwitch: Guest intercept -> #VMEXIT. uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitInfo1, uExitInfo2));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TASK_SWITCH, uExitInfo1, uExitInfo2);
+        RT_NOREF2(uExitInfo1, uExitInfo2);
+    }
+
+    /*
+     * Check the current TSS limit. The last written byte to the current TSS during the
+     * task switch will be 2 bytes at offset 0x5C (32-bit) and 1 byte at offset 0x28 (16-bit).
+     * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
+     *
+     * The AMD docs doesn't mention anything about limit checks with LTR which suggests you can
+     * end up with smaller than "legal" TSS limits.
+     */
+    uint32_t const uCurTSSLimit    = pVCpu->cpum.GstCtx.tr.u32Limit;
+    uint32_t const uCurTSSLimitMin = fIsNewTSS386 ? 0x5F : 0x29;
+    if (uCurTSSLimit < uCurTSSLimitMin)
+    {
+        Log(("iemTaskSwitch: Invalid current TSS limit. enmTaskSwitch=%u uCurTSSLimit=%#x uCurTSSLimitMin=%#x -> #TS\n",
+             enmTaskSwitch, uCurTSSLimit, uCurTSSLimitMin));
+        return iemRaiseTaskSwitchFaultWithErr(pVCpu, SelTSS & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /*
+     * Verify that the new TSS can be accessed and map it. Map only the required contents
+     * and not the entire TSS.
+     */
+    void           *pvNewTSS;
+    uint32_t  const cbNewTSS    = uNewTSSLimitMin + 1;
+    RTGCPTR   const GCPtrNewTSS = X86DESC_BASE(&pNewDescTSS->Legacy);
+    AssertCompile(sizeof(X86TSS32) == X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN + 1);
+    /** @todo Handle if the TSS crosses a page boundary. Intel specifies that it may
+     *        not perform correct translation if this happens. See Intel spec. 7.2.1
+     *        "Task-State Segment". */
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvNewTSS, cbNewTSS, UINT8_MAX, GCPtrNewTSS, IEM_ACCESS_SYS_RW, 0);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemTaskSwitch: Failed to read new TSS. enmTaskSwitch=%u cbNewTSS=%u uNewTSSLimit=%u rc=%Rrc\n", enmTaskSwitch,
+             cbNewTSS, uNewTSSLimit, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * Clear the busy bit in current task's TSS descriptor if it's a task switch due to JMP/IRET.
+     */
+    uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
+    if (   enmTaskSwitch == IEMTASKSWITCH_JUMP
+        || enmTaskSwitch == IEMTASKSWITCH_IRET)
+    {
+        PX86DESC pDescCurTSS;
+        rcStrict = iemMemMap(pVCpu, (void **)&pDescCurTSS, sizeof(*pDescCurTSS), UINT8_MAX,
+                             pVCpu->cpum.GstCtx.gdtr.pGdt + (pVCpu->cpum.GstCtx.tr.Sel & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
+                 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        pDescCurTSS->Gate.u4Type &= ~X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pDescCurTSS, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT. enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
+                 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* Clear EFLAGS.NT (Nested Task) in the eflags memory image, if it's a task switch due to an IRET. */
+        if (enmTaskSwitch == IEMTASKSWITCH_IRET)
+        {
+            Assert(   uNewTSSType == X86_SEL_TYPE_SYS_286_TSS_BUSY
+                   || uNewTSSType == X86_SEL_TYPE_SYS_386_TSS_BUSY);
+            fEFlags &= ~X86_EFL_NT;
+        }
+    }
+
+    /*
+     * Save the CPU state into the current TSS.
+     */
+    RTGCPTR const GCPtrCurTSS = pVCpu->cpum.GstCtx.tr.u64Base;
+    if (GCPtrNewTSS == GCPtrCurTSS)
+    {
+        Log(("iemTaskSwitch: Switching to the same TSS! enmTaskSwitch=%u GCPtr[Cur|New]TSS=%#RGv\n", enmTaskSwitch, GCPtrCurTSS));
+        Log(("uCurCr3=%#x uCurEip=%#x uCurEflags=%#x uCurEax=%#x uCurEsp=%#x uCurEbp=%#x uCurCS=%#04x uCurSS=%#04x uCurLdt=%#x\n",
+             pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, pVCpu->cpum.GstCtx.eax,
+             pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel,
+             pVCpu->cpum.GstCtx.ldtr.Sel));
+    }
+    if (fIsNewTSS386)
+    {
+        /*
+         * Verify that the current TSS (32-bit) can be accessed, only the minimum required size.
+         * See Intel spec. 7.2.1 "Task-State Segment (TSS)" for static and dynamic fields.
+         */
+        void          *pvCurTSS32;
+        uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS32, eip);
+        uint32_t const cbCurTSS  = RT_UOFFSETOF(X86TSS32, selLdt) - RT_UOFFSETOF(X86TSS32, eip);
+        AssertCompile(RTASSERT_OFFSET_OF(X86TSS32, selLdt) - RTASSERT_OFFSET_OF(X86TSS32, eip) == 64);
+        rcStrict = iemMemMap(pVCpu, &pvCurTSS32, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to read current 32-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
+                 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
+        PX86TSS32 pCurTSS32 = (PX86TSS32)((uintptr_t)pvCurTSS32 - offCurTSS);
+        pCurTSS32->eip    = uNextEip;
+        pCurTSS32->eflags = fEFlags;
+        pCurTSS32->eax    = pVCpu->cpum.GstCtx.eax;
+        pCurTSS32->ecx    = pVCpu->cpum.GstCtx.ecx;
+        pCurTSS32->edx    = pVCpu->cpum.GstCtx.edx;
+        pCurTSS32->ebx    = pVCpu->cpum.GstCtx.ebx;
+        pCurTSS32->esp    = pVCpu->cpum.GstCtx.esp;
+        pCurTSS32->ebp    = pVCpu->cpum.GstCtx.ebp;
+        pCurTSS32->esi    = pVCpu->cpum.GstCtx.esi;
+        pCurTSS32->edi    = pVCpu->cpum.GstCtx.edi;
+        pCurTSS32->es     = pVCpu->cpum.GstCtx.es.Sel;
+        pCurTSS32->cs     = pVCpu->cpum.GstCtx.cs.Sel;
+        pCurTSS32->ss     = pVCpu->cpum.GstCtx.ss.Sel;
+        pCurTSS32->ds     = pVCpu->cpum.GstCtx.ds.Sel;
+        pCurTSS32->fs     = pVCpu->cpum.GstCtx.fs.Sel;
+        pCurTSS32->gs     = pVCpu->cpum.GstCtx.gs.Sel;
+
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS32, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to commit current 32-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+    else
+    {
+        /*
+         * Verify that the current TSS (16-bit) can be accessed. Again, only the minimum required size.
+         */
+        void          *pvCurTSS16;
+        uint32_t const offCurTSS = RT_UOFFSETOF(X86TSS16, ip);
+        uint32_t const cbCurTSS  = RT_UOFFSETOF(X86TSS16, selLdt) - RT_UOFFSETOF(X86TSS16, ip);
+        AssertCompile(RTASSERT_OFFSET_OF(X86TSS16, selLdt) - RTASSERT_OFFSET_OF(X86TSS16, ip) == 28);
+        rcStrict = iemMemMap(pVCpu, &pvCurTSS16, cbCurTSS, UINT8_MAX, GCPtrCurTSS + offCurTSS, IEM_ACCESS_SYS_RW, 0);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to read current 16-bit TSS. enmTaskSwitch=%u GCPtrCurTSS=%#RGv cb=%u rc=%Rrc\n",
+                 enmTaskSwitch, GCPtrCurTSS, cbCurTSS, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* !! WARNING !! Access -only- the members (dynamic fields) that are mapped, i.e interval [offCurTSS..cbCurTSS). */
+        PX86TSS16 pCurTSS16 = (PX86TSS16)((uintptr_t)pvCurTSS16 - offCurTSS);
+        pCurTSS16->ip    = uNextEip;
+        pCurTSS16->flags = (uint16_t)fEFlags;
+        pCurTSS16->ax    = pVCpu->cpum.GstCtx.ax;
+        pCurTSS16->cx    = pVCpu->cpum.GstCtx.cx;
+        pCurTSS16->dx    = pVCpu->cpum.GstCtx.dx;
+        pCurTSS16->bx    = pVCpu->cpum.GstCtx.bx;
+        pCurTSS16->sp    = pVCpu->cpum.GstCtx.sp;
+        pCurTSS16->bp    = pVCpu->cpum.GstCtx.bp;
+        pCurTSS16->si    = pVCpu->cpum.GstCtx.si;
+        pCurTSS16->di    = pVCpu->cpum.GstCtx.di;
+        pCurTSS16->es    = pVCpu->cpum.GstCtx.es.Sel;
+        pCurTSS16->cs    = pVCpu->cpum.GstCtx.cs.Sel;
+        pCurTSS16->ss    = pVCpu->cpum.GstCtx.ss.Sel;
+        pCurTSS16->ds    = pVCpu->cpum.GstCtx.ds.Sel;
+
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pvCurTSS16, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to commit current 16-bit TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /*
+     * Update the previous task link field for the new TSS, if the task switch is due to a CALL/INT_XCPT.
+     */
+    if (   enmTaskSwitch == IEMTASKSWITCH_CALL
+        || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
+    {
+        /* 16 or 32-bit TSS doesn't matter, we only access the first, common 16-bit field (selPrev) here. */
+        PX86TSS32 pNewTSS = (PX86TSS32)pvNewTSS;
+        pNewTSS->selPrev  = pVCpu->cpum.GstCtx.tr.Sel;
+    }
+
+    /*
+     * Read the state from the new TSS into temporaries. Setting it immediately as the new CPU state is tricky,
+     * it's done further below with error handling (e.g. CR3 changes will go through PGM).
+     */
+    uint32_t uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEcx, uNewEdx, uNewEbx, uNewEsp, uNewEbp, uNewEsi, uNewEdi;
+    uint16_t uNewES,  uNewCS, uNewSS, uNewDS, uNewFS, uNewGS, uNewLdt;
+    bool     fNewDebugTrap;
+    if (fIsNewTSS386)
+    {
+        PCX86TSS32 pNewTSS32 = (PCX86TSS32)pvNewTSS;
+        uNewCr3       = (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG) ? pNewTSS32->cr3 : 0;
+        uNewEip       = pNewTSS32->eip;
+        uNewEflags    = pNewTSS32->eflags;
+        uNewEax       = pNewTSS32->eax;
+        uNewEcx       = pNewTSS32->ecx;
+        uNewEdx       = pNewTSS32->edx;
+        uNewEbx       = pNewTSS32->ebx;
+        uNewEsp       = pNewTSS32->esp;
+        uNewEbp       = pNewTSS32->ebp;
+        uNewEsi       = pNewTSS32->esi;
+        uNewEdi       = pNewTSS32->edi;
+        uNewES        = pNewTSS32->es;
+        uNewCS        = pNewTSS32->cs;
+        uNewSS        = pNewTSS32->ss;
+        uNewDS        = pNewTSS32->ds;
+        uNewFS        = pNewTSS32->fs;
+        uNewGS        = pNewTSS32->gs;
+        uNewLdt       = pNewTSS32->selLdt;
+        fNewDebugTrap = RT_BOOL(pNewTSS32->fDebugTrap);
+    }
+    else
+    {
+        PCX86TSS16 pNewTSS16 = (PCX86TSS16)pvNewTSS;
+        uNewCr3       = 0;
+        uNewEip       = pNewTSS16->ip;
+        uNewEflags    = pNewTSS16->flags;
+        uNewEax       = UINT32_C(0xffff0000) | pNewTSS16->ax;
+        uNewEcx       = UINT32_C(0xffff0000) | pNewTSS16->cx;
+        uNewEdx       = UINT32_C(0xffff0000) | pNewTSS16->dx;
+        uNewEbx       = UINT32_C(0xffff0000) | pNewTSS16->bx;
+        uNewEsp       = UINT32_C(0xffff0000) | pNewTSS16->sp;
+        uNewEbp       = UINT32_C(0xffff0000) | pNewTSS16->bp;
+        uNewEsi       = UINT32_C(0xffff0000) | pNewTSS16->si;
+        uNewEdi       = UINT32_C(0xffff0000) | pNewTSS16->di;
+        uNewES        = pNewTSS16->es;
+        uNewCS        = pNewTSS16->cs;
+        uNewSS        = pNewTSS16->ss;
+        uNewDS        = pNewTSS16->ds;
+        uNewFS        = 0;
+        uNewGS        = 0;
+        uNewLdt       = pNewTSS16->selLdt;
+        fNewDebugTrap = false;
+    }
+
+    if (GCPtrNewTSS == GCPtrCurTSS)
+        Log(("uNewCr3=%#x uNewEip=%#x uNewEflags=%#x uNewEax=%#x uNewEsp=%#x uNewEbp=%#x uNewCS=%#04x uNewSS=%#04x uNewLdt=%#x\n",
+             uNewCr3, uNewEip, uNewEflags, uNewEax, uNewEsp, uNewEbp, uNewCS, uNewSS, uNewLdt));
+
+    /*
+     * We're done accessing the new TSS.
+     */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewTSS, IEM_ACCESS_SYS_RW);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemTaskSwitch: Failed to commit new TSS. enmTaskSwitch=%u rc=%Rrc\n", enmTaskSwitch, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * Set the busy bit in the new TSS descriptor, if the task switch is a JMP/CALL/INT_XCPT.
+     */
+    if (enmTaskSwitch != IEMTASKSWITCH_IRET)
+    {
+        rcStrict = iemMemMap(pVCpu, (void **)&pNewDescTSS, sizeof(*pNewDescTSS), UINT8_MAX,
+                             pVCpu->cpum.GstCtx.gdtr.pGdt + (SelTSS & X86_SEL_MASK), IEM_ACCESS_SYS_RW, 0);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to read new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
+                 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* Check that the descriptor indicates the new TSS is available (not busy). */
+        AssertMsg(   pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
+                  || pNewDescTSS->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL,
+                     ("Invalid TSS descriptor type=%#x", pNewDescTSS->Legacy.Gate.u4Type));
+
+        pNewDescTSS->Legacy.Gate.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pNewDescTSS, IEM_ACCESS_SYS_RW);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Failed to commit new TSS descriptor in GDT (2). enmTaskSwitch=%u pGdt=%#RX64 rc=%Rrc\n",
+                 enmTaskSwitch, pVCpu->cpum.GstCtx.gdtr.pGdt, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /*
+     * From this point on, we're technically in the new task. We will defer exceptions
+     * until the completion of the task switch but before executing any instructions in the new task.
+     */
+    pVCpu->cpum.GstCtx.tr.Sel      = SelTSS;
+    pVCpu->cpum.GstCtx.tr.ValidSel = SelTSS;
+    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.tr.Attr.u   = X86DESC_GET_HID_ATTR(&pNewDescTSS->Legacy);
+    pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&pNewDescTSS->Legacy);
+    pVCpu->cpum.GstCtx.tr.u64Base  = X86DESC_BASE(&pNewDescTSS->Legacy);
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_TR);
+
+    /* Set the busy bit in TR. */
+    pVCpu->cpum.GstCtx.tr.Attr.n.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
+
+    /* Set EFLAGS.NT (Nested Task) in the eflags loaded from the new TSS, if it's a task switch due to a CALL/INT_XCPT. */
+    if (   enmTaskSwitch == IEMTASKSWITCH_CALL
+        || enmTaskSwitch == IEMTASKSWITCH_INT_XCPT)
+    {
+        uNewEflags |= X86_EFL_NT;
+    }
+
+    pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_LE_ALL;     /** @todo Should we clear DR7.LE bit too? */
+    pVCpu->cpum.GstCtx.cr0   |= X86_CR0_TS;
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR0);
+
+    pVCpu->cpum.GstCtx.eip    = uNewEip;
+    pVCpu->cpum.GstCtx.eax    = uNewEax;
+    pVCpu->cpum.GstCtx.ecx    = uNewEcx;
+    pVCpu->cpum.GstCtx.edx    = uNewEdx;
+    pVCpu->cpum.GstCtx.ebx    = uNewEbx;
+    pVCpu->cpum.GstCtx.esp    = uNewEsp;
+    pVCpu->cpum.GstCtx.ebp    = uNewEbp;
+    pVCpu->cpum.GstCtx.esi    = uNewEsi;
+    pVCpu->cpum.GstCtx.edi    = uNewEdi;
+
+    uNewEflags &= X86_EFL_LIVE_MASK;
+    uNewEflags |= X86_EFL_RA1_MASK;
+    IEMMISC_SET_EFL(pVCpu, uNewEflags);
+
+    /*
+     * Switch the selectors here and do the segment checks later. If we throw exceptions, the selectors
+     * will be valid in the exception handler. We cannot update the hidden parts until we've switched CR3
+     * due to the hidden part data originating from the guest LDT/GDT which is accessed through paging.
+     */
+    pVCpu->cpum.GstCtx.es.Sel       = uNewES;
+    pVCpu->cpum.GstCtx.es.Attr.u   &= ~X86DESCATTR_P;
+
+    pVCpu->cpum.GstCtx.cs.Sel       = uNewCS;
+    pVCpu->cpum.GstCtx.cs.Attr.u   &= ~X86DESCATTR_P;
+
+    pVCpu->cpum.GstCtx.ss.Sel       = uNewSS;
+    pVCpu->cpum.GstCtx.ss.Attr.u   &= ~X86DESCATTR_P;
+
+    pVCpu->cpum.GstCtx.ds.Sel       = uNewDS;
+    pVCpu->cpum.GstCtx.ds.Attr.u   &= ~X86DESCATTR_P;
+
+    pVCpu->cpum.GstCtx.fs.Sel       = uNewFS;
+    pVCpu->cpum.GstCtx.fs.Attr.u   &= ~X86DESCATTR_P;
+
+    pVCpu->cpum.GstCtx.gs.Sel       = uNewGS;
+    pVCpu->cpum.GstCtx.gs.Attr.u   &= ~X86DESCATTR_P;
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+
+    pVCpu->cpum.GstCtx.ldtr.Sel     = uNewLdt;
+    pVCpu->cpum.GstCtx.ldtr.fFlags  = CPUMSELREG_FLAGS_STALE;
+    pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_P;
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_LDTR);
+
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
+    {
+        pVCpu->cpum.GstCtx.es.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.cs.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.ss.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.ds.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.fs.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.gs.Attr.u   |= X86DESCATTR_UNUSABLE;
+        pVCpu->cpum.GstCtx.ldtr.Attr.u |= X86DESCATTR_UNUSABLE;
+    }
+
+    /*
+     * Switch CR3 for the new task.
+     */
+    if (   fIsNewTSS386
+        && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG))
+    {
+        /** @todo Should we update and flush TLBs only if CR3 value actually changes? */
+        int rc = CPUMSetGuestCR3(pVCpu, uNewCr3);
+        AssertRCSuccessReturn(rc, rc);
+
+        /* Inform PGM. */
+        /** @todo Should we raise \#GP(0) here when PAE PDPEs are invalid? */
+        rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
+        AssertRCReturn(rc, rc);
+        /* ignore informational status codes */
+
+        CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_CR3);
+    }
+
+    /*
+     * Switch LDTR for the new task.
+     */
+    if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
+        iemHlpLoadNullDataSelectorProt(pVCpu, &pVCpu->cpum.GstCtx.ldtr, uNewLdt);
+    else
+    {
+        Assert(!pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);   /* Ensures that LDT.TI check passes in iemMemFetchSelDesc() below. */
+
+        IEMSELDESC DescNewLdt;
+        rcStrict = iemMemFetchSelDesc(pVCpu, &DescNewLdt, uNewLdt, X86_XCPT_TS);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: fetching LDT failed. enmTaskSwitch=%u uNewLdt=%u cbGdt=%u rc=%Rrc\n", enmTaskSwitch,
+                 uNewLdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+        if (   !DescNewLdt.Legacy.Gen.u1Present
+            ||  DescNewLdt.Legacy.Gen.u1DescType
+            ||  DescNewLdt.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
+        {
+            Log(("iemTaskSwitch: Invalid LDT. enmTaskSwitch=%u uNewLdt=%u DescNewLdt.Legacy.u=%#RX64 -> #TS\n", enmTaskSwitch,
+                 uNewLdt, DescNewLdt.Legacy.u));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+        }
+
+        pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
+        pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ldtr.u64Base  = X86DESC_BASE(&DescNewLdt.Legacy);
+        pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&DescNewLdt.Legacy);
+        pVCpu->cpum.GstCtx.ldtr.Attr.u   = X86DESC_GET_HID_ATTR(&DescNewLdt.Legacy);
+        if (IEM_IS_GUEST_CPU_INTEL(pVCpu))
+            pVCpu->cpum.GstCtx.ldtr.Attr.u &= ~X86DESCATTR_UNUSABLE;
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ldtr));
+    }
+
+    IEMSELDESC DescSS;
+    if (IEM_IS_V86_MODE(pVCpu))
+    {
+        pVCpu->iem.s.uCpl = 3;
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.es, uNewES);
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.cs, uNewCS);
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ss, uNewSS);
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.ds, uNewDS);
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.fs, uNewFS);
+        iemHlpLoadSelectorInV86Mode(&pVCpu->cpum.GstCtx.gs, uNewGS);
+
+        /* Quick fix: fake DescSS. */ /** @todo fix the code further down? */
+        DescSS.Legacy.u = 0;
+        DescSS.Legacy.Gen.u16LimitLow = (uint16_t)pVCpu->cpum.GstCtx.ss.u32Limit;
+        DescSS.Legacy.Gen.u4LimitHigh = pVCpu->cpum.GstCtx.ss.u32Limit >> 16;
+        DescSS.Legacy.Gen.u16BaseLow  = (uint16_t)pVCpu->cpum.GstCtx.ss.u64Base;
+        DescSS.Legacy.Gen.u8BaseHigh1 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 16);
+        DescSS.Legacy.Gen.u8BaseHigh2 = (uint8_t)(pVCpu->cpum.GstCtx.ss.u64Base >> 24);
+        DescSS.Legacy.Gen.u4Type      = X86_SEL_TYPE_RW_ACC;
+        DescSS.Legacy.Gen.u2Dpl       = 3;
+    }
+    else
+    {
+        uint8_t const uNewCpl = (uNewCS & X86_SEL_RPL);
+
+        /*
+         * Load the stack segment for the new task.
+         */
+        if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
+        {
+            Log(("iemTaskSwitch: Null stack segment. enmTaskSwitch=%u uNewSS=%#x -> #TS\n", enmTaskSwitch, uNewSS));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* Fetch the descriptor. */
+        rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_TS);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: failed to fetch SS. uNewSS=%#x rc=%Rrc\n", uNewSS,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* SS must be a data segment and writable. */
+        if (    !DescSS.Legacy.Gen.u1DescType
+            ||  (DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
+            || !(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE))
+        {
+            Log(("iemTaskSwitch: SS invalid descriptor type. uNewSS=%#x u1DescType=%u u4Type=%#x\n",
+                 uNewSS, DescSS.Legacy.Gen.u1DescType, DescSS.Legacy.Gen.u4Type));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* The SS.RPL, SS.DPL, CS.RPL (CPL) must be equal. */
+        if (   (uNewSS & X86_SEL_RPL) != uNewCpl
+            || DescSS.Legacy.Gen.u2Dpl != uNewCpl)
+        {
+            Log(("iemTaskSwitch: Invalid priv. for SS. uNewSS=%#x SS.DPL=%u uNewCpl=%u -> #TS\n", uNewSS, DescSS.Legacy.Gen.u2Dpl,
+                 uNewCpl));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* Is it there? */
+        if (!DescSS.Legacy.Gen.u1Present)
+        {
+            Log(("iemTaskSwitch: SS not present. uNewSS=%#x -> #NP\n", uNewSS));
+            return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewSS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        uint32_t cbLimit = X86DESC_LIMIT_G(&DescSS.Legacy);
+        uint64_t u64Base = X86DESC_BASE(&DescSS.Legacy);
+
+        /* Set the accessed bit before committing the result into SS. */
+        if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* Commit SS. */
+        pVCpu->cpum.GstCtx.ss.Sel      = uNewSS;
+        pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
+        pVCpu->cpum.GstCtx.ss.Attr.u   = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
+        pVCpu->cpum.GstCtx.ss.u32Limit = cbLimit;
+        pVCpu->cpum.GstCtx.ss.u64Base  = u64Base;
+        pVCpu->cpum.GstCtx.ss.fFlags   = CPUMSELREG_FLAGS_VALID;
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+
+        /* CPL has changed, update IEM before loading rest of segments. */
+        pVCpu->iem.s.uCpl = uNewCpl;
+
+        /*
+         * Load the data segments for the new task.
+         */
+        rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.es, uNewES);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.ds, uNewDS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.fs, uNewFS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        rcStrict = iemHlpTaskSwitchLoadDataSelectorInProtMode(pVCpu, &pVCpu->cpum.GstCtx.gs, uNewGS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /*
+         * Load the code segment for the new task.
+         */
+        if (!(uNewCS & X86_SEL_MASK_OFF_RPL))
+        {
+            Log(("iemTaskSwitch #TS: Null code segment. enmTaskSwitch=%u uNewCS=%#x\n", enmTaskSwitch, uNewCS));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* Fetch the descriptor. */
+        IEMSELDESC DescCS;
+        rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_TS);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: failed to fetch CS. uNewCS=%u rc=%Rrc\n", uNewCS, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* CS must be a code segment. */
+        if (   !DescCS.Legacy.Gen.u1DescType
+            || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+        {
+            Log(("iemTaskSwitch: CS invalid descriptor type. uNewCS=%#x u1DescType=%u u4Type=%#x -> #TS\n", uNewCS,
+                 DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* For conforming CS, DPL must be less than or equal to the RPL. */
+        if (   (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+            && DescCS.Legacy.Gen.u2Dpl > (uNewCS & X86_SEL_RPL))
+        {
+            Log(("iemTaskSwitch: confirming CS DPL > RPL. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS, DescCS.Legacy.Gen.u4Type,
+                 DescCS.Legacy.Gen.u2Dpl));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* For non-conforming CS, DPL must match RPL. */
+        if (   !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+            && DescCS.Legacy.Gen.u2Dpl != (uNewCS & X86_SEL_RPL))
+        {
+            Log(("iemTaskSwitch: non-confirming CS DPL RPL mismatch. uNewCS=%#x u4Type=%#x DPL=%u -> #TS\n", uNewCS,
+                 DescCS.Legacy.Gen.u4Type, DescCS.Legacy.Gen.u2Dpl));
+            return iemRaiseTaskSwitchFaultWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        /* Is it there? */
+        if (!DescCS.Legacy.Gen.u1Present)
+        {
+            Log(("iemTaskSwitch: CS not present. uNewCS=%#x -> #NP\n", uNewCS));
+            return iemRaiseSelectorNotPresentWithErr(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+
+        cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
+        u64Base = X86DESC_BASE(&DescCS.Legacy);
+
+        /* Set the accessed bit before committing the result into CS. */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* Commit CS. */
+        pVCpu->cpum.GstCtx.cs.Sel      = uNewCS;
+        pVCpu->cpum.GstCtx.cs.ValidSel = uNewCS;
+        pVCpu->cpum.GstCtx.cs.Attr.u   = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
+        pVCpu->cpum.GstCtx.cs.u64Base  = u64Base;
+        pVCpu->cpum.GstCtx.cs.fFlags   = CPUMSELREG_FLAGS_VALID;
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+    }
+
+    /** @todo Debug trap. */
+    if (fIsNewTSS386 && fNewDebugTrap)
+        Log(("iemTaskSwitch: Debug Trap set in new TSS. Not implemented!\n"));
+
+    /*
+     * Construct the error code masks based on what caused this task switch.
+     * See Intel Instruction reference for INT.
+     */
+    uint16_t uExt;
+    if (   enmTaskSwitch == IEMTASKSWITCH_INT_XCPT
+        && (   !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+            ||  (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)))
+    {
+        uExt = 1;
+    }
+    else
+        uExt = 0;
+
+    /*
+     * Push any error code on to the new stack.
+     */
+    if (fFlags & IEM_XCPT_FLAGS_ERR)
+    {
+        Assert(enmTaskSwitch == IEMTASKSWITCH_INT_XCPT);
+        uint32_t      cbLimitSS    = X86DESC_LIMIT_G(&DescSS.Legacy);
+        uint8_t const cbStackFrame = fIsNewTSS386 ? 4 : 2;
+
+        /* Check that there is sufficient space on the stack. */
+        /** @todo Factor out segment limit checking for normal/expand down segments
+         *        into a separate function. */
+        if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
+        {
+            if (   pVCpu->cpum.GstCtx.esp - 1 > cbLimitSS
+                || pVCpu->cpum.GstCtx.esp < cbStackFrame)
+            {
+                /** @todo Intel says \#SS(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
+                Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #SS\n",
+                     pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
+                return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
+            }
+        }
+        else
+        {
+            if (   pVCpu->cpum.GstCtx.esp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT32_C(0xffff))
+                || pVCpu->cpum.GstCtx.esp - cbStackFrame < cbLimitSS + UINT32_C(1))
+            {
+                Log(("iemTaskSwitch: SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #SS\n",
+                     pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, cbStackFrame));
+                return iemRaiseStackSelectorNotPresentWithErr(pVCpu, uExt);
+            }
+        }
+
+
+        if (fIsNewTSS386)
+            rcStrict = iemMemStackPushU32(pVCpu, uErr);
+        else
+            rcStrict = iemMemStackPushU16(pVCpu, uErr);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iemTaskSwitch: Can't push error code to new task's stack. %s-bit TSS. rc=%Rrc\n",
+                 fIsNewTSS386 ? "32" : "16", VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /* Check the new EIP against the new CS limit. */
+    if (pVCpu->cpum.GstCtx.eip > pVCpu->cpum.GstCtx.cs.u32Limit)
+    {
+        Log(("iemHlpTaskSwitchLoadDataSelectorInProtMode: New EIP exceeds CS limit. uNewEIP=%#RX32 CS limit=%u -> #GP(0)\n",
+             pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.cs.u32Limit));
+        /** @todo Intel says \#GP(EXT) for INT/XCPT, I couldn't figure out AMD yet. */
+        return iemRaiseGeneralProtectionFault(pVCpu, uExt);
+    }
+
+    Log(("iemTaskSwitch: Success! New CS:EIP=%#04x:%#x SS=%#04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
+         pVCpu->cpum.GstCtx.ss.Sel));
+    return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
+}
+
+
+/**
+ * Implements exceptions and interrupts for protected mode.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr         The number of bytes to offset rIP by in the return
+ *                          address.
+ * @param   u8Vector        The interrupt / exception vector number.
+ * @param   fFlags          The flags.
+ * @param   uErr            The error value if IEM_XCPT_FLAGS_ERR is set.
+ * @param   uCr2            The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
+ */
+static VBOXSTRICTRC
+iemRaiseXcptOrIntInProtMode(PVMCPUCC    pVCpu,
+                            uint8_t     cbInstr,
+                            uint8_t     u8Vector,
+                            uint32_t    fFlags,
+                            uint16_t    uErr,
+                            uint64_t    uCr2) RT_NOEXCEPT
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    /*
+     * Read the IDT entry.
+     */
+    if (pVCpu->cpum.GstCtx.idtr.cbIdt < UINT32_C(8) * u8Vector + 7)
+    {
+        Log(("RaiseXcptOrIntInProtMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
+        return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+    X86DESC Idte;
+    VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.u, UINT8_MAX,
+                                              pVCpu->cpum.GstCtx.idtr.pIdt + UINT32_C(8) * u8Vector);
+    if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
+    {
+        Log(("iemRaiseXcptOrIntInProtMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+    Log(("iemRaiseXcptOrIntInProtMode: vec=%#x P=%u DPL=%u DT=%u:%u A=%u %04x:%04x%04x\n",
+         u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
+         Idte.Gate.u5ParmCount, Idte.Gate.u16Sel, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
+
+    /*
+     * Check the descriptor type, DPL and such.
+     * ASSUMES this is done in the same order as described for call-gate calls.
+     */
+    if (Idte.Gate.u1DescType)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+    bool     fTaskGate   = false;
+    uint8_t  f32BitGate  = true;
+    uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
+    switch (Idte.Gate.u4Type)
+    {
+        case X86_SEL_TYPE_SYS_UNDEFINED:
+        case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
+        case X86_SEL_TYPE_SYS_LDT:
+        case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+        case X86_SEL_TYPE_SYS_286_CALL_GATE:
+        case X86_SEL_TYPE_SYS_UNDEFINED2:
+        case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
+        case X86_SEL_TYPE_SYS_UNDEFINED3:
+        case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+        case X86_SEL_TYPE_SYS_386_CALL_GATE:
+        case X86_SEL_TYPE_SYS_UNDEFINED4:
+        {
+            /** @todo check what actually happens when the type is wrong...
+             *        esp. call gates. */
+            Log(("RaiseXcptOrIntInProtMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
+            return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+        }
+
+        case X86_SEL_TYPE_SYS_286_INT_GATE:
+            f32BitGate = false;
+            RT_FALL_THRU();
+        case X86_SEL_TYPE_SYS_386_INT_GATE:
+            fEflToClear |= X86_EFL_IF;
+            break;
+
+        case X86_SEL_TYPE_SYS_TASK_GATE:
+            fTaskGate = true;
+#ifndef IEM_IMPLEMENTS_TASKSWITCH
+            IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Task gates\n"));
+#endif
+            break;
+
+        case X86_SEL_TYPE_SYS_286_TRAP_GATE:
+            f32BitGate = false;
+        case X86_SEL_TYPE_SYS_386_TRAP_GATE:
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    /* Check DPL against CPL if applicable. */
+    if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
+    {
+        if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
+        {
+            Log(("RaiseXcptOrIntInProtMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
+            return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+        }
+    }
+
+    /* Is it there? */
+    if (!Idte.Gate.u1Present)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - not present -> #NP\n", u8Vector));
+        return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+
+    /* Is it a task-gate? */
+    if (fTaskGate)
+    {
+        /*
+         * Construct the error code masks based on what caused this task switch.
+         * See Intel Instruction reference for INT.
+         */
+        uint16_t const uExt     = (    (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+                                   && !(fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)) ? 0 : 1;
+        uint16_t const uSelMask = X86_SEL_MASK_OFF_RPL;
+        RTSEL          SelTSS   = Idte.Gate.u16Sel;
+
+        /*
+         * Fetch the TSS descriptor in the GDT.
+         */
+        IEMSELDESC DescTSS;
+        rcStrict = iemMemFetchSelDescWithErr(pVCpu, &DescTSS, SelTSS, X86_XCPT_GP, (SelTSS & uSelMask) | uExt);
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("RaiseXcptOrIntInProtMode %#x - failed to fetch TSS selector %#x, rc=%Rrc\n", u8Vector, SelTSS,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* The TSS descriptor must be a system segment and be available (not busy). */
+        if (   DescTSS.Legacy.Gen.u1DescType
+            || (   DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
+                && DescTSS.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL))
+        {
+            Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x of task gate not a system descriptor or not available %#RX64\n",
+                 u8Vector, SelTSS, DescTSS.Legacy.au64));
+            return iemRaiseGeneralProtectionFault(pVCpu, (SelTSS & uSelMask) | uExt);
+        }
+
+        /* The TSS must be present. */
+        if (!DescTSS.Legacy.Gen.u1Present)
+        {
+            Log(("RaiseXcptOrIntInProtMode %#x - TSS selector %#x not present %#RX64\n", u8Vector, SelTSS, DescTSS.Legacy.au64));
+            return iemRaiseSelectorNotPresentWithErr(pVCpu, (SelTSS & uSelMask) | uExt);
+        }
+
+        /* Do the actual task switch. */
+        return iemTaskSwitch(pVCpu, IEMTASKSWITCH_INT_XCPT,
+                             (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip,
+                             fFlags, uErr, uCr2, SelTSS, &DescTSS);
+    }
+
+    /* A null CS is bad. */
+    RTSEL NewCS = Idte.Gate.u16Sel;
+    if (!(NewCS & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Fetch the descriptor for the new CS. */
+    IEMSELDESC DescCS;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP); /** @todo correct exception? */
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /* Must be a code segment. */
+    if (!DescCS.Legacy.Gen.u1DescType)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - data selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /* Don't allow lowering the privilege level. */
+    /** @todo Does the lowering of privileges apply to software interrupts
+     *        only?  This has bearings on the more-privileged or
+     *        same-privilege stack behavior further down.  A testcase would
+     *        be nice. */
+    if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
+             u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /* Make sure the selector is present. */
+    if (!DescCS.Legacy.Gen.u1Present)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
+    }
+
+    /* Check the new EIP against the new CS limit. */
+    uint32_t const uNewEip =    Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_INT_GATE
+                             || Idte.Gate.u4Type == X86_SEL_TYPE_SYS_286_TRAP_GATE
+                           ? Idte.Gate.u16OffsetLow
+                           : Idte.Gate.u16OffsetLow | ((uint32_t)Idte.Gate.u16OffsetHigh << 16);
+    uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
+    if (uNewEip > cbLimitCS)
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - EIP=%#x > cbLimitCS=%#x (CS=%#x) -> #GP(0)\n",
+             u8Vector, uNewEip, cbLimitCS, NewCS));
+        return iemRaiseGeneralProtectionFault(pVCpu, 0);
+    }
+    Log7(("iemRaiseXcptOrIntInProtMode: new EIP=%#x CS=%#x\n", uNewEip, NewCS));
+
+    /* Calc the flag image to push. */
+    uint32_t        fEfl    = IEMMISC_GET_EFL(pVCpu);
+    if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
+        fEfl &= ~X86_EFL_RF;
+    else
+        fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
+
+    /* From V8086 mode only go to CPL 0. */
+    uint8_t const   uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
+                            ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
+    if ((fEfl & X86_EFL_VM) && uNewCpl != 0) /** @todo When exactly is this raised? */
+    {
+        Log(("RaiseXcptOrIntInProtMode %#x - CS=%#x - New CPL (%d) != 0 w/ VM=1 -> #GP\n", u8Vector, NewCS, uNewCpl));
+        return iemRaiseGeneralProtectionFault(pVCpu, 0);
+    }
+
+    /*
+     * If the privilege level changes, we need to get a new stack from the TSS.
+     * This in turns means validating the new SS and ESP...
+     */
+    if (uNewCpl != pVCpu->iem.s.uCpl)
+    {
+        RTSEL    NewSS;
+        uint32_t uNewEsp;
+        rcStrict = iemRaiseLoadStackFromTss32Or16(pVCpu, uNewCpl, &NewSS, &uNewEsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        IEMSELDESC DescSS;
+        rcStrict = iemMiscValidateNewSS(pVCpu, NewSS, uNewCpl, &DescSS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        /* If the new SS is 16-bit, we are only going to use SP, not ESP. */
+        if (!DescSS.Legacy.Gen.u1DefBig)
+        {
+            Log(("iemRaiseXcptOrIntInProtMode: Forcing ESP=%#x to 16 bits\n", uNewEsp));
+            uNewEsp = (uint16_t)uNewEsp;
+        }
+
+        Log7(("iemRaiseXcptOrIntInProtMode: New SS=%#x ESP=%#x (from TSS); current SS=%#x ESP=%#x\n", NewSS, uNewEsp, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
+
+        /* Check that there is sufficient space for the stack frame. */
+        uint32_t cbLimitSS = X86DESC_LIMIT_G(&DescSS.Legacy);
+        uint8_t const cbStackFrame = !(fEfl & X86_EFL_VM)
+                                   ? (fFlags & IEM_XCPT_FLAGS_ERR ? 12 : 10) << f32BitGate
+                                   : (fFlags & IEM_XCPT_FLAGS_ERR ? 20 : 18) << f32BitGate;
+
+        if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_DOWN))
+        {
+            if (   uNewEsp - 1 > cbLimitSS
+                || uNewEsp < cbStackFrame)
+            {
+                Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x is out of bounds -> #GP\n",
+                     u8Vector, NewSS, uNewEsp, cbStackFrame));
+                return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
+            }
+        }
+        else
+        {
+            if (   uNewEsp - 1 > (DescSS.Legacy.Gen.u1DefBig ? UINT32_MAX : UINT16_MAX)
+                || uNewEsp - cbStackFrame < cbLimitSS + UINT32_C(1))
+            {
+                Log(("RaiseXcptOrIntInProtMode: %#x - SS=%#x ESP=%#x cbStackFrame=%#x (expand down) is out of bounds -> #GP\n",
+                     u8Vector, NewSS, uNewEsp, cbStackFrame));
+                return iemRaiseSelectorBoundsBySelector(pVCpu, NewSS);
+            }
+        }
+
+        /*
+         * Start making changes.
+         */
+
+        /* Set the new CPL so that stack accesses use it. */
+        uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
+        pVCpu->iem.s.uCpl = uNewCpl;
+
+        /* Create the stack frame. */
+        RTPTRUNION uStackFrame;
+        rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
+                             uNewEsp - cbStackFrame + X86DESC_BASE(&DescSS.Legacy),
+                             IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        void * const pvStackFrame = uStackFrame.pv;
+        if (f32BitGate)
+        {
+            if (fFlags & IEM_XCPT_FLAGS_ERR)
+                *uStackFrame.pu32++ = uErr;
+            uStackFrame.pu32[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
+            uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
+            uStackFrame.pu32[2] = fEfl;
+            uStackFrame.pu32[3] = pVCpu->cpum.GstCtx.esp;
+            uStackFrame.pu32[4] = pVCpu->cpum.GstCtx.ss.Sel;
+            Log7(("iemRaiseXcptOrIntInProtMode: 32-bit push SS=%#x ESP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp));
+            if (fEfl & X86_EFL_VM)
+            {
+                uStackFrame.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel;
+                uStackFrame.pu32[5] = pVCpu->cpum.GstCtx.es.Sel;
+                uStackFrame.pu32[6] = pVCpu->cpum.GstCtx.ds.Sel;
+                uStackFrame.pu32[7] = pVCpu->cpum.GstCtx.fs.Sel;
+                uStackFrame.pu32[8] = pVCpu->cpum.GstCtx.gs.Sel;
+            }
+        }
+        else
+        {
+            if (fFlags & IEM_XCPT_FLAGS_ERR)
+                *uStackFrame.pu16++ = uErr;
+            uStackFrame.pu16[0] = (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT) ? pVCpu->cpum.GstCtx.ip + cbInstr : pVCpu->cpum.GstCtx.ip;
+            uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl;
+            uStackFrame.pu16[2] = fEfl;
+            uStackFrame.pu16[3] = pVCpu->cpum.GstCtx.sp;
+            uStackFrame.pu16[4] = pVCpu->cpum.GstCtx.ss.Sel;
+            Log7(("iemRaiseXcptOrIntInProtMode: 16-bit push SS=%#x SP=%#x\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.sp));
+            if (fEfl & X86_EFL_VM)
+            {
+                uStackFrame.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
+                uStackFrame.pu16[5] = pVCpu->cpum.GstCtx.es.Sel;
+                uStackFrame.pu16[6] = pVCpu->cpum.GstCtx.ds.Sel;
+                uStackFrame.pu16[7] = pVCpu->cpum.GstCtx.fs.Sel;
+                uStackFrame.pu16[8] = pVCpu->cpum.GstCtx.gs.Sel;
+            }
+        }
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /* Mark the selectors 'accessed' (hope this is the correct time). */
+        /** @todo testcase: excatly _when_ are the accessed bits set - before or
+         *        after pushing the stack frame? (Write protect the gdt + stack to
+         *        find out.) */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewSS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /*
+         * Start comitting the register changes (joins with the DPL=CPL branch).
+         */
+        pVCpu->cpum.GstCtx.ss.Sel            = NewSS;
+        pVCpu->cpum.GstCtx.ss.ValidSel       = NewSS;
+        pVCpu->cpum.GstCtx.ss.fFlags         = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.u32Limit       = cbLimitSS;
+        pVCpu->cpum.GstCtx.ss.u64Base        = X86DESC_BASE(&DescSS.Legacy);
+        pVCpu->cpum.GstCtx.ss.Attr.u         = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
+        /** @todo When coming from 32-bit code and operating with a 16-bit TSS and
+         *        16-bit handler, the high word of ESP remains unchanged (i.e. only
+         *        SP is loaded).
+         *  Need to check the other combinations too:
+         *      - 16-bit TSS, 32-bit handler
+         *      - 32-bit TSS, 16-bit handler */
+        if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            pVCpu->cpum.GstCtx.sp            = (uint16_t)(uNewEsp - cbStackFrame);
+        else
+            pVCpu->cpum.GstCtx.rsp           = uNewEsp - cbStackFrame;
+
+        if (fEfl & X86_EFL_VM)
+        {
+            iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.gs);
+            iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.fs);
+            iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.es);
+            iemHlpLoadNullDataSelectorOnV86Xcpt(pVCpu, &pVCpu->cpum.GstCtx.ds);
+        }
+    }
+    /*
+     * Same privilege, no stack change and smaller stack frame.
+     */
+    else
+    {
+        uint64_t        uNewRsp;
+        RTPTRUNION      uStackFrame;
+        uint8_t const   cbStackFrame = (fFlags & IEM_XCPT_FLAGS_ERR ? 8 : 6) << f32BitGate;
+        rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbStackFrame, f32BitGate ? 3 : 1, &uStackFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        void * const pvStackFrame = uStackFrame.pv;
+
+        if (f32BitGate)
+        {
+            if (fFlags & IEM_XCPT_FLAGS_ERR)
+                *uStackFrame.pu32++ = uErr;
+            uStackFrame.pu32[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
+            uStackFrame.pu32[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
+            uStackFrame.pu32[2] = fEfl;
+        }
+        else
+        {
+            if (fFlags & IEM_XCPT_FLAGS_ERR)
+                *uStackFrame.pu16++ = uErr;
+            uStackFrame.pu16[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.eip + cbInstr : pVCpu->cpum.GstCtx.eip;
+            uStackFrame.pu16[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | pVCpu->iem.s.uCpl;
+            uStackFrame.pu16[2] = fEfl;
+        }
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W); /* don't use the commit here */
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /* Mark the CS selector as 'accessed'. */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /*
+         * Start committing the register changes (joins with the other branch).
+         */
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+    }
+
+    /* ... register committing continues. */
+    pVCpu->cpum.GstCtx.cs.Sel            = (NewCS & ~X86_SEL_RPL) | uNewCpl;
+    pVCpu->cpum.GstCtx.cs.ValidSel       = (NewCS & ~X86_SEL_RPL) | uNewCpl;
+    pVCpu->cpum.GstCtx.cs.fFlags         = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.u32Limit       = cbLimitCS;
+    pVCpu->cpum.GstCtx.cs.u64Base        = X86DESC_BASE(&DescCS.Legacy);
+    pVCpu->cpum.GstCtx.cs.Attr.u         = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+
+    pVCpu->cpum.GstCtx.rip               = uNewEip;  /* (The entire register is modified, see pe16_32 bs3kit tests.) */
+    fEfl &= ~fEflToClear;
+    IEMMISC_SET_EFL(pVCpu, fEfl);
+
+    if (fFlags & IEM_XCPT_FLAGS_CR2)
+        pVCpu->cpum.GstCtx.cr2 = uCr2;
+
+    if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+        iemRaiseXcptAdjustState(pVCpu, u8Vector);
+
+    return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
+}
+
+
+/**
+ * Implements exceptions and interrupts for long mode.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr         The number of bytes to offset rIP by in the return
+ *                          address.
+ * @param   u8Vector        The interrupt / exception vector number.
+ * @param   fFlags          The flags.
+ * @param   uErr            The error value if IEM_XCPT_FLAGS_ERR is set.
+ * @param   uCr2            The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
+ */
+static VBOXSTRICTRC
+iemRaiseXcptOrIntInLongMode(PVMCPUCC    pVCpu,
+                            uint8_t     cbInstr,
+                            uint8_t     u8Vector,
+                            uint32_t    fFlags,
+                            uint16_t    uErr,
+                            uint64_t    uCr2) RT_NOEXCEPT
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    /*
+     * Read the IDT entry.
+     */
+    uint16_t offIdt = (uint16_t)u8Vector << 4;
+    if (pVCpu->cpum.GstCtx.idtr.cbIdt < offIdt + 7)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode: %#x is out of bounds (%#x)\n", u8Vector, pVCpu->cpum.GstCtx.idtr.cbIdt));
+        return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+    X86DESC64 Idte;
+#ifdef _MSC_VER /* Shut up silly compiler warning. */
+    Idte.au64[0] = 0;
+    Idte.au64[1] = 0;
+#endif
+    VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[0], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        rcStrict = iemMemFetchSysU64(pVCpu, &Idte.au64[1], UINT8_MAX, pVCpu->cpum.GstCtx.idtr.pIdt + offIdt + 8);
+    if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
+    {
+        Log(("iemRaiseXcptOrIntInLongMode: failed to fetch IDT entry! vec=%#x rc=%Rrc\n", u8Vector, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+    Log(("iemRaiseXcptOrIntInLongMode: vec=%#x P=%u DPL=%u DT=%u:%u IST=%u %04x:%08x%04x%04x\n",
+         u8Vector, Idte.Gate.u1Present, Idte.Gate.u2Dpl, Idte.Gate.u1DescType, Idte.Gate.u4Type,
+         Idte.Gate.u3IST, Idte.Gate.u16Sel, Idte.Gate.u32OffsetTop, Idte.Gate.u16OffsetHigh, Idte.Gate.u16OffsetLow));
+
+    /*
+     * Check the descriptor type, DPL and such.
+     * ASSUMES this is done in the same order as described for call-gate calls.
+     */
+    if (Idte.Gate.u1DescType)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - not system selector (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+    uint32_t fEflToClear = X86_EFL_TF | X86_EFL_NT | X86_EFL_RF | X86_EFL_VM;
+    switch (Idte.Gate.u4Type)
+    {
+        case AMD64_SEL_TYPE_SYS_INT_GATE:
+            fEflToClear |= X86_EFL_IF;
+            break;
+        case AMD64_SEL_TYPE_SYS_TRAP_GATE:
+            break;
+
+        default:
+            Log(("iemRaiseXcptOrIntInLongMode %#x - invalid type (%#x) -> #GP\n", u8Vector, Idte.Gate.u4Type));
+            return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+
+    /* Check DPL against CPL if applicable. */
+    if ((fFlags & (IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT)
+    {
+        if (pVCpu->iem.s.uCpl > Idte.Gate.u2Dpl)
+        {
+            Log(("iemRaiseXcptOrIntInLongMode %#x - CPL (%d) > DPL (%d) -> #GP\n", u8Vector, pVCpu->iem.s.uCpl, Idte.Gate.u2Dpl));
+            return iemRaiseGeneralProtectionFault(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+        }
+    }
+
+    /* Is it there? */
+    if (!Idte.Gate.u1Present)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - not present -> #NP\n", u8Vector));
+        return iemRaiseSelectorNotPresentWithErr(pVCpu, X86_TRAP_ERR_IDT | ((uint16_t)u8Vector << X86_TRAP_ERR_SEL_SHIFT));
+    }
+
+    /* A null CS is bad. */
+    RTSEL NewCS = Idte.Gate.u16Sel;
+    if (!(NewCS & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x -> #GP\n", u8Vector, NewCS));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Fetch the descriptor for the new CS. */
+    IEMSELDESC DescCS;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, NewCS, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - rc=%Rrc\n", u8Vector, NewCS, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /* Must be a 64-bit code segment. */
+    if (!DescCS.Long.Gen.u1DescType)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - system selector (%#x) -> #GP\n", u8Vector, NewCS, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+    if (   !DescCS.Long.Gen.u1Long
+        || DescCS.Long.Gen.u1DefBig
+        || !(DescCS.Long.Gen.u4Type & X86_SEL_TYPE_CODE) )
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - not 64-bit code selector (%#x, L=%u, D=%u) -> #GP\n",
+             u8Vector, NewCS, DescCS.Legacy.Gen.u4Type, DescCS.Long.Gen.u1Long, DescCS.Long.Gen.u1DefBig));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /* Don't allow lowering the privilege level.  For non-conforming CS
+       selectors, the CS.DPL sets the privilege level the trap/interrupt
+       handler runs at.  For conforming CS selectors, the CPL remains
+       unchanged, but the CS.DPL must be <= CPL. */
+    /** @todo Testcase: Interrupt handler with CS.DPL=1, interrupt dispatched
+     *        when CPU in Ring-0. Result \#GP?  */
+    if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - DPL (%d) > CPL (%d) -> #GP\n",
+             u8Vector, NewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault(pVCpu, NewCS & X86_SEL_MASK_OFF_RPL);
+    }
+
+
+    /* Make sure the selector is present. */
+    if (!DescCS.Legacy.Gen.u1Present)
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - CS=%#x - segment not present -> #NP\n", u8Vector, NewCS));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, NewCS);
+    }
+
+    /* Check that the new RIP is canonical. */
+    uint64_t const uNewRip = Idte.Gate.u16OffsetLow
+                           | ((uint32_t)Idte.Gate.u16OffsetHigh << 16)
+                           | ((uint64_t)Idte.Gate.u32OffsetTop  << 32);
+    if (!IEM_IS_CANONICAL(uNewRip))
+    {
+        Log(("iemRaiseXcptOrIntInLongMode %#x - RIP=%#RX64 - Not canonical -> #GP(0)\n", u8Vector, uNewRip));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * If the privilege level changes or if the IST isn't zero, we need to get
+     * a new stack from the TSS.
+     */
+    uint64_t        uNewRsp;
+    uint8_t const   uNewCpl = DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF
+                            ? pVCpu->iem.s.uCpl : DescCS.Legacy.Gen.u2Dpl;
+    if (   uNewCpl != pVCpu->iem.s.uCpl
+        || Idte.Gate.u3IST != 0)
+    {
+        rcStrict = iemRaiseLoadStackFromTss64(pVCpu, uNewCpl, Idte.Gate.u3IST, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+    else
+        uNewRsp = pVCpu->cpum.GstCtx.rsp;
+    uNewRsp &= ~(uint64_t)0xf;
+
+    /*
+     * Calc the flag image to push.
+     */
+    uint32_t        fEfl    = IEMMISC_GET_EFL(pVCpu);
+    if (fFlags & (IEM_XCPT_FLAGS_DRx_INSTR_BP | IEM_XCPT_FLAGS_T_SOFT_INT))
+        fEfl &= ~X86_EFL_RF;
+    else
+        fEfl |= X86_EFL_RF; /* Vagueness is all I've found on this so far... */ /** @todo Automatically pushing EFLAGS.RF. */
+
+    /*
+     * Start making changes.
+     */
+    /* Set the new CPL so that stack accesses use it. */
+    uint8_t const uOldCpl = pVCpu->iem.s.uCpl;
+    pVCpu->iem.s.uCpl = uNewCpl;
+
+    /* Create the stack frame. */
+    uint32_t   cbStackFrame = sizeof(uint64_t) * (5 + !!(fFlags & IEM_XCPT_FLAGS_ERR));
+    RTPTRUNION uStackFrame;
+    rcStrict = iemMemMap(pVCpu, &uStackFrame.pv, cbStackFrame, UINT8_MAX,
+                         uNewRsp - cbStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS, 0); /* _SYS is a hack ... */
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    void * const pvStackFrame = uStackFrame.pv;
+
+    if (fFlags & IEM_XCPT_FLAGS_ERR)
+        *uStackFrame.pu64++ = uErr;
+    uStackFrame.pu64[0] = fFlags & IEM_XCPT_FLAGS_T_SOFT_INT ? pVCpu->cpum.GstCtx.rip + cbInstr : pVCpu->cpum.GstCtx.rip;
+    uStackFrame.pu64[1] = (pVCpu->cpum.GstCtx.cs.Sel & ~X86_SEL_RPL) | uOldCpl; /* CPL paranoia */
+    uStackFrame.pu64[2] = fEfl;
+    uStackFrame.pu64[3] = pVCpu->cpum.GstCtx.rsp;
+    uStackFrame.pu64[4] = pVCpu->cpum.GstCtx.ss.Sel;
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvStackFrame, IEM_ACCESS_STACK_W | IEM_ACCESS_WHAT_SYS);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Mark the CS selectors 'accessed' (hope this is the correct time). */
+    /** @todo testcase: excatly _when_ are the accessed bits set - before or
+     *        after pushing the stack frame? (Write protect the gdt + stack to
+     *        find out.) */
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, NewCS);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    /*
+     * Start comitting the register changes.
+     */
+    /** @todo research/testcase: Figure out what VT-x and AMD-V loads into the
+     *        hidden registers when interrupting 32-bit or 16-bit code! */
+    if (uNewCpl != uOldCpl)
+    {
+        pVCpu->cpum.GstCtx.ss.Sel        = 0 | uNewCpl;
+        pVCpu->cpum.GstCtx.ss.ValidSel   = 0 | uNewCpl;
+        pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.u32Limit   = UINT32_MAX;
+        pVCpu->cpum.GstCtx.ss.u64Base    = 0;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = (uNewCpl << X86DESCATTR_DPL_SHIFT) | X86DESCATTR_UNUSABLE;
+    }
+    pVCpu->cpum.GstCtx.rsp           = uNewRsp - cbStackFrame;
+    pVCpu->cpum.GstCtx.cs.Sel        = (NewCS & ~X86_SEL_RPL) | uNewCpl;
+    pVCpu->cpum.GstCtx.cs.ValidSel   = (NewCS & ~X86_SEL_RPL) | uNewCpl;
+    pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.u32Limit   = X86DESC_LIMIT_G(&DescCS.Legacy);
+    pVCpu->cpum.GstCtx.cs.u64Base    = X86DESC_BASE(&DescCS.Legacy);
+    pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+    pVCpu->cpum.GstCtx.rip           = uNewRip;
+
+    fEfl &= ~fEflToClear;
+    IEMMISC_SET_EFL(pVCpu, fEfl);
+
+    if (fFlags & IEM_XCPT_FLAGS_CR2)
+        pVCpu->cpum.GstCtx.cr2 = uCr2;
+
+    if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+        iemRaiseXcptAdjustState(pVCpu, u8Vector);
+
+    return fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT ? VINF_IEM_RAISED_XCPT : VINF_SUCCESS;
+}
+
+
+/**
+ * Implements exceptions and interrupts.
+ *
+ * All exceptions and interrupts goes thru this function!
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr         The number of bytes to offset rIP by in the return
+ *                          address.
+ * @param   u8Vector        The interrupt / exception vector number.
+ * @param   fFlags          The flags.
+ * @param   uErr            The error value if IEM_XCPT_FLAGS_ERR is set.
+ * @param   uCr2            The CR2 value if IEM_XCPT_FLAGS_CR2 is set.
+ */
+VBOXSTRICTRC
+iemRaiseXcptOrInt(PVMCPUCC    pVCpu,
+                  uint8_t     cbInstr,
+                  uint8_t     u8Vector,
+                  uint32_t    fFlags,
+                  uint16_t    uErr,
+                  uint64_t    uCr2) RT_NOEXCEPT
+{
+    /*
+     * Get all the state that we might need here.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+#ifndef IEM_WITH_CODE_TLB /** @todo we're doing it afterwards too, that should suffice... */
+    /*
+     * Flush prefetch buffer
+     */
+    pVCpu->iem.s.cbOpcode = pVCpu->iem.s.offOpcode;
+#endif
+
+    /*
+     * Perform the V8086 IOPL check and upgrade the fault without nesting.
+     */
+    if (   pVCpu->cpum.GstCtx.eflags.Bits.u1VM
+        && pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL != 3
+        && (fFlags & (  IEM_XCPT_FLAGS_T_SOFT_INT
+                      | IEM_XCPT_FLAGS_BP_INSTR
+                      | IEM_XCPT_FLAGS_ICEBP_INSTR
+                      | IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
+        && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
+    {
+        Log(("iemRaiseXcptOrInt: V8086 IOPL check failed for int %#x -> #GP(0)\n", u8Vector));
+        fFlags   = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
+        u8Vector = X86_XCPT_GP;
+        uErr     = 0;
+    }
+#ifdef DBGFTRACE_ENABLED
+    RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "Xcpt/%u: %02x %u %x %x %llx %04x:%04llx %04x:%04llx",
+                      pVCpu->iem.s.cXcptRecursions, u8Vector, cbInstr, fFlags, uErr, uCr2,
+                      pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp);
+#endif
+
+    /*
+     * Evaluate whether NMI blocking should be in effect.
+     * Normally, NMI blocking is in effect whenever we inject an NMI.
+     */
+    bool fBlockNmi = u8Vector == X86_XCPT_NMI
+                  && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict0 = iemVmxVmexitEvent(pVCpu, u8Vector, fFlags, uErr, uCr2, cbInstr);
+        if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict0;
+
+        /* If virtual-NMI blocking is in effect for the nested-guest, guest NMIs are not blocked. */
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
+        {
+            Assert(CPUMIsGuestVmxPinCtlsSet(&pVCpu->cpum.GstCtx, VMX_PIN_CTLS_VIRT_NMI));
+            fBlockNmi = false;
+        }
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
+    {
+        /*
+         * If the event is being injected as part of VMRUN, it isn't subject to event
+         * intercepts in the nested-guest. However, secondary exceptions that occur
+         * during injection of any event -are- subject to exception intercepts.
+         *
+         * See AMD spec. 15.20 "Event Injection".
+         */
+        if (!pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents)
+            pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = true;
+        else
+        {
+            /*
+             * Check and handle if the event being raised is intercepted.
+             */
+            VBOXSTRICTRC rcStrict0 = iemHandleSvmEventIntercept(pVCpu, u8Vector, fFlags, uErr, uCr2);
+            if (rcStrict0 != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+                return rcStrict0;
+        }
+    }
+#endif
+
+    /*
+     * Set NMI blocking if necessary.
+     */
+    if (fBlockNmi)
+        CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
+
+    /*
+     * Do recursion accounting.
+     */
+    uint8_t const  uPrevXcpt = pVCpu->iem.s.uCurXcpt;
+    uint32_t const fPrevXcpt = pVCpu->iem.s.fCurXcpt;
+    if (pVCpu->iem.s.cXcptRecursions == 0)
+        Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx\n",
+             u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2));
+    else
+    {
+        Log(("iemRaiseXcptOrInt: %#x at %04x:%RGv cbInstr=%#x fFlags=%#x uErr=%#x uCr2=%llx; prev=%#x depth=%d flags=%#x\n",
+             u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, cbInstr, fFlags, uErr, uCr2, pVCpu->iem.s.uCurXcpt,
+             pVCpu->iem.s.cXcptRecursions + 1, fPrevXcpt));
+
+        if (pVCpu->iem.s.cXcptRecursions >= 4)
+        {
+#ifdef DEBUG_bird
+            AssertFailed();
+#endif
+            IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Too many fault nestings.\n"));
+        }
+
+        /*
+         * Evaluate the sequence of recurring events.
+         */
+        IEMXCPTRAISE enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fPrevXcpt, uPrevXcpt, fFlags, u8Vector,
+                                                         NULL /* pXcptRaiseInfo */);
+        if (enmRaise == IEMXCPTRAISE_CURRENT_XCPT)
+        { /* likely */ }
+        else if (enmRaise == IEMXCPTRAISE_DOUBLE_FAULT)
+        {
+            Log2(("iemRaiseXcptOrInt: Raising double fault. uPrevXcpt=%#x\n", uPrevXcpt));
+            fFlags   = IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR;
+            u8Vector = X86_XCPT_DF;
+            uErr     = 0;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            /* VMX nested-guest #DF intercept needs to be checked here. */
+            if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+            {
+                VBOXSTRICTRC rcStrict0 = iemVmxVmexitEventDoubleFault(pVCpu);
+                if (rcStrict0 != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+                    return rcStrict0;
+            }
+#endif
+            /* SVM nested-guest #DF intercepts need to be checked now. See AMD spec. 15.12 "Exception Intercepts". */
+            if (IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, X86_XCPT_DF))
+                IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_DF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+        else if (enmRaise == IEMXCPTRAISE_TRIPLE_FAULT)
+        {
+            Log2(("iemRaiseXcptOrInt: Raising triple fault. uPrevXcpt=%#x\n", uPrevXcpt));
+            return iemInitiateCpuShutdown(pVCpu);
+        }
+        else if (enmRaise == IEMXCPTRAISE_CPU_HANG)
+        {
+            /* If a nested-guest enters an endless CPU loop condition, we'll emulate it; otherwise guru. */
+            Log2(("iemRaiseXcptOrInt: CPU hang condition detected\n"));
+            if (   !CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
+                && !CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
+                return VERR_EM_GUEST_CPU_HANG;
+        }
+        else
+        {
+            AssertMsgFailed(("Unexpected condition! enmRaise=%#x uPrevXcpt=%#x fPrevXcpt=%#x, u8Vector=%#x fFlags=%#x\n",
+                             enmRaise, uPrevXcpt, fPrevXcpt, u8Vector, fFlags));
+            return VERR_IEM_IPE_9;
+        }
+
+        /*
+         * The 'EXT' bit is set when an exception occurs during deliver of an external
+         * event (such as an interrupt or earlier exception)[1]. Privileged software
+         * exception (INT1) also sets the EXT bit[2]. Exceptions generated by software
+         * interrupts and INTO, INT3 instructions, the 'EXT' bit will not be set.
+         *
+         * [1] - Intel spec. 6.13 "Error Code"
+         * [2] - Intel spec. 26.5.1.1 "Details of Vectored-Event Injection".
+         * [3] - Intel Instruction reference for INT n.
+         */
+        if (   (fPrevXcpt & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_T_EXT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR))
+            && (fFlags & IEM_XCPT_FLAGS_ERR)
+            && u8Vector != X86_XCPT_PF
+            && u8Vector != X86_XCPT_DF)
+        {
+            uErr |= X86_TRAP_ERR_EXTERNAL;
+        }
+    }
+
+    pVCpu->iem.s.cXcptRecursions++;
+    pVCpu->iem.s.uCurXcpt    = u8Vector;
+    pVCpu->iem.s.fCurXcpt    = fFlags;
+    pVCpu->iem.s.uCurXcptErr = uErr;
+    pVCpu->iem.s.uCurXcptCr2 = uCr2;
+
+    /*
+     * Extensive logging.
+     */
+#if defined(LOG_ENABLED) && defined(IN_RING3)
+    if (LogIs3Enabled())
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR_MASK);
+        PVM     pVM = pVCpu->CTX_SUFF(pVM);
+        char    szRegs[4096];
+        DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
+                        "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
+                        "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
+                        "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
+                        "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
+                        "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
+                        "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
+                        "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
+                        "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
+                        "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
+                        "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
+                        "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
+                        "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
+                        "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
+                        "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim}  idtr=%016VR{idtr_base}:%04VR{idtr_lim}  rflags=%08VR{rflags}\n"
+                        "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
+                        "tr  ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
+                        "    sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
+                        "        efer=%016VR{efer}\n"
+                        "         pat=%016VR{pat}\n"
+                        "     sf_mask=%016VR{sf_mask}\n"
+                        "krnl_gs_base=%016VR{krnl_gs_base}\n"
+                        "       lstar=%016VR{lstar}\n"
+                        "        star=%016VR{star} cstar=%016VR{cstar}\n"
+                        "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
+                        );
+
+        char szInstr[256];
+        DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
+                           DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                           szInstr, sizeof(szInstr), NULL);
+        Log3(("%s%s\n", szRegs, szInstr));
+    }
+#endif /* LOG_ENABLED */
+
+    /*
+     * Stats.
+     */
+    if (!(fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT))
+        STAM_REL_STATS({ pVCpu->iem.s.aStatInts[u8Vector] += 1; });
+    else if (u8Vector <= X86_XCPT_LAST)
+    {
+        STAM_REL_COUNTER_INC(&pVCpu->iem.s.aStatXcpts[u8Vector]);
+        EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_XCPT, u8Vector),
+                         pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base, ASMReadTSC());
+    }
+
+    /*
+     * #PF's implies a INVLPG for the CR2 value (see 4.10.1.1 in Intel SDM Vol 3)
+     * to ensure that a stale TLB or paging cache entry will only cause one
+     * spurious #PF.
+     */
+    if (    u8Vector == X86_XCPT_PF
+        && (fFlags & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_CR2)) == (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_CR2))
+        IEMTlbInvalidatePage(pVCpu, uCr2);
+
+    /*
+     * Call the mode specific worker function.
+     */
+    VBOXSTRICTRC    rcStrict;
+    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
+        rcStrict = iemRaiseXcptOrIntInRealMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
+    else if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
+        rcStrict = iemRaiseXcptOrIntInLongMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
+    else
+        rcStrict = iemRaiseXcptOrIntInProtMode(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
+
+    /* Flush the prefetch buffer. */
+#ifdef IEM_WITH_CODE_TLB
+    pVCpu->iem.s.pbInstrBuf = NULL;
+#else
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    /*
+     * Unwind.
+     */
+    pVCpu->iem.s.cXcptRecursions--;
+    pVCpu->iem.s.uCurXcpt = uPrevXcpt;
+    pVCpu->iem.s.fCurXcpt = fPrevXcpt;
+    Log(("iemRaiseXcptOrInt: returns %Rrc (vec=%#x); cs:rip=%04x:%RGv ss:rsp=%04x:%RGv cpl=%u depth=%d\n",
+         VBOXSTRICTRC_VAL(rcStrict), u8Vector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.esp, pVCpu->iem.s.uCpl,
+         pVCpu->iem.s.cXcptRecursions + 1));
+    return rcStrict;
+}
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * See iemRaiseXcptOrInt.  Will not return.
+ */
+DECL_NO_RETURN(void)
+iemRaiseXcptOrIntJmp(PVMCPUCC    pVCpu,
+                     uint8_t     cbInstr,
+                     uint8_t     u8Vector,
+                     uint32_t    fFlags,
+                     uint16_t    uErr,
+                     uint64_t    uCr2) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8Vector, fFlags, uErr, uCr2);
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+}
+#endif
+
+
+/** \#DE - 00.  */
+VBOXSTRICTRC iemRaiseDivideError(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** \#DB - 01.
+ * @note This automatically clear DR7.GD.  */
+VBOXSTRICTRC iemRaiseDebugException(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /* This always clears RF (via IEM_XCPT_FLAGS_DRx_INSTR_BP). */
+    pVCpu->cpum.GstCtx.dr[7] &= ~X86_DR7_GD;
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DB, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_DRx_INSTR_BP, 0, 0);
+}
+
+
+/** \#BR - 05.  */
+VBOXSTRICTRC iemRaiseBoundRangeExceeded(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_BR, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** \#UD - 06.  */
+VBOXSTRICTRC iemRaiseUndefinedOpcode(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** \#NM - 07.  */
+VBOXSTRICTRC iemRaiseDeviceNotAvailable(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NM, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** \#TS(err) - 0a.  */
+VBOXSTRICTRC iemRaiseTaskSwitchFaultWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
+}
+
+
+/** \#TS(tr) - 0a.  */
+VBOXSTRICTRC iemRaiseTaskSwitchFaultCurrentTSS(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             pVCpu->cpum.GstCtx.tr.Sel, 0);
+}
+
+
+/** \#TS(0) - 0a.  */
+VBOXSTRICTRC iemRaiseTaskSwitchFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             0, 0);
+}
+
+
+/** \#TS(err) - 0a.  */
+VBOXSTRICTRC iemRaiseTaskSwitchFaultBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_TS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             uSel & X86_SEL_MASK_OFF_RPL, 0);
+}
+
+
+/** \#NP(err) - 0b.  */
+VBOXSTRICTRC iemRaiseSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
+}
+
+
+/** \#NP(sel) - 0b.  */
+VBOXSTRICTRC iemRaiseSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_NP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             uSel & ~X86_SEL_RPL, 0);
+}
+
+
+/** \#SS(seg) - 0c.  */
+VBOXSTRICTRC iemRaiseStackSelectorNotPresentBySelector(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             uSel & ~X86_SEL_RPL, 0);
+}
+
+
+/** \#SS(err) - 0c.  */
+VBOXSTRICTRC iemRaiseStackSelectorNotPresentWithErr(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_SS, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
+}
+
+
+/** \#GP(n) - 0d.  */
+VBOXSTRICTRC iemRaiseGeneralProtectionFault(PVMCPUCC pVCpu, uint16_t uErr) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErr, 0);
+}
+
+
+/** \#GP(0) - 0d.  */
+VBOXSTRICTRC iemRaiseGeneralProtectionFault0(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#GP(0) - 0d.  */
+DECL_NO_RETURN(void) iemRaiseGeneralProtectionFault0Jmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+#endif
+
+
+/** \#GP(sel) - 0d.  */
+VBOXSTRICTRC iemRaiseGeneralProtectionFaultBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                             Sel & ~X86_SEL_RPL, 0);
+}
+
+
+/** \#GP(0) - 0d.  */
+VBOXSTRICTRC iemRaiseNotCanonical(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+
+/** \#GP(sel) - 0d.  */
+VBOXSTRICTRC iemRaiseSelectorBounds(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
+{
+    NOREF(iSegReg); NOREF(fAccess);
+    return iemRaiseXcptOrInt(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
+                             IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#GP(sel) - 0d, longjmp.  */
+DECL_NO_RETURN(void) iemRaiseSelectorBoundsJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    NOREF(iSegReg); NOREF(fAccess);
+    iemRaiseXcptOrIntJmp(pVCpu, 0, iSegReg == X86_SREG_SS ? X86_XCPT_SS : X86_XCPT_GP,
+                         IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+#endif
+
+/** \#GP(sel) - 0d.  */
+VBOXSTRICTRC iemRaiseSelectorBoundsBySelector(PVMCPUCC pVCpu, RTSEL Sel) RT_NOEXCEPT
+{
+    NOREF(Sel);
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#GP(sel) - 0d, longjmp.  */
+DECL_NO_RETURN(void) iemRaiseSelectorBoundsBySelectorJmp(PVMCPUCC pVCpu, RTSEL Sel) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    NOREF(Sel);
+    iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+#endif
+
+
+/** \#GP(sel) - 0d.  */
+VBOXSTRICTRC iemRaiseSelectorInvalidAccess(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) RT_NOEXCEPT
+{
+    NOREF(iSegReg); NOREF(fAccess);
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#GP(sel) - 0d, longjmp.  */
+DECL_NO_RETURN(void) iemRaiseSelectorInvalidAccessJmp(PVMCPUCC pVCpu, uint32_t iSegReg, uint32_t fAccess) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    NOREF(iSegReg); NOREF(fAccess);
+    iemRaiseXcptOrIntJmp(pVCpu, 0, X86_XCPT_GP, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+#endif
+
+
+/** \#PF(n) - 0e.  */
+VBOXSTRICTRC iemRaisePageFault(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t cbAccess, uint32_t fAccess, int rc) RT_NOEXCEPT
+{
+    uint16_t uErr;
+    switch (rc)
+    {
+        case VERR_PAGE_NOT_PRESENT:
+        case VERR_PAGE_TABLE_NOT_PRESENT:
+        case VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT:
+        case VERR_PAGE_MAP_LEVEL4_NOT_PRESENT:
+            uErr = 0;
+            break;
+
+        default:
+            AssertMsgFailed(("%Rrc\n", rc));
+            RT_FALL_THRU();
+        case VERR_ACCESS_DENIED:
+            uErr = X86_TRAP_PF_P;
+            break;
+
+        /** @todo reserved  */
+    }
+
+    if (pVCpu->iem.s.uCpl == 3)
+        uErr |= X86_TRAP_PF_US;
+
+    if (   (fAccess & IEM_ACCESS_WHAT_MASK) == IEM_ACCESS_WHAT_CODE
+        && (   (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
+            && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) ) )
+        uErr |= X86_TRAP_PF_ID;
+
+#if 0 /* This is so much non-sense, really.  Why was it done like that? */
+    /* Note! RW access callers reporting a WRITE protection fault, will clear
+             the READ flag before calling.  So, read-modify-write accesses (RW)
+             can safely be reported as READ faults. */
+    if ((fAccess & (IEM_ACCESS_TYPE_WRITE | IEM_ACCESS_TYPE_READ)) == IEM_ACCESS_TYPE_WRITE)
+        uErr |= X86_TRAP_PF_RW;
+#else
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+    {
+        /// @todo r=bird: bs3-cpu-basic-2 wants X86_TRAP_PF_RW for xchg and cmpxchg
+        /// (regardless of outcome of the comparison in the latter case).
+        //if (!(fAccess & IEM_ACCESS_TYPE_READ))
+            uErr |= X86_TRAP_PF_RW;
+    }
+#endif
+
+    /* For FXSAVE and FRSTOR the #PF is typically reported at the max address
+       of the memory operand rather than at the start of it. (Not sure what
+       happens if it crosses a page boundrary.)  The current heuristics for
+       this is to report the #PF for the last byte if the access is more than
+       64 bytes. This is probably not correct, but we can work that out later,
+       main objective now is to get FXSAVE to work like for real hardware and
+       make bs3-cpu-basic2 work. */
+    if (cbAccess <= 64)
+    { /* likely*/ }
+    else
+        GCPtrWhere += cbAccess - 1;
+
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_PF, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR | IEM_XCPT_FLAGS_CR2,
+                             uErr, GCPtrWhere);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#PF(n) - 0e, longjmp.  */
+DECL_NO_RETURN(void) iemRaisePageFaultJmp(PVMCPUCC pVCpu, RTGCPTR GCPtrWhere, uint32_t cbAccess,
+                                          uint32_t fAccess, int rc) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(iemRaisePageFault(pVCpu, GCPtrWhere, cbAccess, fAccess, rc)));
+}
+#endif
+
+
+/** \#MF(0) - 10.  */
+VBOXSTRICTRC iemRaiseMathFault(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE)
+        return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_MF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+
+    /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
+    PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq */, 1 /* u8Level */, 0 /* uTagSrc */);
+    return iemRegUpdateRipAndFinishClearingRF(pVCpu);
+}
+
+
+/** \#AC(0) - 11.  */
+VBOXSTRICTRC iemRaiseAlignmentCheckException(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_AC, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, 0, 0);
+}
+
+#ifdef IEM_WITH_SETJMP
+/** \#AC(0) - 11, longjmp.  */
+DECL_NO_RETURN(void) iemRaiseAlignmentCheckExceptionJmp(PVMCPUCC pVCpu) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(iemRaiseAlignmentCheckException(pVCpu)));
+}
+#endif
+
+
+/** \#XF(0)/\#XM(0) - 19.   */
+VBOXSTRICTRC iemRaiseSimdFpException(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_XF, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** Accessed via IEMOP_RAISE_DIVIDE_ERROR.   */
+IEM_CIMPL_DEF_0(iemCImplRaiseDivideError)
+{
+    NOREF(cbInstr);
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_DE, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** Accessed via IEMOP_RAISE_INVALID_LOCK_PREFIX. */
+IEM_CIMPL_DEF_0(iemCImplRaiseInvalidLockPrefix)
+{
+    NOREF(cbInstr);
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** Accessed via IEMOP_RAISE_INVALID_OPCODE. */
+IEM_CIMPL_DEF_0(iemCImplRaiseInvalidOpcode)
+{
+    NOREF(cbInstr);
+    return iemRaiseXcptOrInt(pVCpu, 0, X86_XCPT_UD, IEM_XCPT_FLAGS_T_CPU_XCPT, 0, 0);
+}
+
+
+/** @}  */
+
+/** @name  Common opcode decoders.
+ * @{
+ */
+//#include <iprt/mem.h>
+
+/**
+ * Used to add extra details about a stub case.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ */
+void iemOpStubMsg2(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+#if defined(LOG_ENABLED) && defined(IN_RING3)
+    PVM  pVM = pVCpu->CTX_SUFF(pVM);
+    char szRegs[4096];
+    DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
+                    "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
+                    "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
+                    "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
+                    "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
+                    "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
+                    "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
+                    "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
+                    "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
+                    "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
+                    "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
+                    "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
+                    "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
+                    "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
+                    "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim}  idtr=%016VR{idtr_base}:%04VR{idtr_lim}  rflags=%08VR{rflags}\n"
+                    "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
+                    "tr  ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
+                    "    sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
+                    "        efer=%016VR{efer}\n"
+                    "         pat=%016VR{pat}\n"
+                    "     sf_mask=%016VR{sf_mask}\n"
+                    "krnl_gs_base=%016VR{krnl_gs_base}\n"
+                    "       lstar=%016VR{lstar}\n"
+                    "        star=%016VR{star} cstar=%016VR{cstar}\n"
+                    "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
+                    );
+
+    char szInstr[256];
+    DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
+                       DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                       szInstr, sizeof(szInstr), NULL);
+
+    RTAssertMsg2Weak("%s%s\n", szRegs, szInstr);
+#else
+    RTAssertMsg2Weak("cs:rip=%04x:%RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip);
+#endif
+}
+
+/** @} */
+
+
+
+/** @name   Register Access.
+ * @{
+ */
+
+/**
+ * Adds a 8-bit signed jump offset to RIP/EIP/IP.
+ *
+ * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
+ * segment limit.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr             Instruction size.
+ * @param   offNextInstr        The offset of the next instruction.
+ * @param   enmEffOpSize        Effective operand size.
+ */
+VBOXSTRICTRC iemRegRipRelativeJumpS8AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int8_t offNextInstr,
+                                                        IEMMODE enmEffOpSize) RT_NOEXCEPT
+{
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + (int16_t)offNextInstr;
+            if (RT_LIKELY(   uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
+                          || pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT /* no CS limit checks in 64-bit mode */))
+                pVCpu->cpum.GstCtx.rip = uNewIp;
+            else
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+            Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
+
+            uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + (int32_t)offNextInstr;
+            if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
+                pVCpu->cpum.GstCtx.rip = uNewEip;
+            else
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            break;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
+
+            uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
+            if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
+                pVCpu->cpum.GstCtx.rip = uNewRip;
+            else
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            break;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = cbInstr;
+#endif
+
+    /*
+     * Clear RF and finish the instruction (maybe raise #DB).
+     */
+    return iemRegFinishClearingRF(pVCpu);
+}
+
+
+/**
+ * Adds a 16-bit signed jump offset to RIP/EIP/IP.
+ *
+ * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
+ * segment limit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr             Instruction size.
+ * @param   offNextInstr        The offset of the next instruction.
+ */
+VBOXSTRICTRC iemRegRipRelativeJumpS16AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int16_t offNextInstr) RT_NOEXCEPT
+{
+    Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT);
+
+    uint16_t const uNewIp = pVCpu->cpum.GstCtx.ip + cbInstr + offNextInstr;
+    if (RT_LIKELY(   uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
+                  || pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT /* no limit checking in 64-bit mode */))
+        pVCpu->cpum.GstCtx.rip = uNewIp;
+    else
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    /*
+     * Clear RF and finish the instruction (maybe raise #DB).
+     */
+    return iemRegFinishClearingRF(pVCpu);
+}
+
+
+/**
+ * Adds a 32-bit signed jump offset to RIP/EIP/IP.
+ *
+ * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
+ * segment limit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbInstr             Instruction size.
+ * @param   offNextInstr        The offset of the next instruction.
+ * @param   enmEffOpSize        Effective operand size.
+ */
+VBOXSTRICTRC iemRegRipRelativeJumpS32AndFinishClearingRF(PVMCPUCC pVCpu, uint8_t cbInstr, int32_t offNextInstr,
+                                                         IEMMODE enmEffOpSize) RT_NOEXCEPT
+{
+    if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX); Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+
+        uint32_t const uNewEip = pVCpu->cpum.GstCtx.eip + cbInstr + offNextInstr;
+        if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
+            pVCpu->cpum.GstCtx.rip = uNewEip;
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    else
+    {
+        Assert(enmEffOpSize == IEMMODE_64BIT);
+
+        uint64_t const uNewRip = pVCpu->cpum.GstCtx.rip + cbInstr + (int64_t)offNextInstr;
+        if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
+            pVCpu->cpum.GstCtx.rip = uNewRip;
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    /*
+     * Clear RF and finish the instruction (maybe raise #DB).
+     */
+    return iemRegFinishClearingRF(pVCpu);
+}
+
+
+/**
+ * Performs a near jump to the specified address.
+ *
+ * May raise a \#GP(0) if the new IP outside the code segment limit.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   uNewIp              The new IP value.
+ */
+VBOXSTRICTRC iemRegRipJumpU16AndFinishClearningRF(PVMCPUCC pVCpu, uint16_t uNewIp) RT_NOEXCEPT
+{
+    if (RT_LIKELY(   uNewIp <= pVCpu->cpum.GstCtx.cs.u32Limit
+                  || pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT /* no limit checks in 64-bit mode */))
+        pVCpu->cpum.GstCtx.rip = uNewIp;
+    else
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    /** @todo Test 16-bit jump in 64-bit mode.  */
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    /*
+     * Clear RF and finish the instruction (maybe raise #DB).
+     */
+    return iemRegFinishClearingRF(pVCpu);
+}
+
+
+/**
+ * Performs a near jump to the specified address.
+ *
+ * May raise a \#GP(0) if the new RIP is outside the code segment limit.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   uNewEip             The new EIP value.
+ */
+VBOXSTRICTRC iemRegRipJumpU32AndFinishClearningRF(PVMCPUCC pVCpu, uint32_t uNewEip) RT_NOEXCEPT
+{
+    Assert(pVCpu->cpum.GstCtx.rip <= UINT32_MAX);
+    Assert(pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+
+    if (RT_LIKELY(uNewEip <= pVCpu->cpum.GstCtx.cs.u32Limit))
+        pVCpu->cpum.GstCtx.rip = uNewEip;
+    else
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    /*
+     * Clear RF and finish the instruction (maybe raise #DB).
+     */
+    return iemRegFinishClearingRF(pVCpu);
+}
+
+
+/**
+ * Performs a near jump to the specified address.
+ *
+ * May raise a \#GP(0) if the new RIP is non-canonical or outside the code
+ * segment limit.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   uNewRip             The new RIP value.
+ */
+VBOXSTRICTRC iemRegRipJumpU64AndFinishClearningRF(PVMCPUCC pVCpu, uint64_t uNewRip) RT_NOEXCEPT
+{
+    Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
+
+    if (RT_LIKELY(IEM_IS_CANONICAL(uNewRip)))
+        pVCpu->cpum.GstCtx.rip = uNewRip;
+    else
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+#ifndef IEM_WITH_CODE_TLB
+    /* Flush the prefetch buffer. */
+    pVCpu->iem.s.cbOpcode = IEM_GET_INSTR_LEN(pVCpu);
+#endif
+
+    /*
+     * Clear RF and finish the instruction (maybe raise #DB).
+     */
+    return iemRegFinishClearingRF(pVCpu);
+}
+
+/** @}  */
+
+
+/** @name   FPU access and helpers.
+ *
+ * @{
+ */
+
+/**
+ * Updates the x87.DS and FPUDP registers.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pFpuCtx             The FPU context.
+ * @param   iEffSeg             The effective segment register.
+ * @param   GCPtrEff            The effective address relative to @a iEffSeg.
+ */
+DECLINLINE(void) iemFpuUpdateDP(PVMCPUCC pVCpu, PX86FXSTATE pFpuCtx, uint8_t iEffSeg, RTGCPTR GCPtrEff)
+{
+    RTSEL sel;
+    switch (iEffSeg)
+    {
+        case X86_SREG_DS: sel = pVCpu->cpum.GstCtx.ds.Sel; break;
+        case X86_SREG_SS: sel = pVCpu->cpum.GstCtx.ss.Sel; break;
+        case X86_SREG_CS: sel = pVCpu->cpum.GstCtx.cs.Sel; break;
+        case X86_SREG_ES: sel = pVCpu->cpum.GstCtx.es.Sel; break;
+        case X86_SREG_FS: sel = pVCpu->cpum.GstCtx.fs.Sel; break;
+        case X86_SREG_GS: sel = pVCpu->cpum.GstCtx.gs.Sel; break;
+        default:
+            AssertMsgFailed(("%d\n", iEffSeg));
+            sel = pVCpu->cpum.GstCtx.ds.Sel;
+    }
+    /** @todo pFpuCtx->DS and FPUDP needs to be kept seperately. */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        pFpuCtx->DS    = 0;
+        pFpuCtx->FPUDP = (uint32_t)GCPtrEff + ((uint32_t)sel << 4);
+    }
+    else if (!IEM_IS_LONG_MODE(pVCpu))
+    {
+        pFpuCtx->DS    = sel;
+        pFpuCtx->FPUDP = GCPtrEff;
+    }
+    else
+        *(uint64_t *)&pFpuCtx->FPUDP = GCPtrEff;
+}
+
+
+/**
+ * Rotates the stack registers in the push direction.
+ *
+ * @param   pFpuCtx             The FPU context.
+ * @remarks This is a complete waste of time, but fxsave stores the registers in
+ *          stack order.
+ */
+DECLINLINE(void) iemFpuRotateStackPush(PX86FXSTATE pFpuCtx)
+{
+    RTFLOAT80U r80Tmp = pFpuCtx->aRegs[7].r80;
+    pFpuCtx->aRegs[7].r80 = pFpuCtx->aRegs[6].r80;
+    pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[5].r80;
+    pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[4].r80;
+    pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[3].r80;
+    pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[2].r80;
+    pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[1].r80;
+    pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[0].r80;
+    pFpuCtx->aRegs[0].r80 = r80Tmp;
+}
+
+
+/**
+ * Rotates the stack registers in the pop direction.
+ *
+ * @param   pFpuCtx             The FPU context.
+ * @remarks This is a complete waste of time, but fxsave stores the registers in
+ *          stack order.
+ */
+DECLINLINE(void) iemFpuRotateStackPop(PX86FXSTATE pFpuCtx)
+{
+    RTFLOAT80U r80Tmp = pFpuCtx->aRegs[0].r80;
+    pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[1].r80;
+    pFpuCtx->aRegs[1].r80 = pFpuCtx->aRegs[2].r80;
+    pFpuCtx->aRegs[2].r80 = pFpuCtx->aRegs[3].r80;
+    pFpuCtx->aRegs[3].r80 = pFpuCtx->aRegs[4].r80;
+    pFpuCtx->aRegs[4].r80 = pFpuCtx->aRegs[5].r80;
+    pFpuCtx->aRegs[5].r80 = pFpuCtx->aRegs[6].r80;
+    pFpuCtx->aRegs[6].r80 = pFpuCtx->aRegs[7].r80;
+    pFpuCtx->aRegs[7].r80 = r80Tmp;
+}
+
+
+/**
+ * Updates FSW and pushes a FPU result onto the FPU stack if no pending
+ * exception prevents it.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The FPU operation result to push.
+ * @param   pFpuCtx             The FPU context.
+ */
+static void iemFpuMaybePushResult(PVMCPU pVCpu, PIEMFPURESULT pResult, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
+{
+    /* Update FSW and bail if there are pending exceptions afterwards. */
+    uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
+    fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
+    if (   (fFsw         & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
+        & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
+    {
+        if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FCW & X86_FSW_ES))
+            Log11(("iemFpuMaybePushResult: %04x:%08RX64: FSW %#x -> %#x\n",
+                   pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
+        pFpuCtx->FSW = fFsw;
+        return;
+    }
+
+    uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
+    if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
+    {
+        /* All is fine, push the actual value. */
+        pFpuCtx->FTW |= RT_BIT(iNewTop);
+        pFpuCtx->aRegs[7].r80 = pResult->r80Result;
+    }
+    else if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked stack overflow, push QNaN. */
+        fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
+        iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
+    }
+    else
+    {
+        /* Raise stack overflow, don't push anything. */
+        pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
+        Log11(("iemFpuMaybePushResult: %04x:%08RX64: stack overflow (FSW=%#x)\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
+        return;
+    }
+
+    fFsw &= ~X86_FSW_TOP_MASK;
+    fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
+    pFpuCtx->FSW = fFsw;
+
+    iemFpuRotateStackPush(pFpuCtx);
+    RT_NOREF(pVCpu);
+}
+
+
+/**
+ * Stores a result in a FPU register and updates the FSW and FTW.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pFpuCtx             The FPU context.
+ * @param   pResult             The result to store.
+ * @param   iStReg              Which FPU register to store it in.
+ */
+static void iemFpuStoreResultOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
+{
+    Assert(iStReg < 8);
+    uint16_t       fNewFsw = pFpuCtx->FSW;
+    uint16_t const iReg    = (X86_FSW_TOP_GET(fNewFsw) + iStReg) & X86_FSW_TOP_SMASK;
+    fNewFsw &= ~X86_FSW_C_MASK;
+    fNewFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
+    if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
+        Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
+    pFpuCtx->FSW  = fNewFsw;
+    pFpuCtx->FTW |= RT_BIT(iReg);
+    pFpuCtx->aRegs[iStReg].r80 = pResult->r80Result;
+    RT_NOREF(pVCpu);
+}
+
+
+/**
+ * Only updates the FPU status word (FSW) with the result of the current
+ * instruction.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pFpuCtx             The FPU context.
+ * @param   u16FSW              The FSW output of the current instruction.
+ */
+static void iemFpuUpdateFSWOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint16_t u16FSW) RT_NOEXCEPT
+{
+    uint16_t fNewFsw = pFpuCtx->FSW;
+    fNewFsw &= ~X86_FSW_C_MASK;
+    fNewFsw |= u16FSW & ~X86_FSW_TOP_MASK;
+    if ((fNewFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
+        Log11(("iemFpuStoreResultOnly: %04x:%08RX64: FSW %#x -> %#x\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fNewFsw));
+    pFpuCtx->FSW = fNewFsw;
+    RT_NOREF(pVCpu);
+}
+
+
+/**
+ * Pops one item off the FPU stack if no pending exception prevents it.
+ *
+ * @param   pFpuCtx             The FPU context.
+ */
+static void iemFpuMaybePopOne(PX86FXSTATE pFpuCtx) RT_NOEXCEPT
+{
+    /* Check pending exceptions. */
+    uint16_t uFSW = pFpuCtx->FSW;
+    if (   (pFpuCtx->FSW & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
+        & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
+        return;
+
+    /* TOP--. */
+    uint16_t iOldTop = uFSW & X86_FSW_TOP_MASK;
+    uFSW &= ~X86_FSW_TOP_MASK;
+    uFSW |= (iOldTop + (UINT16_C(9) << X86_FSW_TOP_SHIFT)) & X86_FSW_TOP_MASK;
+    pFpuCtx->FSW = uFSW;
+
+    /* Mark the previous ST0 as empty. */
+    iOldTop >>= X86_FSW_TOP_SHIFT;
+    pFpuCtx->FTW &= ~RT_BIT(iOldTop);
+
+    /* Rotate the registers. */
+    iemFpuRotateStackPop(pFpuCtx);
+}
+
+
+/**
+ * Pushes a FPU result onto the FPU stack if no pending exception prevents it.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The FPU operation result to push.
+ */
+void iemFpuPushResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
+}
+
+
+/**
+ * Pushes a FPU result onto the FPU stack if no pending exception prevents it,
+ * and sets FPUDP and FPUDS.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The FPU operation result to push.
+ * @param   iEffSeg             The effective segment register.
+ * @param   GCPtrEff            The effective address relative to @a iEffSeg.
+ */
+void iemFpuPushResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuMaybePushResult(pVCpu, pResult, pFpuCtx);
+}
+
+
+/**
+ * Replace ST0 with the first value and push the second onto the FPU stack,
+ * unless a pending exception prevents it.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The FPU operation result to store and push.
+ */
+void iemFpuPushResultTwo(PVMCPUCC pVCpu, PIEMFPURESULTTWO pResult) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+
+    /* Update FSW and bail if there are pending exceptions afterwards. */
+    uint16_t fFsw = pFpuCtx->FSW & ~X86_FSW_C_MASK;
+    fFsw |= pResult->FSW & ~X86_FSW_TOP_MASK;
+    if (   (fFsw         & (X86_FSW_IE | X86_FSW_ZE | X86_FSW_DE))
+        & ~(pFpuCtx->FCW & (X86_FCW_IM | X86_FCW_ZM | X86_FCW_DM)))
+    {
+        if ((fFsw & X86_FSW_ES) && !(pFpuCtx->FSW & X86_FSW_ES))
+            Log11(("iemFpuPushResultTwo: %04x:%08RX64: FSW %#x -> %#x\n",
+                   pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW, fFsw));
+        pFpuCtx->FSW = fFsw;
+        return;
+    }
+
+    uint16_t iNewTop = (X86_FSW_TOP_GET(fFsw) + 7) & X86_FSW_TOP_SMASK;
+    if (!(pFpuCtx->FTW & RT_BIT(iNewTop)))
+    {
+        /* All is fine, push the actual value. */
+        pFpuCtx->FTW |= RT_BIT(iNewTop);
+        pFpuCtx->aRegs[0].r80 = pResult->r80Result1;
+        pFpuCtx->aRegs[7].r80 = pResult->r80Result2;
+    }
+    else if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked stack overflow, push QNaN. */
+        fFsw |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1;
+        iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
+        iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
+    }
+    else
+    {
+        /* Raise stack overflow, don't push anything. */
+        pFpuCtx->FSW |= pResult->FSW & ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_C1 | X86_FSW_B | X86_FSW_ES;
+        Log11(("iemFpuPushResultTwo: %04x:%08RX64: stack overflow (FSW=%#x)\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
+        return;
+    }
+
+    fFsw &= ~X86_FSW_TOP_MASK;
+    fFsw |= iNewTop << X86_FSW_TOP_SHIFT;
+    pFpuCtx->FSW = fFsw;
+
+    iemFpuRotateStackPush(pFpuCtx);
+}
+
+
+/**
+ * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
+ * FOP.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The result to store.
+ * @param   iStReg              Which FPU register to store it in.
+ */
+void iemFpuStoreResult(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
+}
+
+
+/**
+ * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, and
+ * FOP, and then pops the stack.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The result to store.
+ * @param   iStReg              Which FPU register to store it in.
+ */
+void iemFpuStoreResultThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+/**
+ * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
+ * FPUDP, and FPUDS.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The result to store.
+ * @param   iStReg              Which FPU register to store it in.
+ * @param   iEffSeg             The effective memory operand selector register.
+ * @param   GCPtrEff            The effective memory operand offset.
+ */
+void iemFpuStoreResultWithMemOp(PVMCPUCC pVCpu, PIEMFPURESULT pResult, uint8_t iStReg,
+                                uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
+}
+
+
+/**
+ * Stores a result in a FPU register, updates the FSW, FTW, FPUIP, FPUCS, FOP,
+ * FPUDP, and FPUDS, and then pops the stack.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The result to store.
+ * @param   iStReg              Which FPU register to store it in.
+ * @param   iEffSeg             The effective memory operand selector register.
+ * @param   GCPtrEff            The effective memory operand offset.
+ */
+void iemFpuStoreResultWithMemOpThenPop(PVMCPUCC pVCpu, PIEMFPURESULT pResult,
+                                       uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStoreResultOnly(pVCpu, pFpuCtx, pResult, iStReg);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+/**
+ * Updates the FOP, FPUIP, and FPUCS.  For FNOP.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+void iemFpuUpdateOpcodeAndIp(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+}
+
+
+/**
+ * Updates the FSW, FOP, FPUIP, and FPUCS.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16FSW              The FSW from the current instruction.
+ */
+void iemFpuUpdateFSW(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
+}
+
+
+/**
+ * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16FSW              The FSW from the current instruction.
+ */
+void iemFpuUpdateFSWThenPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+/**
+ * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16FSW              The FSW from the current instruction.
+ * @param   iEffSeg             The effective memory operand selector register.
+ * @param   GCPtrEff            The effective memory operand offset.
+ */
+void iemFpuUpdateFSWWithMemOp(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
+}
+
+
+/**
+ * Updates the FSW, FOP, FPUIP, and FPUCS, then pops the stack twice.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16FSW              The FSW from the current instruction.
+ */
+void iemFpuUpdateFSWThenPopPop(PVMCPUCC pVCpu, uint16_t u16FSW) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
+    iemFpuMaybePopOne(pFpuCtx);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+/**
+ * Updates the FSW, FOP, FPUIP, FPUCS, FPUDP, and FPUDS, then pops the stack.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16FSW              The FSW from the current instruction.
+ * @param   iEffSeg             The effective memory operand selector register.
+ * @param   GCPtrEff            The effective memory operand offset.
+ */
+void iemFpuUpdateFSWWithMemOpThenPop(PVMCPUCC pVCpu, uint16_t u16FSW, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuUpdateFSWOnly(pVCpu, pFpuCtx, u16FSW);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+/**
+ * Worker routine for raising an FPU stack underflow exception.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pFpuCtx             The FPU context.
+ * @param   iStReg              The stack register being accessed.
+ */
+static void iemFpuStackUnderflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx, uint8_t iStReg)
+{
+    Assert(iStReg < 8 || iStReg == UINT8_MAX);
+    if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked underflow. */
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
+        uint16_t iReg = (X86_FSW_TOP_GET(pFpuCtx->FSW) + iStReg) & X86_FSW_TOP_SMASK;
+        if (iStReg != UINT8_MAX)
+        {
+            pFpuCtx->FTW |= RT_BIT(iReg);
+            iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
+        }
+    }
+    else
+    {
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+        Log11(("iemFpuStackUnderflowOnly: %04x:%08RX64: underflow (FSW=%#x)\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
+    }
+    RT_NOREF(pVCpu);
+}
+
+
+/**
+ * Raises a FPU stack underflow exception.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iStReg              The destination register that should be loaded
+ *                              with QNaN if \#IS is not masked. Specify
+ *                              UINT8_MAX if none (like for fcom).
+ */
+void iemFpuStackUnderflow(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
+}
+
+
+void iemFpuStackUnderflowWithMemOp(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
+}
+
+
+void iemFpuStackUnderflowThenPop(PVMCPUCC pVCpu, uint8_t iStReg) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+void iemFpuStackUnderflowWithMemOpThenPop(PVMCPUCC pVCpu, uint8_t iStReg, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, iStReg);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+void iemFpuStackUnderflowThenPopPop(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackUnderflowOnly(pVCpu, pFpuCtx, UINT8_MAX);
+    iemFpuMaybePopOne(pFpuCtx);
+    iemFpuMaybePopOne(pFpuCtx);
+}
+
+
+void iemFpuStackPushUnderflow(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+
+    if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked overflow - Push QNaN. */
+        uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
+        pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
+        pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
+        pFpuCtx->FTW |= RT_BIT(iNewTop);
+        iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
+        iemFpuRotateStackPush(pFpuCtx);
+    }
+    else
+    {
+        /* Exception pending - don't change TOP or the register stack. */
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+        Log11(("iemFpuStackPushUnderflow: %04x:%08RX64: underflow (FSW=%#x)\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
+    }
+}
+
+
+void iemFpuStackPushUnderflowTwo(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+
+    if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked overflow - Push QNaN. */
+        uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
+        pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
+        pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
+        pFpuCtx->FTW |= RT_BIT(iNewTop);
+        iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
+        iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
+        iemFpuRotateStackPush(pFpuCtx);
+    }
+    else
+    {
+        /* Exception pending - don't change TOP or the register stack. */
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+        Log11(("iemFpuStackPushUnderflowTwo: %04x:%08RX64: underflow (FSW=%#x)\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
+    }
+}
+
+
+/**
+ * Worker routine for raising an FPU stack overflow exception on a push.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pFpuCtx             The FPU context.
+ */
+static void iemFpuStackPushOverflowOnly(PVMCPU pVCpu, PX86FXSTATE pFpuCtx) RT_NOEXCEPT
+{
+    if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked overflow. */
+        uint16_t iNewTop = (X86_FSW_TOP_GET(pFpuCtx->FSW) + 7) & X86_FSW_TOP_SMASK;
+        pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_C_MASK);
+        pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF;
+        pFpuCtx->FSW |= iNewTop << X86_FSW_TOP_SHIFT;
+        pFpuCtx->FTW |= RT_BIT(iNewTop);
+        iemFpuStoreQNan(&pFpuCtx->aRegs[7].r80);
+        iemFpuRotateStackPush(pFpuCtx);
+    }
+    else
+    {
+        /* Exception pending - don't change TOP or the register stack. */
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+        Log11(("iemFpuStackPushOverflowOnly: %04x:%08RX64: overflow (FSW=%#x)\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
+    }
+    RT_NOREF(pVCpu);
+}
+
+
+/**
+ * Raises a FPU stack overflow exception on a push.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+void iemFpuStackPushOverflow(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
+}
+
+
+/**
+ * Raises a FPU stack overflow exception on a push with a memory operand.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iEffSeg             The effective memory operand selector register.
+ * @param   GCPtrEff            The effective memory operand offset.
+ */
+void iemFpuStackPushOverflowWithMemOp(PVMCPUCC pVCpu, uint8_t iEffSeg, RTGCPTR GCPtrEff) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemFpuUpdateDP(pVCpu, pFpuCtx, iEffSeg, GCPtrEff);
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemFpuStackPushOverflowOnly(pVCpu, pFpuCtx);
+}
+
+/** @}  */
+
+
+/** @name   SSE+AVX SIMD access and helpers.
+ *
+ * @{
+ */
+/**
+ * Stores a result in a SIMD XMM register, updates the MXCSR.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pResult             The result to store.
+ * @param   iXmmReg             Which SIMD XMM register to store the result in.
+ */
+void iemSseStoreResult(PVMCPUCC pVCpu, PCIEMSSERESULT pResult, uint8_t iXmmReg) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    pFpuCtx->MXCSR |= pResult->MXCSR & X86_MXCSR_XCPT_FLAGS;
+
+    /* The result is only updated if there is no unmasked exception pending. */
+    if ((  ~((pFpuCtx->MXCSR & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT)
+         & (pFpuCtx->MXCSR & X86_MXCSR_XCPT_FLAGS)) == 0)
+        pVCpu->cpum.GstCtx.XState.x87.aXMM[iXmmReg] = pResult->uResult;
+}
+
+
+/**
+ * Updates the MXCSR.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   fMxcsr              The new MXCSR value.
+ */
+void iemSseUpdateMxcsr(PVMCPUCC pVCpu, uint32_t fMxcsr) RT_NOEXCEPT
+{
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    pFpuCtx->MXCSR |= fMxcsr & X86_MXCSR_XCPT_FLAGS;
+}
+/** @}  */
+
+
+/** @name   Memory access.
+ *
+ * @{
+ */
+
+
+/**
+ * Updates the IEMCPU::cbWritten counter if applicable.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   fAccess             The access being accounted for.
+ * @param   cbMem               The access size.
+ */
+DECL_FORCE_INLINE(void) iemMemUpdateWrittenCounter(PVMCPUCC pVCpu, uint32_t fAccess, size_t cbMem)
+{
+    if (   (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_STACK | IEM_ACCESS_TYPE_WRITE)
+        || (fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE)) == (IEM_ACCESS_WHAT_DATA  | IEM_ACCESS_TYPE_WRITE) )
+        pVCpu->iem.s.cbWritten += (uint32_t)cbMem;
+}
+
+
+/**
+ * Applies the segment limit, base and attributes.
+ *
+ * This may raise a \#GP or \#SS.
+ *
+ * @returns VBox strict status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   fAccess             The kind of access which is being performed.
+ * @param   iSegReg             The index of the segment register to apply.
+ *                              This is UINT8_MAX if none (for IDT, GDT, LDT,
+ *                              TSS, ++).
+ * @param   cbMem               The access size.
+ * @param   pGCPtrMem           Pointer to the guest memory address to apply
+ *                              segmentation to.  Input and output parameter.
+ */
+VBOXSTRICTRC iemMemApplySegment(PVMCPUCC pVCpu, uint32_t fAccess, uint8_t iSegReg, size_t cbMem, PRTGCPTR pGCPtrMem) RT_NOEXCEPT
+{
+    if (iSegReg == UINT8_MAX)
+        return VINF_SUCCESS;
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+    PCPUMSELREGHID pSel = iemSRegGetHid(pVCpu, iSegReg);
+    switch (pVCpu->iem.s.enmCpuMode)
+    {
+        case IEMMODE_16BIT:
+        case IEMMODE_32BIT:
+        {
+            RTGCPTR32 GCPtrFirst32 = (RTGCPTR32)*pGCPtrMem;
+            RTGCPTR32 GCPtrLast32  = GCPtrFirst32 + (uint32_t)cbMem - 1;
+
+            if (   pSel->Attr.n.u1Present
+                && !pSel->Attr.n.u1Unusable)
+            {
+                Assert(pSel->Attr.n.u1DescType);
+                if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
+                {
+                    if (   (fAccess & IEM_ACCESS_TYPE_WRITE)
+                        && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE) )
+                        return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
+
+                    if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
+                    {
+                        /** @todo CPL check. */
+                    }
+
+                    /*
+                     * There are two kinds of data selectors, normal and expand down.
+                     */
+                    if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
+                    {
+                        if (   GCPtrFirst32 > pSel->u32Limit
+                            || GCPtrLast32  > pSel->u32Limit) /* yes, in real mode too (since 80286). */
+                            return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
+                    }
+                    else
+                    {
+                       /*
+                        * The upper boundary is defined by the B bit, not the G bit!
+                        */
+                       if (   GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
+                           || GCPtrLast32  > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
+                          return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
+                    }
+                    *pGCPtrMem = GCPtrFirst32 += (uint32_t)pSel->u64Base;
+                }
+                else
+                {
+                    /*
+                     * Code selector and usually be used to read thru, writing is
+                     * only permitted in real and V8086 mode.
+                     */
+                    if (   (   (fAccess & IEM_ACCESS_TYPE_WRITE)
+                            || (   (fAccess & IEM_ACCESS_TYPE_READ)
+                               && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)) )
+                        && !IEM_IS_REAL_OR_V86_MODE(pVCpu) )
+                        return iemRaiseSelectorInvalidAccess(pVCpu, iSegReg, fAccess);
+
+                    if (   GCPtrFirst32 > pSel->u32Limit
+                        || GCPtrLast32  > pSel->u32Limit) /* yes, in real mode too (since 80286). */
+                        return iemRaiseSelectorBounds(pVCpu, iSegReg, fAccess);
+
+                    if (!IEM_IS_REAL_OR_V86_MODE(pVCpu))
+                    {
+                        /** @todo CPL check. */
+                    }
+
+                    *pGCPtrMem  = GCPtrFirst32 += (uint32_t)pSel->u64Base;
+                }
+            }
+            else
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            RTGCPTR GCPtrMem = *pGCPtrMem;
+            if (iSegReg == X86_SREG_GS || iSegReg == X86_SREG_FS)
+                *pGCPtrMem = GCPtrMem + pSel->u64Base;
+
+            Assert(cbMem >= 1);
+            if (RT_LIKELY(X86_IS_CANONICAL(GCPtrMem) && X86_IS_CANONICAL(GCPtrMem + cbMem - 1)))
+                return VINF_SUCCESS;
+            /** @todo We should probably raise \#SS(0) here if segment is SS; see AMD spec.
+             *        4.12.2 "Data Limit Checks in 64-bit Mode". */
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        default:
+            AssertFailedReturn(VERR_IEM_IPE_7);
+    }
+}
+
+
+/**
+ * Translates a virtual address to a physical physical address and checks if we
+ * can access the page as specified.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   GCPtrMem            The virtual address.
+ * @param   cbAccess            The access size, for raising \#PF correctly for
+ *                              FXSAVE and such.
+ * @param   fAccess             The intended access.
+ * @param   pGCPhysMem          Where to return the physical address.
+ */
+VBOXSTRICTRC iemMemPageTranslateAndCheckAccess(PVMCPUCC pVCpu, RTGCPTR GCPtrMem, uint32_t cbAccess,
+                                               uint32_t fAccess, PRTGCPHYS pGCPhysMem) RT_NOEXCEPT
+{
+    /** @todo Need a different PGM interface here.  We're currently using
+     *        generic / REM interfaces. this won't cut it for R0. */
+    /** @todo If/when PGM handles paged real-mode, we can remove the hack in
+     *        iemSvmWorldSwitch/iemVmxWorldSwitch to work around raising a page-fault
+     *        here. */
+    PGMPTWALK Walk;
+    int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
+    if (RT_FAILURE(rc))
+    {
+        Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
+        /** @todo Check unassigned memory in unpaged mode. */
+        /** @todo Reserved bits in page tables. Requires new PGM interface. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        if (Walk.fFailed & PGM_WALKFAIL_EPT)
+            IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
+#endif
+        *pGCPhysMem = NIL_RTGCPHYS;
+        return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess, rc);
+    }
+
+    /* If the page is writable and does not have the no-exec bit set, all
+       access is allowed.  Otherwise we'll have to check more carefully... */
+    if ((Walk.fEffective & (X86_PTE_RW | X86_PTE_US | X86_PTE_PAE_NX)) != (X86_PTE_RW | X86_PTE_US))
+    {
+        /* Write to read only memory? */
+        if (   (fAccess & IEM_ACCESS_TYPE_WRITE)
+            && !(Walk.fEffective & X86_PTE_RW)
+            && (   (    pVCpu->iem.s.uCpl == 3
+                    && !(fAccess & IEM_ACCESS_WHAT_SYS))
+                || (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
+        {
+            Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
+            *pGCPhysMem = NIL_RTGCPHYS;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            if (Walk.fFailed & PGM_WALKFAIL_EPT)
+                IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+#endif
+            return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
+        }
+
+        /* Kernel memory accessed by userland? */
+        if (   !(Walk.fEffective & X86_PTE_US)
+            && pVCpu->iem.s.uCpl == 3
+            && !(fAccess & IEM_ACCESS_WHAT_SYS))
+        {
+            Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
+            *pGCPhysMem = NIL_RTGCPHYS;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            if (Walk.fFailed & PGM_WALKFAIL_EPT)
+                IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+#endif
+            return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess, VERR_ACCESS_DENIED);
+        }
+
+        /* Executing non-executable memory? */
+        if (   (fAccess & IEM_ACCESS_TYPE_EXEC)
+            && (Walk.fEffective & X86_PTE_PAE_NX)
+            && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_NXE) )
+        {
+            Log(("iemMemPageTranslateAndCheckAccess: GCPtrMem=%RGv - NX -> #PF\n", GCPtrMem));
+            *pGCPhysMem = NIL_RTGCPHYS;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            if (Walk.fFailed & PGM_WALKFAIL_EPT)
+                IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+#endif
+            return iemRaisePageFault(pVCpu, GCPtrMem, cbAccess, fAccess & ~(IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_WRITE),
+                                     VERR_ACCESS_DENIED);
+        }
+    }
+
+    /*
+     * Set the dirty / access flags.
+     * ASSUMES this is set when the address is translated rather than on committ...
+     */
+    /** @todo testcase: check when A and D bits are actually set by the CPU.  */
+    uint32_t fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A;
+    if ((Walk.fEffective & fAccessedDirty) != fAccessedDirty)
+    {
+        int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
+        AssertRC(rc2);
+        /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
+        Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
+    }
+
+    RTGCPHYS const GCPhys = Walk.GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
+    *pGCPhysMem = GCPhys;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Looks up a memory mapping entry.
+ *
+ * @returns The mapping index (positive) or VERR_NOT_FOUND (negative).
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem           The memory address.
+ * @param   fAccess         The access to.
+ */
+DECLINLINE(int) iemMapLookup(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess)
+{
+    Assert(pVCpu->iem.s.cActiveMappings <= RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
+    fAccess &= IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK;
+    if (   pVCpu->iem.s.aMemMappings[0].pv == pvMem
+        && (pVCpu->iem.s.aMemMappings[0].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
+        return 0;
+    if (   pVCpu->iem.s.aMemMappings[1].pv == pvMem
+        && (pVCpu->iem.s.aMemMappings[1].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
+        return 1;
+    if (   pVCpu->iem.s.aMemMappings[2].pv == pvMem
+        && (pVCpu->iem.s.aMemMappings[2].fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == fAccess)
+        return 2;
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Finds a free memmap entry when using iNextMapping doesn't work.
+ *
+ * @returns Memory mapping index, 1024 on failure.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+static unsigned iemMemMapFindFree(PVMCPUCC pVCpu)
+{
+    /*
+     * The easy case.
+     */
+    if (pVCpu->iem.s.cActiveMappings == 0)
+    {
+        pVCpu->iem.s.iNextMapping = 1;
+        return 0;
+    }
+
+    /* There should be enough mappings for all instructions. */
+    AssertReturn(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings), 1024);
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->iem.s.aMemMappings); i++)
+        if (pVCpu->iem.s.aMemMappings[i].fAccess == IEM_ACCESS_INVALID)
+            return i;
+
+    AssertFailedReturn(1024);
+}
+
+
+/**
+ * Commits a bounce buffer that needs writing back and unmaps it.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   iMemMap         The index of the buffer to commit.
+ * @param   fPostponeFail   Whether we can postpone writer failures to ring-3.
+ *                          Always false in ring-3, obviously.
+ */
+static VBOXSTRICTRC iemMemBounceBufferCommitAndUnmap(PVMCPUCC pVCpu, unsigned iMemMap, bool fPostponeFail)
+{
+    Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
+    Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
+#ifdef IN_RING3
+    Assert(!fPostponeFail);
+    RT_NOREF_PV(fPostponeFail);
+#endif
+
+    /*
+     * Do the writing.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned)
+    {
+        uint16_t const  cbFirst  = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
+        uint16_t const  cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
+        uint8_t const  *pbBuf    = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
+        if (!pVCpu->iem.s.fBypassHandlers)
+        {
+            /*
+             * Carefully and efficiently dealing with access handler return
+             * codes make this a little bloated.
+             */
+            VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM,
+                                                 pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
+                                                 pbBuf,
+                                                 cbFirst,
+                                                 PGMACCESSORIGIN_IEM);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                if (cbSecond)
+                {
+                    rcStrict = PGMPhysWrite(pVM,
+                                            pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
+                                            pbBuf + cbFirst,
+                                            cbSecond,
+                                            PGMACCESSORIGIN_IEM);
+                    if (rcStrict == VINF_SUCCESS)
+                    { /* nothing */ }
+                    else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                    }
+#ifndef IN_RING3
+                    else if (fPostponeFail)
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                        pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
+                        VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
+                        return iemSetPassUpStatus(pVCpu, rcStrict);
+                    }
+#endif
+                    else
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                        return rcStrict;
+                    }
+                }
+            }
+            else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+            {
+                if (!cbSecond)
+                {
+                    Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc\n",
+                         pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict) ));
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                }
+                else
+                {
+                    VBOXSTRICTRC rcStrict2 = PGMPhysWrite(pVM,
+                                                          pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
+                                                          pbBuf + cbFirst,
+                                                          cbSecond,
+                                                          PGMACCESSORIGIN_IEM);
+                    if (rcStrict2 == VINF_SUCCESS)
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
+                        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                    }
+                    else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
+                        PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
+                        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                    }
+#ifndef IN_RING3
+                    else if (fPostponeFail)
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                        pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_2ND;
+                        VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
+                        return iemSetPassUpStatus(pVCpu, rcStrict);
+                    }
+#endif
+                    else
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict2) ));
+                        return rcStrict2;
+                    }
+                }
+            }
+#ifndef IN_RING3
+            else if (fPostponeFail)
+            {
+                Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (postponed)\n",
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                if (!cbSecond)
+                    pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST;
+                else
+                    pVCpu->iem.s.aMemMappings[iMemMap].fAccess |= IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND;
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_IEM);
+                return iemSetPassUpStatus(pVCpu, rcStrict);
+            }
+#endif
+            else
+            {
+                Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysWrite GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, VBOXSTRICTRC_VAL(rcStrict),
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
+                return rcStrict;
+            }
+        }
+        else
+        {
+            /*
+             * No access handlers, much simpler.
+             */
+            int rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pbBuf, cbFirst);
+            if (RT_SUCCESS(rc))
+            {
+                if (cbSecond)
+                {
+                    rc = PGMPhysSimpleWriteGCPhys(pVM, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pbBuf + cbFirst, cbSecond);
+                    if (RT_SUCCESS(rc))
+                    { /* likely */ }
+                    else
+                    {
+                        Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x GCPhysSecond=%RGp/%#x %Rrc (!!)\n",
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                             pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond, rc));
+                        return rc;
+                    }
+                }
+            }
+            else
+            {
+                Log(("iemMemBounceBufferCommitAndUnmap: PGMPhysSimpleWriteGCPhys GCPhysFirst=%RGp/%#x %Rrc [GCPhysSecond=%RGp/%#x] (!!)\n",
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst, rc,
+                     pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond));
+                return rc;
+            }
+        }
+    }
+
+#if defined(IEM_LOG_MEMORY_WRITES)
+    Log(("IEM Wrote %RGp: %.*Rhxs\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
+         RT_MAX(RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst, 64), 1), &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0]));
+    if (pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond)
+        Log(("IEM Wrote %RGp: %.*Rhxs [2nd page]\n", pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
+             RT_MIN(pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond, 64),
+             &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst]));
+
+    size_t cbWrote = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst + pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
+    g_cbIemWrote = cbWrote;
+    memcpy(g_abIemWrote, &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0], RT_MIN(cbWrote, sizeof(g_abIemWrote)));
+#endif
+
+    /*
+     * Free the mapping entry.
+     */
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+    Assert(pVCpu->iem.s.cActiveMappings != 0);
+    pVCpu->iem.s.cActiveMappings--;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * iemMemMap worker that deals with a request crossing pages.
+ */
+static VBOXSTRICTRC
+iemMemBounceBufferMapCrossPage(PVMCPUCC pVCpu, int iMemMap, void **ppvMem, size_t cbMem, RTGCPTR GCPtrFirst, uint32_t fAccess)
+{
+    Assert(cbMem <= GUEST_PAGE_SIZE);
+
+    /*
+     * Do the address translations.
+     */
+    uint32_t const cbFirstPage  = GUEST_PAGE_SIZE - (uint32_t)(GCPtrFirst & GUEST_PAGE_OFFSET_MASK);
+    RTGCPHYS GCPhysFirst;
+    VBOXSTRICTRC rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrFirst, cbFirstPage, fAccess, &GCPhysFirst);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    Assert((GCPhysFirst & GUEST_PAGE_OFFSET_MASK) == (GCPtrFirst & GUEST_PAGE_OFFSET_MASK));
+
+    uint32_t const cbSecondPage = (uint32_t)cbMem - cbFirstPage;
+    RTGCPHYS GCPhysSecond;
+    rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, (GCPtrFirst + (cbMem - 1)) & ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK,
+                                                 cbSecondPage, fAccess, &GCPhysSecond);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    Assert((GCPhysSecond & GUEST_PAGE_OFFSET_MASK) == 0);
+    GCPhysSecond &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK; /** @todo why? */
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Read in the current memory content if it's a read, execute or partial
+     * write access.
+     */
+    uint8_t * const pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
+
+    if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
+    {
+        if (!pVCpu->iem.s.fBypassHandlers)
+        {
+            /*
+             * Must carefully deal with access handler status codes here,
+             * makes the code a bit bloated.
+             */
+            rcStrict = PGMPhysRead(pVM, GCPhysFirst, pbBuf, cbFirstPage, PGMACCESSORIGIN_IEM);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                rcStrict = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
+                if (rcStrict == VINF_SUCCESS)
+                { /*likely */ }
+                else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                else
+                {
+                    Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (!!)\n",
+                         GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict) ));
+                    return rcStrict;
+                }
+            }
+            else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+            {
+                VBOXSTRICTRC rcStrict2 = PGMPhysRead(pVM, GCPhysSecond, pbBuf + cbFirstPage, cbSecondPage, PGMACCESSORIGIN_IEM);
+                if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
+                {
+                    PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                }
+                else
+                {
+                    Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysSecond=%RGp rcStrict2=%Rrc (rcStrict=%Rrc) (!!)\n",
+                         GCPhysSecond, VBOXSTRICTRC_VAL(rcStrict2), VBOXSTRICTRC_VAL(rcStrict2) ));
+                    return rcStrict2;
+                }
+            }
+            else
+            {
+                Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
+                     GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
+                return rcStrict;
+            }
+        }
+        else
+        {
+            /*
+             * No informational status codes here, much more straight forward.
+             */
+            int rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf, GCPhysFirst, cbFirstPage);
+            if (RT_SUCCESS(rc))
+            {
+                Assert(rc == VINF_SUCCESS);
+                rc = PGMPhysSimpleReadGCPhys(pVM, pbBuf + cbFirstPage, GCPhysSecond, cbSecondPage);
+                if (RT_SUCCESS(rc))
+                    Assert(rc == VINF_SUCCESS);
+                else
+                {
+                    Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysSecond=%RGp rc=%Rrc (!!)\n", GCPhysSecond, rc));
+                    return rc;
+                }
+            }
+            else
+            {
+                Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rc=%Rrc (!!)\n", GCPhysFirst, rc));
+                return rc;
+            }
+        }
+    }
+#ifdef VBOX_STRICT
+    else
+        memset(pbBuf, 0xcc, cbMem);
+    if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
+        memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
+#endif
+    AssertCompileMemberAlignment(VMCPU, iem.s.aBounceBuffers, 64);
+
+    /*
+     * Commit the bounce buffer entry.
+     */
+    pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst    = GCPhysFirst;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond   = GCPhysSecond;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst        = (uint16_t)cbFirstPage;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond       = (uint16_t)cbSecondPage;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned    = false;
+    pVCpu->iem.s.aMemMappings[iMemMap].pv               = pbBuf;
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess          = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
+    pVCpu->iem.s.iNextMapping = iMemMap + 1;
+    pVCpu->iem.s.cActiveMappings++;
+
+    iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
+    *ppvMem = pbBuf;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * iemMemMap woker that deals with iemMemPageMap failures.
+ */
+static VBOXSTRICTRC iemMemBounceBufferMapPhys(PVMCPUCC pVCpu, unsigned iMemMap, void **ppvMem, size_t cbMem,
+                                              RTGCPHYS GCPhysFirst, uint32_t fAccess, VBOXSTRICTRC rcMap)
+{
+    /*
+     * Filter out conditions we can handle and the ones which shouldn't happen.
+     */
+    if (   rcMap != VERR_PGM_PHYS_TLB_CATCH_WRITE
+        && rcMap != VERR_PGM_PHYS_TLB_CATCH_ALL
+        && rcMap != VERR_PGM_PHYS_TLB_UNASSIGNED)
+    {
+        AssertReturn(RT_FAILURE_NP(rcMap), VERR_IEM_IPE_8);
+        return rcMap;
+    }
+    pVCpu->iem.s.cPotentialExits++;
+
+    /*
+     * Read in the current memory content if it's a read, execute or partial
+     * write access.
+     */
+    uint8_t *pbBuf = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
+    if (fAccess & (IEM_ACCESS_TYPE_READ | IEM_ACCESS_TYPE_EXEC | IEM_ACCESS_PARTIAL_WRITE))
+    {
+        if (rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED)
+            memset(pbBuf, 0xff, cbMem);
+        else
+        {
+            int rc;
+            if (!pVCpu->iem.s.fBypassHandlers)
+            {
+                VBOXSTRICTRC rcStrict = PGMPhysRead(pVCpu->CTX_SUFF(pVM), GCPhysFirst, pbBuf, cbMem, PGMACCESSORIGIN_IEM);
+                if (rcStrict == VINF_SUCCESS)
+                { /* nothing */ }
+                else if (PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                else
+                {
+                    Log(("iemMemBounceBufferMapPhys: PGMPhysRead GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
+                         GCPhysFirst, VBOXSTRICTRC_VAL(rcStrict) ));
+                    return rcStrict;
+                }
+            }
+            else
+            {
+                rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), pbBuf, GCPhysFirst, cbMem);
+                if (RT_SUCCESS(rc))
+                { /* likely */ }
+                else
+                {
+                    Log(("iemMemBounceBufferMapPhys: PGMPhysSimpleReadGCPhys GCPhysFirst=%RGp rcStrict=%Rrc (!!)\n",
+                         GCPhysFirst, rc));
+                    return rc;
+                }
+            }
+        }
+    }
+#ifdef VBOX_STRICT
+    else
+        memset(pbBuf, 0xcc, cbMem);
+#endif
+#ifdef VBOX_STRICT
+    if (cbMem < sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab))
+        memset(pbBuf + cbMem, 0xaa, sizeof(pVCpu->iem.s.aBounceBuffers[iMemMap].ab) - cbMem);
+#endif
+
+    /*
+     * Commit the bounce buffer entry.
+     */
+    pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst    = GCPhysFirst;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond   = NIL_RTGCPHYS;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst        = (uint16_t)cbMem;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond       = 0;
+    pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned    = rcMap == VERR_PGM_PHYS_TLB_UNASSIGNED;
+    pVCpu->iem.s.aMemMappings[iMemMap].pv               = pbBuf;
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess          = fAccess | IEM_ACCESS_BOUNCE_BUFFERED;
+    pVCpu->iem.s.iNextMapping = iMemMap + 1;
+    pVCpu->iem.s.cActiveMappings++;
+
+    iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
+    *ppvMem = pbBuf;
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Maps the specified guest memory for the given kind of access.
+ *
+ * This may be using bounce buffering of the memory if it's crossing a page
+ * boundary or if there is an access handler installed for any of it.  Because
+ * of lock prefix guarantees, we're in for some extra clutter when this
+ * happens.
+ *
+ * This may raise a \#GP, \#SS, \#PF or \#AC.
+ *
+ * @returns VBox strict status code.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ * @param   ppvMem      Where to return the pointer to the mapped memory.
+ * @param   cbMem       The number of bytes to map.  This is usually 1, 2, 4, 6,
+ *                      8, 12, 16, 32 or 512.  When used by string operations
+ *                      it can be up to a page.
+ * @param   iSegReg     The index of the segment register to use for this
+ *                      access.  The base and limits are checked. Use UINT8_MAX
+ *                      to indicate that no segmentation is required (for IDT,
+ *                      GDT and LDT accesses).
+ * @param   GCPtrMem    The address of the guest memory.
+ * @param   fAccess     How the memory is being accessed.  The
+ *                      IEM_ACCESS_TYPE_XXX bit is used to figure out how to map
+ *                      the memory, while the IEM_ACCESS_WHAT_XXX bit is used
+ *                      when raising exceptions.
+ * @param   uAlignCtl   Alignment control:
+ *                          - Bits 15:0 is the alignment mask.
+ *                          - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
+ *                            IEM_MEMMAP_F_ALIGN_SSE, and
+ *                            IEM_MEMMAP_F_ALIGN_GP_OR_AC.
+ *                      Pass zero to skip alignment.
+ */
+VBOXSTRICTRC iemMemMap(PVMCPUCC pVCpu, void **ppvMem, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem,
+                       uint32_t fAccess, uint32_t uAlignCtl) RT_NOEXCEPT
+{
+    /*
+     * Check the input and figure out which mapping entry to use.
+     */
+    Assert(cbMem <= sizeof(pVCpu->iem.s.aBounceBuffers[0]));
+    Assert(   cbMem <= 64 || cbMem == 512 || cbMem == 256 || cbMem == 108 || cbMem == 104 || cbMem == 102 || cbMem == 94
+           || (iSegReg == UINT8_MAX && uAlignCtl == 0 && fAccess == IEM_ACCESS_DATA_R /* for the CPUID logging interface */) );
+    Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
+    Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
+
+    unsigned iMemMap = pVCpu->iem.s.iNextMapping;
+    if (   iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
+        || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
+    {
+        iMemMap = iemMemMapFindFree(pVCpu);
+        AssertLogRelMsgReturn(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
+                              ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
+                               pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
+                               pVCpu->iem.s.aMemMappings[2].fAccess),
+                              VERR_IEM_IPE_9);
+    }
+
+    /*
+     * Map the memory, checking that we can actually access it.  If something
+     * slightly complicated happens, fall back on bounce buffering.
+     */
+    VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
+    if (rcStrict == VINF_SUCCESS)
+    { /* likely */ }
+    else
+        return rcStrict;
+
+    if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE) /* Crossing a page boundary? */
+    { /* likely */ }
+    else
+        return iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, ppvMem, cbMem, GCPtrMem, fAccess);
+
+    /*
+     * Alignment check.
+     */
+    if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
+    { /* likelyish */ }
+    else
+    {
+        /* Misaligned access. */
+        if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
+        {
+            if (   !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
+                || (   (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
+                    && (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
+            {
+                AssertCompile(X86_CR0_AM == X86_EFL_AC);
+
+                if (iemMemAreAlignmentChecksEnabled(pVCpu))
+                    return iemRaiseAlignmentCheckException(pVCpu);
+            }
+            else if (   (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
+                     && (GCPtrMem & 3) /* The value 4 matches 10980xe's FXSAVE and helps make bs3-cpu-basic2 work. */
+                    /** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
+                     * implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++.  Using 4 for now as
+                     * that's what FXSAVE does on a 10980xe. */
+                     && iemMemAreAlignmentChecksEnabled(pVCpu))
+                return iemRaiseAlignmentCheckException(pVCpu);
+            else
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+    }
+
+#ifdef IEM_WITH_DATA_TLB
+    Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
+
+    /*
+     * Get the TLB entry for this page.
+     */
+    uint64_t const     uTag  = IEMTLB_CALC_TAG(    &pVCpu->iem.s.DataTlb, GCPtrMem);
+    PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
+    if (pTlbe->uTag == uTag)
+    {
+# ifdef VBOX_WITH_STATISTICS
+        pVCpu->iem.s.DataTlb.cTlbHits++;
+# endif
+    }
+    else
+    {
+        pVCpu->iem.s.DataTlb.cTlbMisses++;
+        PGMPTWALK Walk;
+        int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
+        if (RT_FAILURE(rc))
+        {
+            Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            if (Walk.fFailed & PGM_WALKFAIL_EPT)
+                IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
+# endif
+            return iemRaisePageFault(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, rc);
+        }
+
+        Assert(Walk.fSucceeded);
+        pTlbe->uTag             = uTag;
+        pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A); /* skipping NX */
+        pTlbe->GCPhys           = Walk.GCPhys;
+        pTlbe->pbMappingR3      = NULL;
+    }
+
+    /*
+     * Check TLB page table level access flags.
+     */
+    /* If the page is either supervisor only or non-writable, we need to do
+       more careful access checks. */
+    if (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PT_NO_USER | IEMTLBE_F_PT_NO_WRITE))
+    {
+        /* Write to read only memory? */
+        if (   (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_WRITE)
+            && (fAccess & IEM_ACCESS_TYPE_WRITE)
+            && (   (    pVCpu->iem.s.uCpl == 3
+                    && !(fAccess & IEM_ACCESS_WHAT_SYS))
+                || (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)))
+        {
+            Log(("iemMemMap: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            if (Walk.fFailed & PGM_WALKFAIL_EPT)
+                IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+# endif
+            return iemRaisePageFault(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
+        }
+
+        /* Kernel memory accessed by userland? */
+        if (   (pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PT_NO_USER)
+            && pVCpu->iem.s.uCpl == 3
+            && !(fAccess & IEM_ACCESS_WHAT_SYS))
+        {
+            Log(("iemMemMap: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            if (Walk.fFailed & PGM_WALKFAIL_EPT)
+                IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+# endif
+            return iemRaisePageFault(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, VERR_ACCESS_DENIED);
+        }
+    }
+
+    /*
+     * Set the dirty / access flags.
+     * ASSUMES this is set when the address is translated rather than on commit...
+     */
+    /** @todo testcase: check when A and D bits are actually set by the CPU.  */
+    uint64_t const fTlbAccessedDirty = (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PT_NO_DIRTY : 0) | IEMTLBE_F_PT_NO_ACCESSED;
+    if (pTlbe->fFlagsAndPhysRev & fTlbAccessedDirty)
+    {
+        uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A;
+        int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
+        AssertRC(rc2);
+        /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
+        Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
+        pTlbe->fFlagsAndPhysRev &= ~fTlbAccessedDirty;
+    }
+
+    /*
+     * Look up the physical page info if necessary.
+     */
+    uint8_t *pbMem = NULL;
+    if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
+# ifdef IN_RING3
+        pbMem = pTlbe->pbMappingR3;
+# else
+        pbMem = NULL;
+# endif
+    else
+    {
+        AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE     == IEMTLBE_F_PG_NO_WRITE);
+        AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ      == IEMTLBE_F_PG_NO_READ);
+        AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
+        AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED   == IEMTLBE_F_PG_UNASSIGNED);
+        if (RT_LIKELY(pVCpu->iem.s.CodeTlb.uTlbPhysRev > IEMTLB_PHYS_REV_INCR))
+        { /* likely */ }
+        else
+            IEMTlbInvalidateAllPhysicalSlow(pVCpu);
+        pTlbe->pbMappingR3       = NULL;
+        pTlbe->fFlagsAndPhysRev &= ~(  IEMTLBE_F_PHYS_REV
+                                     | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_UNASSIGNED);
+        int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
+                                            &pbMem, &pTlbe->fFlagsAndPhysRev);
+        AssertRCReturn(rc, rc);
+# ifdef IN_RING3
+        pTlbe->pbMappingR3 = pbMem;
+# endif
+    }
+
+    /*
+     * Check the physical page level access and mapping.
+     */
+    if (   !(pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ))
+        || !(pTlbe->fFlagsAndPhysRev & (  (fAccess & IEM_ACCESS_TYPE_WRITE ? IEMTLBE_F_PG_NO_WRITE : 0)
+                                        | (fAccess & IEM_ACCESS_TYPE_READ  ? IEMTLBE_F_PG_NO_READ  : 0))) )
+    { /* probably likely */ }
+    else
+        return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem,
+                                         pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
+                                           pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
+                                         : pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ    ? VERR_PGM_PHYS_TLB_CATCH_ALL
+                                                                                             : VERR_PGM_PHYS_TLB_CATCH_WRITE);
+    Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
+
+    if (pbMem)
+    {
+        Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
+        pbMem    = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
+        fAccess |= IEM_ACCESS_NOT_LOCKED;
+    }
+    else
+    {
+        Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
+        RTGCPHYS const GCPhysFirst = pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
+        rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+        if (rcStrict != VINF_SUCCESS)
+            return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
+    }
+
+    void * const pvMem = pbMem;
+
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+        Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
+    if (fAccess & IEM_ACCESS_TYPE_READ)
+        Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
+
+#else  /* !IEM_WITH_DATA_TLB */
+
+    RTGCPHYS GCPhysFirst;
+    rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, &GCPhysFirst);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+        Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
+    if (fAccess & IEM_ACCESS_TYPE_READ)
+        Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
+
+    void *pvMem;
+    rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+    if (rcStrict != VINF_SUCCESS)
+        return iemMemBounceBufferMapPhys(pVCpu, iMemMap, ppvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
+
+#endif /* !IEM_WITH_DATA_TLB */
+
+    /*
+     * Fill in the mapping table entry.
+     */
+    pVCpu->iem.s.aMemMappings[iMemMap].pv      = pvMem;
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
+    pVCpu->iem.s.iNextMapping     = iMemMap + 1;
+    pVCpu->iem.s.cActiveMappings += 1;
+
+    iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
+    *ppvMem = pvMem;
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Commits the guest memory if bounce buffered and unmaps it.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem               The mapping.
+ * @param   fAccess             The kind of access.
+ */
+VBOXSTRICTRC iemMemCommitAndUnmap(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
+{
+    int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
+    AssertReturn(iMemMap >= 0, iMemMap);
+
+    /* If it's bounce buffered, we may need to write back the buffer. */
+    if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
+    {
+        if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
+            return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
+    }
+    /* Otherwise unlock it. */
+    else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
+        PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+
+    /* Free the entry. */
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+    Assert(pVCpu->iem.s.cActiveMappings != 0);
+    pVCpu->iem.s.cActiveMappings--;
+    return VINF_SUCCESS;
+}
+
+#ifdef IEM_WITH_SETJMP
+
+/**
+ * Maps the specified guest memory for the given kind of access, longjmp on
+ * error.
+ *
+ * This may be using bounce buffering of the memory if it's crossing a page
+ * boundary or if there is an access handler installed for any of it.  Because
+ * of lock prefix guarantees, we're in for some extra clutter when this
+ * happens.
+ *
+ * This may raise a \#GP, \#SS, \#PF or \#AC.
+ *
+ * @returns Pointer to the mapped memory.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ * @param   cbMem       The number of bytes to map.  This is usually 1,
+ *                      2, 4, 6, 8, 12, 16, 32 or 512.  When used by
+ *                      string operations it can be up to a page.
+ * @param   iSegReg     The index of the segment register to use for
+ *                      this access.  The base and limits are checked.
+ *                      Use UINT8_MAX to indicate that no segmentation
+ *                      is required (for IDT, GDT and LDT accesses).
+ * @param   GCPtrMem    The address of the guest memory.
+ * @param   fAccess     How the memory is being accessed.  The
+ *                      IEM_ACCESS_TYPE_XXX bit is used to figure out
+ *                      how to map the memory, while the
+ *                      IEM_ACCESS_WHAT_XXX bit is used when raising
+ *                      exceptions.
+ * @param   uAlignCtl   Alignment control:
+ *                          - Bits 15:0 is the alignment mask.
+ *                          - Bits 31:16 for flags like IEM_MEMMAP_F_ALIGN_GP,
+ *                            IEM_MEMMAP_F_ALIGN_SSE, and
+ *                            IEM_MEMMAP_F_ALIGN_GP_OR_AC.
+ *                      Pass zero to skip alignment.
+ */
+void *iemMemMapJmp(PVMCPUCC pVCpu, size_t cbMem, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t fAccess,
+                   uint32_t uAlignCtl) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /*
+     * Check the input, check segment access and adjust address
+     * with segment base.
+     */
+    Assert(cbMem <= 64 || cbMem == 512 || cbMem == 108 || cbMem == 104 || cbMem == 94); /* 512 is the max! */
+    Assert(~(fAccess & ~(IEM_ACCESS_TYPE_MASK | IEM_ACCESS_WHAT_MASK)));
+    Assert(pVCpu->iem.s.cActiveMappings < RT_ELEMENTS(pVCpu->iem.s.aMemMappings));
+
+    VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, fAccess, iSegReg, cbMem, &GCPtrMem);
+    if (rcStrict == VINF_SUCCESS) { /*likely*/ }
+    else IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+
+    /*
+     * Alignment check.
+     */
+    if ( (GCPtrMem & (uAlignCtl & UINT16_MAX)) == 0 )
+    { /* likelyish */ }
+    else
+    {
+        /* Misaligned access. */
+        if ((fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
+        {
+            if (   !(uAlignCtl & IEM_MEMMAP_F_ALIGN_GP)
+                || (   (uAlignCtl & IEM_MEMMAP_F_ALIGN_SSE)
+                    && (pVCpu->cpum.GstCtx.XState.x87.MXCSR & X86_MXCSR_MM)) )
+            {
+                AssertCompile(X86_CR0_AM == X86_EFL_AC);
+
+                if (iemMemAreAlignmentChecksEnabled(pVCpu))
+                    iemRaiseAlignmentCheckExceptionJmp(pVCpu);
+            }
+            else if (   (uAlignCtl & IEM_MEMMAP_F_ALIGN_GP_OR_AC)
+                     && (GCPtrMem & 3) /* The value 4 matches 10980xe's FXSAVE and helps make bs3-cpu-basic2 work. */
+                    /** @todo may only apply to 2, 4 or 8 byte misalignments depending on the CPU
+                     * implementation. See FXSAVE/FRSTOR/XSAVE/XRSTOR/++.  Using 4 for now as
+                     * that's what FXSAVE does on a 10980xe. */
+                     && iemMemAreAlignmentChecksEnabled(pVCpu))
+                iemRaiseAlignmentCheckExceptionJmp(pVCpu);
+            else
+                iemRaiseGeneralProtectionFault0Jmp(pVCpu);
+        }
+    }
+
+    /*
+     * Figure out which mapping entry to use.
+     */
+    unsigned iMemMap = pVCpu->iem.s.iNextMapping;
+    if (   iMemMap >= RT_ELEMENTS(pVCpu->iem.s.aMemMappings)
+        || pVCpu->iem.s.aMemMappings[iMemMap].fAccess != IEM_ACCESS_INVALID)
+    {
+        iMemMap = iemMemMapFindFree(pVCpu);
+        AssertLogRelMsgStmt(iMemMap < RT_ELEMENTS(pVCpu->iem.s.aMemMappings),
+                            ("active=%d fAccess[0] = {%#x, %#x, %#x}\n", pVCpu->iem.s.cActiveMappings,
+                             pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess,
+                             pVCpu->iem.s.aMemMappings[2].fAccess),
+                            IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_9));
+    }
+
+    /*
+     * Crossing a page boundary?
+     */
+    if ((GCPtrMem & GUEST_PAGE_OFFSET_MASK) + cbMem <= GUEST_PAGE_SIZE)
+    { /* No (likely). */ }
+    else
+    {
+        void *pvMem;
+        rcStrict = iemMemBounceBufferMapCrossPage(pVCpu, iMemMap, &pvMem, cbMem, GCPtrMem, fAccess);
+        if (rcStrict == VINF_SUCCESS)
+            return pvMem;
+        IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+    }
+
+#ifdef IEM_WITH_DATA_TLB
+    Assert(!(fAccess & IEM_ACCESS_TYPE_EXEC));
+
+    /*
+     * Get the TLB entry for this page.
+     */
+    uint64_t const     uTag  = IEMTLB_CALC_TAG(    &pVCpu->iem.s.DataTlb, GCPtrMem);
+    PIEMTLBENTRY const pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
+    if (pTlbe->uTag == uTag)
+        STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
+    else
+    {
+        pVCpu->iem.s.DataTlb.cTlbMisses++;
+        PGMPTWALK Walk;
+        int rc = PGMGstGetPage(pVCpu, GCPtrMem, &Walk);
+        if (RT_FAILURE(rc))
+        {
+            Log(("iemMemMap: GCPtrMem=%RGv - failed to fetch page -> #PF\n", GCPtrMem));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            if (Walk.fFailed & PGM_WALKFAIL_EPT)
+                IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR, 0 /* cbInstr */);
+# endif
+            iemRaisePageFaultJmp(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, rc);
+        }
+
+        Assert(Walk.fSucceeded);
+        pTlbe->uTag             = uTag;
+        pTlbe->fFlagsAndPhysRev = ~Walk.fEffective & (X86_PTE_US | X86_PTE_RW | X86_PTE_D | X86_PTE_A); /* skipping NX */
+        pTlbe->GCPhys           = Walk.GCPhys;
+        pTlbe->pbMappingR3      = NULL;
+    }
+
+    /*
+     * Check the flags and physical revision.
+     */
+    /** @todo make the caller pass these in with fAccess. */
+    uint64_t const fNoUser          = (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS && pVCpu->iem.s.uCpl == 3
+                                    ? IEMTLBE_F_PT_NO_USER : 0;
+    uint64_t const fNoWriteNoDirty  = fAccess & IEM_ACCESS_TYPE_WRITE
+                                    ? IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PT_NO_DIRTY
+                                      | (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_WP)
+                                         || (pVCpu->iem.s.uCpl == 3 && (fAccess & IEM_ACCESS_WHAT_MASK) != IEM_ACCESS_WHAT_SYS)
+                                         ? IEMTLBE_F_PT_NO_WRITE : 0)
+                                    : 0;
+    uint64_t const fNoRead          = fAccess & IEM_ACCESS_TYPE_READ ? IEMTLBE_F_PG_NO_READ : 0;
+    uint8_t       *pbMem            = NULL;
+    if (   (pTlbe->fFlagsAndPhysRev & (IEMTLBE_F_PHYS_REV | IEMTLBE_F_PT_NO_ACCESSED | fNoRead | fNoWriteNoDirty | fNoUser))
+        == pVCpu->iem.s.DataTlb.uTlbPhysRev)
+# ifdef IN_RING3
+        pbMem = pTlbe->pbMappingR3;
+# else
+        pbMem = NULL;
+# endif
+    else
+    {
+        /*
+         * Okay, something isn't quite right or needs refreshing.
+         */
+        /* Write to read only memory? */
+        if (pTlbe->fFlagsAndPhysRev & fNoWriteNoDirty & IEMTLBE_F_PT_NO_WRITE)
+        {
+            Log(("iemMemMapJmp: GCPtrMem=%RGv - read-only page -> #PF\n", GCPtrMem));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            if (Walk.fFailed & PGM_WALKFAIL_EPT)
+                IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+# endif
+            iemRaisePageFaultJmp(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess & ~IEM_ACCESS_TYPE_READ, VERR_ACCESS_DENIED);
+        }
+
+        /* Kernel memory accessed by userland? */
+        if (pTlbe->fFlagsAndPhysRev & fNoUser & IEMTLBE_F_PT_NO_USER)
+        {
+            Log(("iemMemMapJmp: GCPtrMem=%RGv - user access to kernel page -> #PF\n", GCPtrMem));
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            if (Walk.fFailed & PGM_WALKFAIL_EPT)
+                IEM_VMX_VMEXIT_EPT_RET(pVCpu, &Walk, fAccess, IEM_SLAT_FAIL_LINEAR_TO_PAGE_TABLE, 0 /* cbInstr */);
+# endif
+            iemRaisePageFaultJmp(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, VERR_ACCESS_DENIED);
+        }
+
+        /* Set the dirty / access flags.
+           ASSUMES this is set when the address is translated rather than on commit... */
+        /** @todo testcase: check when A and D bits are actually set by the CPU.  */
+        if (pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) | IEMTLBE_F_PT_NO_ACCESSED))
+        {
+            uint32_t const fAccessedDirty = fAccess & IEM_ACCESS_TYPE_WRITE ? X86_PTE_D | X86_PTE_A : X86_PTE_A;
+            int rc2 = PGMGstModifyPage(pVCpu, GCPtrMem, 1, fAccessedDirty, ~(uint64_t)fAccessedDirty);
+            AssertRC(rc2);
+            /** @todo Nested VMX: Accessed/dirty bit currently not supported, asserted below. */
+            Assert(!(CPUMGetGuestIa32VmxEptVpidCap(pVCpu) & VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY_MASK));
+            pTlbe->fFlagsAndPhysRev &= ~((fNoWriteNoDirty & IEMTLBE_F_PT_NO_DIRTY) | IEMTLBE_F_PT_NO_ACCESSED);
+        }
+
+        /*
+         * Check if the physical page info needs updating.
+         */
+        if ((pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PHYS_REV) == pVCpu->iem.s.DataTlb.uTlbPhysRev)
+# ifdef IN_RING3
+            pbMem = pTlbe->pbMappingR3;
+# else
+            pbMem = NULL;
+# endif
+        else
+        {
+            AssertCompile(PGMIEMGCPHYS2PTR_F_NO_WRITE     == IEMTLBE_F_PG_NO_WRITE);
+            AssertCompile(PGMIEMGCPHYS2PTR_F_NO_READ      == IEMTLBE_F_PG_NO_READ);
+            AssertCompile(PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 == IEMTLBE_F_NO_MAPPINGR3);
+            AssertCompile(PGMIEMGCPHYS2PTR_F_UNASSIGNED   == IEMTLBE_F_PG_UNASSIGNED);
+            pTlbe->pbMappingR3       = NULL;
+            pTlbe->fFlagsAndPhysRev &= ~(  IEMTLBE_F_PHYS_REV
+                                         | IEMTLBE_F_NO_MAPPINGR3 | IEMTLBE_F_PG_NO_READ | IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_UNASSIGNED);
+            int rc = PGMPhysIemGCPhys2PtrNoLock(pVCpu->CTX_SUFF(pVM), pVCpu, pTlbe->GCPhys, &pVCpu->iem.s.DataTlb.uTlbPhysRev,
+                                                &pbMem, &pTlbe->fFlagsAndPhysRev);
+            AssertRCStmt(rc, IEM_DO_LONGJMP(pVCpu, rc));
+# ifdef IN_RING3
+            pTlbe->pbMappingR3 = pbMem;
+# endif
+        }
+
+        /*
+         * Check the physical page level access and mapping.
+         */
+        if (!(pTlbe->fFlagsAndPhysRev & ((fNoWriteNoDirty | fNoRead) & (IEMTLBE_F_PG_NO_WRITE | IEMTLBE_F_PG_NO_READ))))
+        { /* probably likely */ }
+        else
+        {
+            rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, (void **)&pbMem, cbMem,
+                                                 pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), fAccess,
+                                                   pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_UNASSIGNED ? VERR_PGM_PHYS_TLB_UNASSIGNED
+                                                 : pTlbe->fFlagsAndPhysRev & IEMTLBE_F_PG_NO_READ    ? VERR_PGM_PHYS_TLB_CATCH_ALL
+                                                                                                     : VERR_PGM_PHYS_TLB_CATCH_WRITE);
+            if (rcStrict == VINF_SUCCESS)
+                return pbMem;
+            IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+        }
+    }
+    Assert(!(pTlbe->fFlagsAndPhysRev & IEMTLBE_F_NO_MAPPINGR3)); /* ASSUMPTIONS about PGMPhysIemGCPhys2PtrNoLock behaviour. */
+
+    if (pbMem)
+    {
+        Assert(!((uintptr_t)pbMem & GUEST_PAGE_OFFSET_MASK));
+        pbMem    = pbMem + (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
+        fAccess |= IEM_ACCESS_NOT_LOCKED;
+    }
+    else
+    {
+        Assert(!(fAccess & IEM_ACCESS_NOT_LOCKED));
+        RTGCPHYS const GCPhysFirst = pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK);
+        rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, (void **)&pbMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+        if (rcStrict == VINF_SUCCESS)
+            return pbMem;
+        IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+    }
+
+    void * const pvMem = pbMem;
+
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+        Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
+    if (fAccess & IEM_ACCESS_TYPE_READ)
+        Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, pTlbe->GCPhys | (GCPtrMem & GUEST_PAGE_OFFSET_MASK), cbMem));
+
+#else  /* !IEM_WITH_DATA_TLB */
+
+
+    RTGCPHYS GCPhysFirst;
+    rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, (uint32_t)cbMem, fAccess, &GCPhysFirst);
+    if (rcStrict == VINF_SUCCESS) { /*likely*/ }
+    else IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+        Log8(("IEM WR %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
+    if (fAccess & IEM_ACCESS_TYPE_READ)
+        Log9(("IEM RD %RGv (%RGp) LB %#zx\n", GCPtrMem, GCPhysFirst, cbMem));
+
+    void *pvMem;
+    rcStrict = iemMemPageMap(pVCpu, GCPhysFirst, fAccess, &pvMem, &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+    if (rcStrict == VINF_SUCCESS)
+    { /* likely */ }
+    else
+    {
+        rcStrict = iemMemBounceBufferMapPhys(pVCpu, iMemMap, &pvMem, cbMem, GCPhysFirst, fAccess, rcStrict);
+        if (rcStrict == VINF_SUCCESS)
+            return pvMem;
+        IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+    }
+
+#endif /* !IEM_WITH_DATA_TLB */
+
+    /*
+     * Fill in the mapping table entry.
+     */
+    pVCpu->iem.s.aMemMappings[iMemMap].pv      = pvMem;
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = fAccess;
+    pVCpu->iem.s.iNextMapping = iMemMap + 1;
+    pVCpu->iem.s.cActiveMappings++;
+
+    iemMemUpdateWrittenCounter(pVCpu, fAccess, cbMem);
+    return pvMem;
+}
+
+
+/**
+ * Commits the guest memory if bounce buffered and unmaps it, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem               The mapping.
+ * @param   fAccess             The kind of access.
+ */
+void iemMemCommitAndUnmapJmp(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
+    AssertStmt(iMemMap >= 0, IEM_DO_LONGJMP(pVCpu, iMemMap));
+
+    /* If it's bounce buffered, we may need to write back the buffer. */
+    if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
+    {
+        if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
+        {
+            VBOXSTRICTRC rcStrict = iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, false /*fPostponeFail*/);
+            if (rcStrict == VINF_SUCCESS)
+                return;
+            IEM_DO_LONGJMP(pVCpu, VBOXSTRICTRC_VAL(rcStrict));
+        }
+    }
+    /* Otherwise unlock it. */
+    else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
+        PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+
+    /* Free the entry. */
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+    Assert(pVCpu->iem.s.cActiveMappings != 0);
+    pVCpu->iem.s.cActiveMappings--;
+}
+
+#endif /* IEM_WITH_SETJMP */
+
+#ifndef IN_RING3
+/**
+ * Commits the guest memory if bounce buffered and unmaps it, if any bounce
+ * buffer part shows trouble it will be postponed to ring-3 (sets FF and stuff).
+ *
+ * Allows the instruction to be completed and retired, while the IEM user will
+ * return to ring-3 immediately afterwards and do the postponed writes there.
+ *
+ * @returns VBox status code (no strict statuses).  Caller must check
+ *          VMCPU_FF_IEM before repeating string instructions and similar stuff.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem               The mapping.
+ * @param   fAccess             The kind of access.
+ */
+VBOXSTRICTRC iemMemCommitAndUnmapPostponeTroubleToR3(PVMCPUCC pVCpu, void *pvMem, uint32_t fAccess) RT_NOEXCEPT
+{
+    int iMemMap = iemMapLookup(pVCpu, pvMem, fAccess);
+    AssertReturn(iMemMap >= 0, iMemMap);
+
+    /* If it's bounce buffered, we may need to write back the buffer. */
+    if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED)
+    {
+        if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE)
+            return iemMemBounceBufferCommitAndUnmap(pVCpu, iMemMap, true /*fPostponeFail*/);
+    }
+    /* Otherwise unlock it. */
+    else if (!(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_NOT_LOCKED))
+        PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+
+    /* Free the entry. */
+    pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+    Assert(pVCpu->iem.s.cActiveMappings != 0);
+    pVCpu->iem.s.cActiveMappings--;
+    return VINF_SUCCESS;
+}
+#endif
+
+
+/**
+ * Rollbacks mappings, releasing page locks and such.
+ *
+ * The caller shall only call this after checking cActiveMappings.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ */
+void iemMemRollback(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    Assert(pVCpu->iem.s.cActiveMappings > 0);
+
+    uint32_t iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
+    while (iMemMap-- > 0)
+    {
+        uint32_t const fAccess = pVCpu->iem.s.aMemMappings[iMemMap].fAccess;
+        if (fAccess != IEM_ACCESS_INVALID)
+        {
+            AssertMsg(!(fAccess & ~IEM_ACCESS_VALID_MASK) && fAccess != 0, ("%#x\n", fAccess));
+            pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+            if (!(fAccess & (IEM_ACCESS_BOUNCE_BUFFERED | IEM_ACCESS_NOT_LOCKED)))
+                PGMPhysReleasePageMappingLock(pVCpu->CTX_SUFF(pVM), &pVCpu->iem.s.aMemMappingLocks[iMemMap].Lock);
+            AssertMsg(pVCpu->iem.s.cActiveMappings > 0,
+                      ("iMemMap=%u fAccess=%#x pv=%p GCPhysFirst=%RGp GCPhysSecond=%RGp\n",
+                       iMemMap, fAccess, pVCpu->iem.s.aMemMappings[iMemMap].pv,
+                       pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond));
+            pVCpu->iem.s.cActiveMappings--;
+        }
+    }
+}
+
+
+/**
+ * Fetches a data byte.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu8Dst              Where to return the byte.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataU8(PVMCPUCC pVCpu, uint8_t *pu8Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint8_t const *pu8Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Src, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu8Dst = *pu8Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data byte, longjmp on error.
+ *
+ * @returns The byte.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+uint8_t iemMemFetchDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    uint8_t const *pu8Src = (uint8_t const *)iemMemMapJmp(pVCpu, sizeof(*pu8Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 0);
+    uint8_t const  bRet   = *pu8Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu8Src, IEM_ACCESS_DATA_R);
+    return bRet;
+}
+#endif /* IEM_WITH_SETJMP */
+
+
+/**
+ * Fetches a data word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16Dst             Where to return the word.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint16_t const *pu16Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, sizeof(*pu16Src) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = *pu16Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data word, longjmp on error.
+ *
+ * @returns The word
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+uint16_t iemMemFetchDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    uint16_t const *pu16Src = (uint16_t const *)iemMemMapJmp(pVCpu, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
+                                                             sizeof(*pu16Src) - 1);
+    uint16_t const u16Ret = *pu16Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu16Src, IEM_ACCESS_DATA_R);
+    return u16Ret;
+}
+#endif
+
+
+/**
+ * Fetches a data dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32Dst             Where to return the dword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint32_t const *pu32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Dst = *pu32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+/**
+ * Fetches a data dword and zero extends it to a qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Dst             Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataU32_ZX_U64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint32_t const *pu32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = *pu32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+
+/**
+ * Fetches a data dword, longjmp on error, fallback/safe version.
+ *
+ * @returns The dword
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+uint32_t iemMemFetchDataU32SafeJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
+                                                             sizeof(*pu32Src) - 1);
+    uint32_t const  u32Ret  = *pu32Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
+    return u32Ret;
+}
+
+
+/**
+ * Fetches a data dword, longjmp on error.
+ *
+ * @returns The dword
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+uint32_t iemMemFetchDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+# if defined(IEM_WITH_DATA_TLB) && defined(IN_RING3)
+    /*
+     * Convert from segmented to flat address and check that it doesn't cross a page boundrary.
+     */
+    RTGCPTR GCPtrEff = iemMemApplySegmentToReadJmp(pVCpu, iSegReg, sizeof(uint32_t), GCPtrMem);
+    if (RT_LIKELY((GCPtrEff & GUEST_PAGE_OFFSET_MASK) <= GUEST_PAGE_SIZE - sizeof(uint32_t)))
+    {
+        /*
+         * TLB lookup.
+         */
+        uint64_t const uTag  = IEMTLB_CALC_TAG(    &pVCpu->iem.s.DataTlb, GCPtrEff);
+        PIEMTLBENTRY   pTlbe = IEMTLB_TAG_TO_ENTRY(&pVCpu->iem.s.DataTlb, uTag);
+        if (pTlbe->uTag == uTag)
+        {
+            /*
+             * Check TLB page table level access flags.
+             */
+            uint64_t const fNoUser = pVCpu->iem.s.uCpl == 3 ? IEMTLBE_F_PT_NO_USER : 0;
+            if (   (pTlbe->fFlagsAndPhysRev & (  IEMTLBE_F_PHYS_REV       | IEMTLBE_F_PG_UNASSIGNED | IEMTLBE_F_PG_NO_READ
+                                               | IEMTLBE_F_PT_NO_ACCESSED | IEMTLBE_F_NO_MAPPINGR3  | fNoUser))
+                == pVCpu->iem.s.DataTlb.uTlbPhysRev)
+            {
+                STAM_STATS({pVCpu->iem.s.DataTlb.cTlbHits++;});
+
+                /*
+                 * Alignment check:
+                 */
+                /** @todo check priority \#AC vs \#PF */
+                if (   !(GCPtrEff & (sizeof(uint32_t) - 1))
+                    || !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
+                    || !pVCpu->cpum.GstCtx.eflags.Bits.u1AC
+                    || pVCpu->iem.s.uCpl != 3)
+                {
+                    /*
+                     * Fetch and return the dword
+                     */
+                    Assert(pTlbe->pbMappingR3); /* (Only ever cleared by the owning EMT.) */
+                    Assert(!((uintptr_t)pTlbe->pbMappingR3 & GUEST_PAGE_OFFSET_MASK));
+                    return *(uint32_t const *)&pTlbe->pbMappingR3[GCPtrEff & GUEST_PAGE_OFFSET_MASK];
+                }
+                Log10(("iemMemFetchDataU32Jmp: Raising #AC for %RGv\n", GCPtrEff));
+                iemRaiseAlignmentCheckExceptionJmp(pVCpu);
+            }
+        }
+    }
+
+    /* Fall back on the slow careful approach in case of TLB miss, MMIO, exception
+       outdated page pointer, or other troubles. */
+    Log10(("iemMemFetchDataU32Jmp: %u:%RGv fallback\n", iSegReg, GCPtrMem));
+    return iemMemFetchDataU32SafeJmp(pVCpu, iSegReg, GCPtrMem);
+
+# else
+    uint32_t const *pu32Src = (uint32_t const *)iemMemMapJmp(pVCpu, sizeof(*pu32Src), iSegReg, GCPtrMem,
+                                                             IEM_ACCESS_DATA_R, sizeof(*pu32Src) - 1);
+    uint32_t const  u32Ret  = *pu32Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu32Src, IEM_ACCESS_DATA_R);
+    return u32Ret;
+# endif
+}
+#endif
+
+
+#ifdef SOME_UNUSED_FUNCTION
+/**
+ * Fetches a data dword and sign extends it to a qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Dst             Where to return the sign extended value.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataS32SxU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    int32_t const *pi32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pi32Src, sizeof(*pi32Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, sizeof(*pi32Src) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = *pi32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pi32Src, IEM_ACCESS_DATA_R);
+    }
+#ifdef __GNUC__ /* warning: GCC may be a royal pain */
+    else
+        *pu64Dst = 0;
+#endif
+    return rc;
+}
+#endif
+
+
+/**
+ * Fetches a data qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Dst             Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint64_t const *pu64Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = *pu64Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data qword, longjmp on error.
+ *
+ * @returns The qword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+uint64_t iemMemFetchDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem,
+                                                             IEM_ACCESS_DATA_R, sizeof(*pu64Src) - 1);
+    uint64_t const u64Ret = *pu64Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
+    return u64Ret;
+}
+#endif
+
+
+/**
+ * Fetches a data qword, aligned at a 16 byte boundrary (for SSE).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Dst             Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataU64AlignedU128(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint64_t const *pu64Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, 15 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = *pu64Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data qword, longjmp on error.
+ *
+ * @returns The qword.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+uint64_t iemMemFetchDataU64AlignedU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    uint64_t const *pu64Src = (uint64_t const *)iemMemMapJmp(pVCpu, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
+                                                             15 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
+    uint64_t const u64Ret = *pu64Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu64Src, IEM_ACCESS_DATA_R);
+    return u64Ret;
+}
+#endif
+
+
+/**
+ * Fetches a data tword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pr80Dst             Where to return the tword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataR80(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    PCRTFLOAT80U pr80Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pr80Src, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7);
+    if (rc == VINF_SUCCESS)
+    {
+        *pr80Dst = *pr80Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data tword, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pr80Dst             Where to return the tword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+void iemMemFetchDataR80Jmp(PVMCPUCC pVCpu, PRTFLOAT80U pr80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    PCRTFLOAT80U pr80Src = (PCRTFLOAT80U)iemMemMapJmp(pVCpu, sizeof(*pr80Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R, 7);
+    *pr80Dst = *pr80Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pr80Src, IEM_ACCESS_DATA_R);
+}
+#endif
+
+
+/**
+ * Fetches a data decimal tword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pd80Dst             Where to return the tword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataD80(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    PCRTPBCD80U pd80Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pd80Src, sizeof(*pd80Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, 7 /** @todo FBLD alignment check */);
+    if (rc == VINF_SUCCESS)
+    {
+        *pd80Dst = *pd80Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data decimal tword, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pd80Dst             Where to return the tword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+void iemMemFetchDataD80Jmp(PVMCPUCC pVCpu, PRTPBCD80U pd80Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    PCRTPBCD80U pd80Src = (PCRTPBCD80U)iemMemMapJmp(pVCpu, sizeof(*pd80Src), iSegReg, GCPtrMem,
+                                                    IEM_ACCESS_DATA_R, 7 /** @todo FBSTP alignment check */);
+    *pd80Dst = *pd80Src;
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pd80Src, IEM_ACCESS_DATA_R);
+}
+#endif
+
+
+/**
+ * Fetches a data dqword (double qword), generally SSE related.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu128Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataU128(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    PCRTUINT128U pu128Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
+    if (rc == VINF_SUCCESS)
+    {
+        pu128Dst->au64[0] = pu128Src->au64[0];
+        pu128Dst->au64[1] = pu128Src->au64[1];
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data dqword (double qword), generally SSE related.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu128Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+void iemMemFetchDataU128Jmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem,
+                                                       IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
+    pu128Dst->au64[0] = pu128Src->au64[0];
+    pu128Dst->au64[1] = pu128Src->au64[1];
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
+}
+#endif
+
+
+/**
+ * Fetches a data dqword (double qword) at an aligned address, generally SSE
+ * related.
+ *
+ * Raises \#GP(0) if not aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu128Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataU128AlignedSse(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    PCRTUINT128U pu128Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Src, sizeof(*pu128Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, (sizeof(*pu128Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
+    if (rc == VINF_SUCCESS)
+    {
+        pu128Dst->au64[0] = pu128Src->au64[0];
+        pu128Dst->au64[1] = pu128Src->au64[1];
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data dqword (double qword) at an aligned address, generally SSE
+ * related, longjmp on error.
+ *
+ * Raises \#GP(0) if not aligned.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu128Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+void iemMemFetchDataU128AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT128U pu128Dst, uint8_t iSegReg,
+                                      RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    PCRTUINT128U pu128Src = (PCRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
+                                                       (sizeof(*pu128Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
+    pu128Dst->au64[0] = pu128Src->au64[0];
+    pu128Dst->au64[1] = pu128Src->au64[1];
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu128Src, IEM_ACCESS_DATA_R);
+}
+#endif
+
+
+/**
+ * Fetches a data oword (octo word), generally AVX related.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu256Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataU256(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    PCRTUINT256U pu256Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
+    if (rc == VINF_SUCCESS)
+    {
+        pu256Dst->au64[0] = pu256Src->au64[0];
+        pu256Dst->au64[1] = pu256Src->au64[1];
+        pu256Dst->au64[2] = pu256Src->au64[2];
+        pu256Dst->au64[3] = pu256Src->au64[3];
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data oword (octo word), generally AVX related.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu256Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+void iemMemFetchDataU256Jmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem,
+                                                       IEM_ACCESS_DATA_R, 0 /* NO_AC variant */);
+    pu256Dst->au64[0] = pu256Src->au64[0];
+    pu256Dst->au64[1] = pu256Src->au64[1];
+    pu256Dst->au64[2] = pu256Src->au64[2];
+    pu256Dst->au64[3] = pu256Src->au64[3];
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
+}
+#endif
+
+
+/**
+ * Fetches a data oword (octo word) at an aligned address, generally AVX
+ * related.
+ *
+ * Raises \#GP(0) if not aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu256Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchDataU256AlignedSse(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    PCRTUINT256U pu256Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Src, sizeof(*pu256Src), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_R, (sizeof(*pu256Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
+    if (rc == VINF_SUCCESS)
+    {
+        pu256Dst->au64[0] = pu256Src->au64[0];
+        pu256Dst->au64[1] = pu256Src->au64[1];
+        pu256Dst->au64[2] = pu256Src->au64[2];
+        pu256Dst->au64[3] = pu256Src->au64[3];
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Fetches a data oword (octo word) at an aligned address, generally AVX
+ * related, longjmp on error.
+ *
+ * Raises \#GP(0) if not aligned.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu256Dst            Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+void iemMemFetchDataU256AlignedSseJmp(PVMCPUCC pVCpu, PRTUINT256U pu256Dst, uint8_t iSegReg,
+                                      RTGCPTR GCPtrMem) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    PCRTUINT256U pu256Src = (PCRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Src), iSegReg, GCPtrMem, IEM_ACCESS_DATA_R,
+                                                       (sizeof(*pu256Src) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
+    pu256Dst->au64[0] = pu256Src->au64[0];
+    pu256Dst->au64[1] = pu256Src->au64[1];
+    pu256Dst->au64[2] = pu256Src->au64[2];
+    pu256Dst->au64[3] = pu256Src->au64[3];
+    iemMemCommitAndUnmapJmp(pVCpu, (void *)pu256Src, IEM_ACCESS_DATA_R);
+}
+#endif
+
+
+
+/**
+ * Fetches a descriptor register (lgdt, lidt).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pcbLimit            Where to return the limit.
+ * @param   pGCPtrBase          Where to return the base.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   enmOpSize           The effective operand size.
+ */
+VBOXSTRICTRC iemMemFetchDataXdtr(PVMCPUCC pVCpu, uint16_t *pcbLimit, PRTGCPTR pGCPtrBase, uint8_t iSegReg,
+                                 RTGCPTR GCPtrMem, IEMMODE enmOpSize) RT_NOEXCEPT
+{
+    /*
+     * Just like SIDT and SGDT, the LIDT and LGDT instructions are a
+     * little special:
+     *      - The two reads are done separately.
+     *      - Operand size override works in 16-bit and 32-bit code, but 64-bit.
+     *      - We suspect the 386 to actually commit the limit before the base in
+     *        some cases (search for 386 in  bs3CpuBasic2_lidt_lgdt_One).  We
+     *        don't try emulate this eccentric behavior, because it's not well
+     *        enough understood and rather hard to trigger.
+     *      - The 486 seems to do a dword limit read when the operand size is 32-bit.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemFetchDataU64(pVCpu, pGCPtrBase, iSegReg, GCPtrMem + 2);
+    }
+    else
+    {
+        uint32_t uTmp = 0; /* (Visual C++ maybe used uninitialized) */
+        if (enmOpSize == IEMMODE_32BIT)
+        {
+            if (IEM_GET_TARGET_CPU(pVCpu) != IEMTARGETCPU_486)
+            {
+                rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
+                if (rcStrict == VINF_SUCCESS)
+                    rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
+            }
+            else
+            {
+                rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    *pcbLimit = (uint16_t)uTmp;
+                    rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
+                }
+            }
+            if (rcStrict == VINF_SUCCESS)
+                *pGCPtrBase = uTmp;
+        }
+        else
+        {
+            rcStrict = iemMemFetchDataU16(pVCpu, pcbLimit, iSegReg, GCPtrMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                rcStrict = iemMemFetchDataU32(pVCpu, &uTmp, iSegReg, GCPtrMem + 2);
+                if (rcStrict == VINF_SUCCESS)
+                    *pGCPtrBase = uTmp & UINT32_C(0x00ffffff);
+            }
+        }
+    }
+    return rcStrict;
+}
+
+
+
+/**
+ * Stores a data byte.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u8Value             The value to store.
+ */
+VBOXSTRICTRC iemMemStoreDataU8(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint8_t *pu8Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu8Dst, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu8Dst = u8Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data byte, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u8Value             The value to store.
+ */
+void iemMemStoreDataU8Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint8_t u8Value) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    uint8_t *pu8Dst = (uint8_t *)iemMemMapJmp(pVCpu, sizeof(*pu8Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W, 0);
+    *pu8Dst = u8Value;
+    iemMemCommitAndUnmapJmp(pVCpu, pu8Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u16Value            The value to store.
+ */
+VBOXSTRICTRC iemMemStoreDataU16(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint16_t *pu16Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = u16Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data word, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u16Value            The value to store.
+ */
+void iemMemStoreDataU16Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint16_t u16Value) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    uint16_t *pu16Dst = (uint16_t *)iemMemMapJmp(pVCpu, sizeof(*pu16Dst), iSegReg, GCPtrMem,
+                                                 IEM_ACCESS_DATA_W, sizeof(*pu16Dst) - 1);
+    *pu16Dst = u16Value;
+    iemMemCommitAndUnmapJmp(pVCpu, pu16Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u32Value            The value to store.
+ */
+VBOXSTRICTRC iemMemStoreDataU32(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint32_t *pu32Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Dst = u32Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u32Value            The value to store.
+ */
+void iemMemStoreDataU32Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint32_t u32Value) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    uint32_t *pu32Dst = (uint32_t *)iemMemMapJmp(pVCpu, sizeof(*pu32Dst), iSegReg, GCPtrMem,
+                                                 IEM_ACCESS_DATA_W, sizeof(*pu32Dst) - 1);
+    *pu32Dst = u32Value;
+    iemMemCommitAndUnmapJmp(pVCpu, pu32Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u64Value            The value to store.
+ */
+VBOXSTRICTRC iemMemStoreDataU64(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint64_t *pu64Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = u64Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data qword, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u64Value            The value to store.
+ */
+void iemMemStoreDataU64Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, uint64_t u64Value) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    uint64_t *pu64Dst = (uint64_t *)iemMemMapJmp(pVCpu, sizeof(*pu64Dst), iSegReg, GCPtrMem,
+                                                 IEM_ACCESS_DATA_W, sizeof(*pu64Dst) - 1);
+    *pu64Dst = u64Value;
+    iemMemCommitAndUnmapJmp(pVCpu, pu64Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data dqword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u128Value            The value to store.
+ */
+VBOXSTRICTRC iemMemStoreDataU128(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    PRTUINT128U pu128Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
+    if (rc == VINF_SUCCESS)
+    {
+        pu128Dst->au64[0] = u128Value.au64[0];
+        pu128Dst->au64[1] = u128Value.au64[1];
+        rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data dqword, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u128Value            The value to store.
+ */
+void iemMemStoreDataU128Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem,
+                                                     IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
+    pu128Dst->au64[0] = u128Value.au64[0];
+    pu128Dst->au64[1] = u128Value.au64[1];
+    iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data dqword, SSE aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u128Value           The value to store.
+ */
+VBOXSTRICTRC iemMemStoreDataU128AlignedSse(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, RTUINT128U u128Value) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    PRTUINT128U pu128Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu128Dst, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
+                                (sizeof(*pu128Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
+    if (rc == VINF_SUCCESS)
+    {
+        pu128Dst->au64[0] = u128Value.au64[0];
+        pu128Dst->au64[1] = u128Value.au64[1];
+        rc = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data dqword, SSE aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   u128Value           The value to store.
+ */
+void iemMemStoreDataU128AlignedSseJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem,
+                                      RTUINT128U u128Value) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    PRTUINT128U pu128Dst = (PRTUINT128U)iemMemMapJmp(pVCpu, sizeof(*pu128Dst), iSegReg, GCPtrMem, IEM_ACCESS_DATA_W,
+                                                     (sizeof(*pu128Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_SSE);
+    pu128Dst->au64[0] = u128Value.au64[0];
+    pu128Dst->au64[1] = u128Value.au64[1];
+    iemMemCommitAndUnmapJmp(pVCpu, pu128Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data dqword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   pu256Value          Pointer to the value to store.
+ */
+VBOXSTRICTRC iemMemStoreDataU256(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    PRTUINT256U pu256Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
+    if (rc == VINF_SUCCESS)
+    {
+        pu256Dst->au64[0] = pu256Value->au64[0];
+        pu256Dst->au64[1] = pu256Value->au64[1];
+        pu256Dst->au64[2] = pu256Value->au64[2];
+        pu256Dst->au64[3] = pu256Value->au64[3];
+        rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data dqword, longjmp on error.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   pu256Value          Pointer to the value to store.
+ */
+void iemMemStoreDataU256Jmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem,
+                                                     IEM_ACCESS_DATA_W, 0 /* NO_AC variant */);
+    pu256Dst->au64[0] = pu256Value->au64[0];
+    pu256Dst->au64[1] = pu256Value->au64[1];
+    pu256Dst->au64[2] = pu256Value->au64[2];
+    pu256Dst->au64[3] = pu256Value->au64[3];
+    iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a data dqword, AVX \#GP(0) aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   pu256Value          Pointer to the value to store.
+ */
+VBOXSTRICTRC iemMemStoreDataU256AlignedAvx(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem, PCRTUINT256U pu256Value) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    PRTUINT256U pu256Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu256Dst, sizeof(*pu256Dst), iSegReg, GCPtrMem,
+                                IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP);
+    if (rc == VINF_SUCCESS)
+    {
+        pu256Dst->au64[0] = pu256Value->au64[0];
+        pu256Dst->au64[1] = pu256Value->au64[1];
+        pu256Dst->au64[2] = pu256Value->au64[2];
+        pu256Dst->au64[3] = pu256Value->au64[3];
+        rc = iemMemCommitAndUnmap(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
+    }
+    return rc;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Stores a data dqword, AVX aligned.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ * @param   pu256Value          Pointer to the value to store.
+ */
+void iemMemStoreDataU256AlignedAvxJmp(PVMCPUCC pVCpu, uint8_t iSegReg, RTGCPTR GCPtrMem,
+                                      PCRTUINT256U pu256Value) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    /* The lazy approach for now... */
+    PRTUINT256U pu256Dst = (PRTUINT256U)iemMemMapJmp(pVCpu, sizeof(*pu256Dst), iSegReg, GCPtrMem,
+                                                     IEM_ACCESS_DATA_W, (sizeof(*pu256Dst) - 1) | IEM_MEMMAP_F_ALIGN_GP);
+    pu256Dst->au64[0] = pu256Value->au64[0];
+    pu256Dst->au64[1] = pu256Value->au64[1];
+    pu256Dst->au64[2] = pu256Value->au64[2];
+    pu256Dst->au64[3] = pu256Value->au64[3];
+    iemMemCommitAndUnmapJmp(pVCpu, pu256Dst, IEM_ACCESS_DATA_W);
+}
+#endif
+
+
+/**
+ * Stores a descriptor register (sgdt, sidt).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbLimit             The limit.
+ * @param   GCPtrBase           The base address.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemStoreDataXdtr(PVMCPUCC pVCpu, uint16_t cbLimit, RTGCPTR GCPtrBase, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /*
+     * The SIDT and SGDT instructions actually stores the data using two
+     * independent writes (see bs3CpuBasic2_sidt_sgdt_One).  The instructions
+     * does not respond to opsize prefixes.
+     */
+    VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iSegReg, GCPtrMem, cbLimit);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_16BIT)
+            rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2,
+                                          IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_286
+                                          ? (uint32_t)GCPtrBase | UINT32_C(0xff000000) : (uint32_t)GCPtrBase);
+        else if (pVCpu->iem.s.enmCpuMode == IEMMODE_32BIT)
+            rcStrict = iemMemStoreDataU32(pVCpu, iSegReg, GCPtrMem + 2, (uint32_t)GCPtrBase);
+        else
+            rcStrict = iemMemStoreDataU64(pVCpu, iSegReg, GCPtrMem + 2, GCPtrBase);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Pushes a word onto the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16Value            The value to push.
+ */
+VBOXSTRICTRC iemMemStackPushU16(PVMCPUCC pVCpu, uint16_t u16Value) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPush(pVCpu, 2, &uNewRsp);
+
+    /* Write the word the lazy way. */
+    uint16_t *pu16Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = u16Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pushes a dword onto the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u32Value            The value to push.
+ */
+VBOXSTRICTRC iemMemStackPushU32(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
+
+    /* Write the dword the lazy way. */
+    uint32_t *pu32Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Dst = u32Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pushes a dword segment register value onto the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u32Value            The value to push.
+ */
+VBOXSTRICTRC iemMemStackPushU32SReg(PVMCPUCC pVCpu, uint32_t u32Value) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPush(pVCpu, 4, &uNewRsp);
+
+    /* The intel docs talks about zero extending the selector register
+       value.  My actual intel CPU here might be zero extending the value
+       but it still only writes the lower word... */
+    /** @todo Test this on new HW and on AMD and in 64-bit mode.  Also test what
+     * happens when crossing an electric page boundrary, is the high word checked
+     * for write accessibility or not? Probably it is.  What about segment limits?
+     * It appears this behavior is also shared with trap error codes.
+     *
+     * Docs indicate the behavior changed maybe in Pentium or Pentium Pro. Check
+     * ancient hardware when it actually did change. */
+    uint16_t *pu16Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(uint32_t), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_RW, sizeof(*pu16Dst) - 1); /** @todo 2 or 4 alignment check for PUSH SS? */
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = (uint16_t)u32Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_RW);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pushes a qword onto the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u64Value            The value to push.
+ */
+VBOXSTRICTRC iemMemStackPushU64(PVMCPUCC pVCpu, uint64_t u64Value) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPush(pVCpu, 8, &uNewRsp);
+
+    /* Write the word the lazy way. */
+    uint64_t *pu64Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = u64Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pops a word from the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16Value           Where to store the popped value.
+ */
+VBOXSTRICTRC iemMemStackPopU16(PVMCPUCC pVCpu, uint16_t *pu16Value) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPop(pVCpu, 2, &uNewRsp);
+
+    /* Write the word the lazy way. */
+    uint16_t const *pu16Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Value = *pu16Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rc == VINF_SUCCESS)
+            pVCpu->cpum.GstCtx.rsp = uNewRsp;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Pops a dword from the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32Value           Where to store the popped value.
+ */
+VBOXSTRICTRC iemMemStackPopU32(PVMCPUCC pVCpu, uint32_t *pu32Value) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPop(pVCpu, 4, &uNewRsp);
+
+    /* Write the word the lazy way. */
+    uint32_t const *pu32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Value = *pu32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rc == VINF_SUCCESS)
+            pVCpu->cpum.GstCtx.rsp = uNewRsp;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Pops a qword from the stack.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Value           Where to store the popped value.
+ */
+VBOXSTRICTRC iemMemStackPopU64(PVMCPUCC pVCpu, uint64_t *pu64Value) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    uint64_t    uNewRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPop(pVCpu, 8, &uNewRsp);
+
+    /* Write the word the lazy way. */
+    uint64_t const *pu64Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Value = *pu64Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rc == VINF_SUCCESS)
+            pVCpu->cpum.GstCtx.rsp = uNewRsp;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Pushes a word onto the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16Value            The value to push.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+VBOXSTRICTRC iemMemStackPushU16Ex(PVMCPUCC pVCpu, uint16_t u16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 2);
+
+    /* Write the word the lazy way. */
+    uint16_t *pu16Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Dst, sizeof(*pu16Dst), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_W, sizeof(*pu16Dst) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = u16Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu16Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        *pTmpRsp = NewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pushes a dword onto the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u32Value            The value to push.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+VBOXSTRICTRC iemMemStackPushU32Ex(PVMCPUCC pVCpu, uint32_t u32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 4);
+
+    /* Write the word the lazy way. */
+    uint32_t *pu32Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Dst, sizeof(*pu32Dst), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_W, sizeof(*pu32Dst) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Dst = u32Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu32Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        *pTmpRsp = NewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pushes a dword onto the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u64Value            The value to push.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+VBOXSTRICTRC iemMemStackPushU64Ex(PVMCPUCC pVCpu, uint64_t u64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPushEx(pVCpu, &NewRsp, 8);
+
+    /* Write the word the lazy way. */
+    uint64_t *pu64Dst;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Dst, sizeof(*pu64Dst), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_W, sizeof(*pu64Dst) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = u64Value;
+        rc = iemMemCommitAndUnmap(pVCpu, pu64Dst, IEM_ACCESS_STACK_W);
+    }
+
+    /* Commit the new RSP value unless we an access handler made trouble. */
+    if (rc == VINF_SUCCESS)
+        *pTmpRsp = NewRsp;
+
+    return rc;
+}
+
+
+/**
+ * Pops a word from the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16Value           Where to store the popped value.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+VBOXSTRICTRC iemMemStackPopU16Ex(PVMCPUCC pVCpu, uint16_t *pu16Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 2);
+
+    /* Write the word the lazy way. */
+    uint16_t const *pu16Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_R, sizeof(*pu16Src) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Value = *pu16Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rc == VINF_SUCCESS)
+            *pTmpRsp = NewRsp;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Pops a dword from the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32Value           Where to store the popped value.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+VBOXSTRICTRC iemMemStackPopU32Ex(PVMCPUCC pVCpu, uint32_t *pu32Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 4);
+
+    /* Write the word the lazy way. */
+    uint32_t const *pu32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), X86_SREG_SS, GCPtrTop,
+                                IEM_ACCESS_STACK_R, sizeof(*pu32Src) - 1);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Value = *pu32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rc == VINF_SUCCESS)
+            *pTmpRsp = NewRsp;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Pops a qword from the stack, using a temporary stack pointer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Value           Where to store the popped value.
+ * @param   pTmpRsp             Pointer to the temporary stack pointer.
+ */
+VBOXSTRICTRC iemMemStackPopU64Ex(PVMCPUCC pVCpu, uint64_t *pu64Value, PRTUINT64U pTmpRsp) RT_NOEXCEPT
+{
+    /* Increment the stack pointer. */
+    RTUINT64U   NewRsp = *pTmpRsp;
+    RTGCPTR     GCPtrTop = iemRegGetRspForPopEx(pVCpu, &NewRsp, 8);
+
+    /* Write the word the lazy way. */
+    uint64_t const *pu64Src;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), X86_SREG_SS, GCPtrTop,
+                                      IEM_ACCESS_STACK_R, sizeof(*pu64Src) - 1);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        *pu64Value = *pu64Src;
+        rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_STACK_R);
+
+        /* Commit the new RSP value. */
+        if (rcStrict == VINF_SUCCESS)
+            *pTmpRsp = NewRsp;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Begin a special stack push (used by interrupt, exceptions and such).
+ *
+ * This will raise \#SS or \#PF if appropriate.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbMem               The number of bytes to push onto the stack.
+ * @param   cbAlign             The alignment mask (7, 3, 1).
+ * @param   ppvMem              Where to return the pointer to the stack memory.
+ *                              As with the other memory functions this could be
+ *                              direct access or bounce buffered access, so
+ *                              don't commit register until the commit call
+ *                              succeeds.
+ * @param   puNewRsp            Where to return the new RSP value.  This must be
+ *                              passed unchanged to
+ *                              iemMemStackPushCommitSpecial().
+ */
+VBOXSTRICTRC iemMemStackPushBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
+                                         void **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT
+{
+    Assert(cbMem < UINT8_MAX);
+    RTGCPTR     GCPtrTop = iemRegGetRspForPush(pVCpu, (uint8_t)cbMem, puNewRsp);
+    return iemMemMap(pVCpu, ppvMem, cbMem, X86_SREG_SS, GCPtrTop,
+                     IEM_ACCESS_STACK_W, cbAlign);
+}
+
+
+/**
+ * Commits a special stack push (started by iemMemStackPushBeginSpecial).
+ *
+ * This will update the rSP.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem               The pointer returned by
+ *                              iemMemStackPushBeginSpecial().
+ * @param   uNewRsp             The new RSP value returned by
+ *                              iemMemStackPushBeginSpecial().
+ */
+VBOXSTRICTRC iemMemStackPushCommitSpecial(PVMCPUCC pVCpu, void *pvMem, uint64_t uNewRsp) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem, IEM_ACCESS_STACK_W);
+    if (rcStrict == VINF_SUCCESS)
+        pVCpu->cpum.GstCtx.rsp = uNewRsp;
+    return rcStrict;
+}
+
+
+/**
+ * Begin a special stack pop (used by iret, retf and such).
+ *
+ * This will raise \#SS or \#PF if appropriate.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   cbMem               The number of bytes to pop from the stack.
+ * @param   cbAlign             The alignment mask (7, 3, 1).
+ * @param   ppvMem              Where to return the pointer to the stack memory.
+ * @param   puNewRsp            Where to return the new RSP value.  This must be
+ *                              assigned to CPUMCTX::rsp manually some time
+ *                              after iemMemStackPopDoneSpecial() has been
+ *                              called.
+ */
+VBOXSTRICTRC iemMemStackPopBeginSpecial(PVMCPUCC pVCpu, size_t cbMem, uint32_t cbAlign,
+                                        void const **ppvMem, uint64_t *puNewRsp) RT_NOEXCEPT
+{
+    Assert(cbMem < UINT8_MAX);
+    RTGCPTR     GCPtrTop = iemRegGetRspForPop(pVCpu, (uint8_t)cbMem, puNewRsp);
+    return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop, IEM_ACCESS_STACK_R, cbAlign);
+}
+
+
+/**
+ * Continue a special stack pop (used by iret and retf), for the purpose of
+ * retrieving a new stack pointer.
+ *
+ * This will raise \#SS or \#PF if appropriate.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   off                 Offset from the top of the stack. This is zero
+ *                              except in the retf case.
+ * @param   cbMem               The number of bytes to pop from the stack.
+ * @param   ppvMem              Where to return the pointer to the stack memory.
+ * @param   uCurNewRsp          The current uncommitted RSP value.  (No need to
+ *                              return this because all use of this function is
+ *                              to retrieve a new value and anything we return
+ *                              here would be discarded.)
+ */
+VBOXSTRICTRC iemMemStackPopContinueSpecial(PVMCPUCC pVCpu, size_t off, size_t cbMem,
+                                           void const **ppvMem, uint64_t uCurNewRsp) RT_NOEXCEPT
+{
+    Assert(cbMem < UINT8_MAX);
+
+    /* The essense of iemRegGetRspForPopEx and friends: */ /** @todo put this into a inlined function? */
+    RTGCPTR GCPtrTop;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        GCPtrTop = uCurNewRsp;
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+        GCPtrTop = (uint32_t)uCurNewRsp;
+    else
+        GCPtrTop = (uint16_t)uCurNewRsp;
+
+    return iemMemMap(pVCpu, (void **)ppvMem, cbMem, X86_SREG_SS, GCPtrTop + off, IEM_ACCESS_STACK_R,
+                     0 /* checked in iemMemStackPopBeginSpecial */);
+}
+
+
+/**
+ * Done with a special stack pop (started by iemMemStackPopBeginSpecial or
+ * iemMemStackPopContinueSpecial).
+ *
+ * The caller will manually commit the rSP.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pvMem               The pointer returned by
+ *                              iemMemStackPopBeginSpecial() or
+ *                              iemMemStackPopContinueSpecial().
+ */
+VBOXSTRICTRC iemMemStackPopDoneSpecial(PVMCPUCC pVCpu, void const *pvMem) RT_NOEXCEPT
+{
+    return iemMemCommitAndUnmap(pVCpu, (void *)pvMem, IEM_ACCESS_STACK_R);
+}
+
+
+/**
+ * Fetches a system table byte.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pbDst               Where to return the byte.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchSysU8(PVMCPUCC pVCpu, uint8_t *pbDst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint8_t const *pbSrc;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pbSrc, sizeof(*pbSrc), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
+    if (rc == VINF_SUCCESS)
+    {
+        *pbDst = *pbSrc;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pbSrc, IEM_ACCESS_SYS_R);
+    }
+    return rc;
+}
+
+
+/**
+ * Fetches a system table word.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu16Dst             Where to return the word.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchSysU16(PVMCPUCC pVCpu, uint16_t *pu16Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint16_t const *pu16Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu16Src, sizeof(*pu16Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu16Dst = *pu16Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu16Src, IEM_ACCESS_SYS_R);
+    }
+    return rc;
+}
+
+
+/**
+ * Fetches a system table dword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu32Dst             Where to return the dword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchSysU32(PVMCPUCC pVCpu, uint32_t *pu32Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint32_t const *pu32Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu32Src, sizeof(*pu32Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu32Dst = *pu32Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu32Src, IEM_ACCESS_SYS_R);
+    }
+    return rc;
+}
+
+
+/**
+ * Fetches a system table qword.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pu64Dst             Where to return the qword.
+ * @param   iSegReg             The index of the segment register to use for
+ *                              this access.  The base and limits are checked.
+ * @param   GCPtrMem            The address of the guest memory.
+ */
+VBOXSTRICTRC iemMemFetchSysU64(PVMCPUCC pVCpu, uint64_t *pu64Dst, uint8_t iSegReg, RTGCPTR GCPtrMem) RT_NOEXCEPT
+{
+    /* The lazy approach for now... */
+    uint64_t const *pu64Src;
+    VBOXSTRICTRC rc = iemMemMap(pVCpu, (void **)&pu64Src, sizeof(*pu64Src), iSegReg, GCPtrMem, IEM_ACCESS_SYS_R, 0);
+    if (rc == VINF_SUCCESS)
+    {
+        *pu64Dst = *pu64Src;
+        rc = iemMemCommitAndUnmap(pVCpu, (void *)pu64Src, IEM_ACCESS_SYS_R);
+    }
+    return rc;
+}
+
+
+/**
+ * Fetches a descriptor table entry with caller specified error code.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pDesc               Where to return the descriptor table entry.
+ * @param   uSel                The selector which table entry to fetch.
+ * @param   uXcpt               The exception to raise on table lookup error.
+ * @param   uErrorCode          The error code associated with the exception.
+ */
+static VBOXSTRICTRC iemMemFetchSelDescWithErr(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel,
+                                              uint8_t uXcpt, uint16_t uErrorCode)  RT_NOEXCEPT
+{
+    AssertPtr(pDesc);
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
+
+    /** @todo did the 286 require all 8 bytes to be accessible? */
+    /*
+     * Get the selector table base and check bounds.
+     */
+    RTGCPTR GCPtrBase;
+    if (uSel & X86_SEL_LDT)
+    {
+        if (   !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
+            || (uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
+        {
+            Log(("iemMemFetchSelDesc: LDT selector %#x is out of bounds (%3x) or ldtr is NP (%#x)\n",
+                 uSel, pVCpu->cpum.GstCtx.ldtr.u32Limit, pVCpu->cpum.GstCtx.ldtr.Sel));
+            return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                                     uErrorCode, 0);
+        }
+
+        Assert(pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present);
+        GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
+    }
+    else
+    {
+        if ((uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
+        {
+            Log(("iemMemFetchSelDesc: GDT selector %#x is out of bounds (%3x)\n", uSel, pVCpu->cpum.GstCtx.gdtr.cbGdt));
+            return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR,
+                                     uErrorCode, 0);
+        }
+        GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
+    }
+
+    /*
+     * Read the legacy descriptor and maybe the long mode extensions if
+     * required.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_286)
+        rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
+    else
+    {
+        rcStrict     = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[0], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 0);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 2);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemFetchSysU16(pVCpu, &pDesc->Legacy.au16[2], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 4);
+        if (rcStrict == VINF_SUCCESS)
+            pDesc->Legacy.au16[3] = 0;
+        else
+            return rcStrict;
+    }
+
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (   !IEM_IS_LONG_MODE(pVCpu)
+            || pDesc->Legacy.Gen.u1DescType)
+            pDesc->Long.au64[1] = 0;
+        else if (   (uint32_t)(uSel | X86_SEL_RPL_LDT) + 8
+                 <= (uSel & X86_SEL_LDT ? pVCpu->cpum.GstCtx.ldtr.u32Limit : pVCpu->cpum.GstCtx.gdtr.cbGdt))
+            rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel | X86_SEL_RPL_LDT) + 1);
+        else
+        {
+            Log(("iemMemFetchSelDesc: system selector %#x is out of bounds\n", uSel));
+            /** @todo is this the right exception? */
+            return iemRaiseXcptOrInt(pVCpu, 0, uXcpt, IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_ERR, uErrorCode, 0);
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Fetches a descriptor table entry.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   pDesc               Where to return the descriptor table entry.
+ * @param   uSel                The selector which table entry to fetch.
+ * @param   uXcpt               The exception to raise on table lookup error.
+ */
+VBOXSTRICTRC iemMemFetchSelDesc(PVMCPUCC pVCpu, PIEMSELDESC pDesc, uint16_t uSel, uint8_t uXcpt) RT_NOEXCEPT
+{
+    return iemMemFetchSelDescWithErr(pVCpu, pDesc, uSel, uXcpt, uSel & X86_SEL_MASK_OFF_RPL);
+}
+
+
+/**
+ * Marks the selector descriptor as accessed (only non-system descriptors).
+ *
+ * This function ASSUMES that iemMemFetchSelDesc has be called previously and
+ * will therefore skip the limit checks.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   uSel                The selector.
+ */
+VBOXSTRICTRC iemMemMarkSelDescAccessed(PVMCPUCC pVCpu, uint16_t uSel) RT_NOEXCEPT
+{
+    /*
+     * Get the selector table base and calculate the entry address.
+     */
+    RTGCPTR GCPtr = uSel & X86_SEL_LDT
+                  ? pVCpu->cpum.GstCtx.ldtr.u64Base
+                  : pVCpu->cpum.GstCtx.gdtr.pGdt;
+    GCPtr += uSel & X86_SEL_MASK;
+
+    /*
+     * ASMAtomicBitSet will assert if the address is misaligned, so do some
+     * ugly stuff to avoid this.  This will make sure it's an atomic access
+     * as well more or less remove any question about 8-bit or 32-bit accesss.
+     */
+    VBOXSTRICTRC        rcStrict;
+    uint32_t volatile  *pu32;
+    if ((GCPtr & 3) == 0)
+    {
+        /* The normal case, map the 32-bit bits around the accessed bit (40). */
+        GCPtr += 2 + 2;
+        rcStrict = iemMemMap(pVCpu, (void **)&pu32, 4, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        ASMAtomicBitSet(pu32, 8); /* X86_SEL_TYPE_ACCESSED is 1, but it is preceeded by u8BaseHigh1. */
+    }
+    else
+    {
+        /* The misaligned GDT/LDT case, map the whole thing. */
+        rcStrict = iemMemMap(pVCpu, (void **)&pu32, 8, UINT8_MAX, GCPtr, IEM_ACCESS_SYS_RW, 0);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        switch ((uintptr_t)pu32 & 3)
+        {
+            case 0: ASMAtomicBitSet(pu32,                         40 + 0 -  0); break;
+            case 1: ASMAtomicBitSet((uint8_t volatile *)pu32 + 3, 40 + 0 - 24); break;
+            case 2: ASMAtomicBitSet((uint8_t volatile *)pu32 + 2, 40 + 0 - 16); break;
+            case 3: ASMAtomicBitSet((uint8_t volatile *)pu32 + 1, 40 + 0 -  8); break;
+        }
+    }
+
+    return iemMemCommitAndUnmap(pVCpu, (void *)pu32, IEM_ACCESS_SYS_RW);
+}
+
+/** @} */
+
+/** @name   Opcode Helpers.
+ * @{
+ */
+
+/**
+ * Calculates the effective address of a ModR/M memory operand.
+ *
+ * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
+ *
+ * @return  Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   bRm                 The ModRM byte.
+ * @param   cbImm               The size of any immediate following the
+ *                              effective address opcode bytes. Important for
+ *                              RIP relative addressing.
+ * @param   pGCPtrEff           Where to return the effective address.
+ */
+VBOXSTRICTRC iemOpHlpCalcRmEffAddr(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff) RT_NOEXCEPT
+{
+    Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
+# define SET_SS_DEF() \
+    do \
+    { \
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
+            pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
+    } while (0)
+
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+    {
+/** @todo Check the effective address size crap! */
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
+        {
+            uint16_t u16EffAddr;
+
+            /* Handle the disp16 form with no registers first. */
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
+                IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
+            else
+            {
+                /* Get the displacment. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:  u16EffAddr = 0;                             break;
+                    case 1:  IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
+                    case 2:  IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);       break;
+                    default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
+                }
+
+                /* Add the base and index registers to the disp. */
+                switch (bRm & X86_MODRM_RM_MASK)
+                {
+                    case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
+                    case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
+                    case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
+                    case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
+                    case 4: u16EffAddr += pVCpu->cpum.GstCtx.si;            break;
+                    case 5: u16EffAddr += pVCpu->cpum.GstCtx.di;            break;
+                    case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp;            SET_SS_DEF(); break;
+                    case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx;            break;
+                }
+            }
+
+            *pGCPtrEff = u16EffAddr;
+        }
+        else
+        {
+            Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+            uint32_t u32EffAddr;
+
+            /* Handle the disp32 form with no registers first. */
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+                IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
+            else
+            {
+                /* Get the register (or SIB) value. */
+                switch ((bRm & X86_MODRM_RM_MASK))
+                {
+                    case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                    case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                    case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                    case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                    case 4: /* SIB */
+                    {
+                        uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                        /* Get the index and scale it. */
+                        switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
+                        {
+                            case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                            case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                            case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                            case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                            case 4: u32EffAddr = 0; /*none */ break;
+                            case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
+                            case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                            case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                        }
+                        u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                        /* add base */
+                        switch (bSib & X86_SIB_BASE_MASK)
+                        {
+                            case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
+                            case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
+                            case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
+                            case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
+                            case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
+                            case 5:
+                                if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                                {
+                                    u32EffAddr += pVCpu->cpum.GstCtx.ebp;
+                                    SET_SS_DEF();
+                                }
+                                else
+                                {
+                                    uint32_t u32Disp;
+                                    IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                                    u32EffAddr += u32Disp;
+                                }
+                                break;
+                            case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
+                            case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
+                            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                        }
+                        break;
+                    }
+                    case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
+                    case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                    case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+
+                /* Get and add the displacement. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:
+                        break;
+                    case 1:
+                    {
+                        int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                        u32EffAddr += i8Disp;
+                        break;
+                    }
+                    case 2:
+                    {
+                        uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                        u32EffAddr += u32Disp;
+                        break;
+                    }
+                    default:
+                        AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
+                }
+
+            }
+            if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
+                *pGCPtrEff = u32EffAddr;
+            else
+            {
+                Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
+                *pGCPtrEff = u32EffAddr & UINT16_MAX;
+            }
+        }
+    }
+    else
+    {
+        uint64_t u64EffAddr;
+
+        /* Handle the rip+disp32 form with no registers first. */
+        if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+        {
+            IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
+            u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
+        }
+        else
+        {
+            /* Get the register (or SIB) value. */
+            switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
+            {
+                case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+                case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+                case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+                case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+                case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
+                case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+                case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+                case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+                case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+                case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+                case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+                case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+                case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+                case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+                /* SIB */
+                case 4:
+                case 12:
+                {
+                    uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                    /* Get the index and scale it. */
+                    switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
+                    {
+                        case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+                        case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+                        case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+                        case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+                        case  4: u64EffAddr = 0; /*none */ break;
+                        case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
+                        case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+                        case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+                        case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+                        case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+                        case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+                        case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+                        case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
+                        case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+                        case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+                        case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                    }
+                    u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                    /* add base */
+                    switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
+                    {
+                        case  0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
+                        case  1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
+                        case  2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
+                        case  3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
+                        case  4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
+                        case  6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
+                        case  7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
+                        case  8: u64EffAddr += pVCpu->cpum.GstCtx.r8;  break;
+                        case  9: u64EffAddr += pVCpu->cpum.GstCtx.r9;  break;
+                        case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
+                        case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
+                        case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
+                        case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
+                        case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
+                        /* complicated encodings */
+                        case 5:
+                        case 13:
+                            if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                            {
+                                if (!pVCpu->iem.s.uRexB)
+                                {
+                                    u64EffAddr += pVCpu->cpum.GstCtx.rbp;
+                                    SET_SS_DEF();
+                                }
+                                else
+                                    u64EffAddr += pVCpu->cpum.GstCtx.r13;
+                            }
+                            else
+                            {
+                                uint32_t u32Disp;
+                                IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                                u64EffAddr += (int32_t)u32Disp;
+                            }
+                            break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                    }
+                    break;
+                }
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+
+            /* Get and add the displacement. */
+            switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+            {
+                case 0:
+                    break;
+                case 1:
+                {
+                    int8_t i8Disp;
+                    IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                    u64EffAddr += i8Disp;
+                    break;
+                }
+                case 2:
+                {
+                    uint32_t u32Disp;
+                    IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                    u64EffAddr += (int32_t)u32Disp;
+                    break;
+                }
+                IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
+            }
+
+        }
+
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
+            *pGCPtrEff = u64EffAddr;
+        else
+        {
+            Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+            *pGCPtrEff = u64EffAddr & UINT32_MAX;
+        }
+    }
+
+    Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Calculates the effective address of a ModR/M memory operand.
+ *
+ * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
+ *
+ * @return  Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   bRm                 The ModRM byte.
+ * @param   cbImm               The size of any immediate following the
+ *                              effective address opcode bytes. Important for
+ *                              RIP relative addressing.
+ * @param   pGCPtrEff           Where to return the effective address.
+ * @param   offRsp              RSP displacement.
+ */
+VBOXSTRICTRC iemOpHlpCalcRmEffAddrEx(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm, PRTGCPTR pGCPtrEff, int8_t offRsp) RT_NOEXCEPT
+{
+    Log5(("iemOpHlpCalcRmEffAddr: bRm=%#x\n", bRm));
+# define SET_SS_DEF() \
+    do \
+    { \
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
+            pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
+    } while (0)
+
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+    {
+/** @todo Check the effective address size crap! */
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
+        {
+            uint16_t u16EffAddr;
+
+            /* Handle the disp16 form with no registers first. */
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
+                IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
+            else
+            {
+                /* Get the displacment. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:  u16EffAddr = 0;                             break;
+                    case 1:  IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
+                    case 2:  IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);       break;
+                    default: AssertFailedReturn(VERR_IEM_IPE_1); /* (caller checked for these) */
+                }
+
+                /* Add the base and index registers to the disp. */
+                switch (bRm & X86_MODRM_RM_MASK)
+                {
+                    case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
+                    case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
+                    case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
+                    case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
+                    case 4: u16EffAddr += pVCpu->cpum.GstCtx.si;            break;
+                    case 5: u16EffAddr += pVCpu->cpum.GstCtx.di;            break;
+                    case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp;            SET_SS_DEF(); break;
+                    case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx;            break;
+                }
+            }
+
+            *pGCPtrEff = u16EffAddr;
+        }
+        else
+        {
+            Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+            uint32_t u32EffAddr;
+
+            /* Handle the disp32 form with no registers first. */
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+                IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
+            else
+            {
+                /* Get the register (or SIB) value. */
+                switch ((bRm & X86_MODRM_RM_MASK))
+                {
+                    case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                    case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                    case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                    case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                    case 4: /* SIB */
+                    {
+                        uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                        /* Get the index and scale it. */
+                        switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
+                        {
+                            case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                            case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                            case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                            case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                            case 4: u32EffAddr = 0; /*none */ break;
+                            case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
+                            case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                            case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                        }
+                        u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                        /* add base */
+                        switch (bSib & X86_SIB_BASE_MASK)
+                        {
+                            case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
+                            case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
+                            case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
+                            case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
+                            case 4:
+                                u32EffAddr += pVCpu->cpum.GstCtx.esp + offRsp;
+                                SET_SS_DEF();
+                                break;
+                            case 5:
+                                if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                                {
+                                    u32EffAddr += pVCpu->cpum.GstCtx.ebp;
+                                    SET_SS_DEF();
+                                }
+                                else
+                                {
+                                    uint32_t u32Disp;
+                                    IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                                    u32EffAddr += u32Disp;
+                                }
+                                break;
+                            case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
+                            case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
+                            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                        }
+                        break;
+                    }
+                    case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
+                    case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                    case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+
+                /* Get and add the displacement. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:
+                        break;
+                    case 1:
+                    {
+                        int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                        u32EffAddr += i8Disp;
+                        break;
+                    }
+                    case 2:
+                    {
+                        uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                        u32EffAddr += u32Disp;
+                        break;
+                    }
+                    default:
+                        AssertFailedReturn(VERR_IEM_IPE_2); /* (caller checked for these) */
+                }
+
+            }
+            if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
+                *pGCPtrEff = u32EffAddr;
+            else
+            {
+                Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
+                *pGCPtrEff = u32EffAddr & UINT16_MAX;
+            }
+        }
+    }
+    else
+    {
+        uint64_t u64EffAddr;
+
+        /* Handle the rip+disp32 form with no registers first. */
+        if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+        {
+            IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
+            u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
+        }
+        else
+        {
+            /* Get the register (or SIB) value. */
+            switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
+            {
+                case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+                case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+                case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+                case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+                case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
+                case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+                case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+                case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+                case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+                case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+                case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+                case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+                case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+                case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+                /* SIB */
+                case 4:
+                case 12:
+                {
+                    uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                    /* Get the index and scale it. */
+                    switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
+                    {
+                        case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+                        case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+                        case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+                        case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+                        case  4: u64EffAddr = 0; /*none */ break;
+                        case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
+                        case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+                        case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+                        case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+                        case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+                        case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+                        case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+                        case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
+                        case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+                        case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+                        case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                    }
+                    u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                    /* add base */
+                    switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
+                    {
+                        case  0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
+                        case  1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
+                        case  2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
+                        case  3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
+                        case  4: u64EffAddr += pVCpu->cpum.GstCtx.rsp + offRsp; SET_SS_DEF(); break;
+                        case  6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
+                        case  7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
+                        case  8: u64EffAddr += pVCpu->cpum.GstCtx.r8;  break;
+                        case  9: u64EffAddr += pVCpu->cpum.GstCtx.r9;  break;
+                        case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
+                        case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
+                        case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
+                        case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
+                        case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
+                        /* complicated encodings */
+                        case 5:
+                        case 13:
+                            if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                            {
+                                if (!pVCpu->iem.s.uRexB)
+                                {
+                                    u64EffAddr += pVCpu->cpum.GstCtx.rbp;
+                                    SET_SS_DEF();
+                                }
+                                else
+                                    u64EffAddr += pVCpu->cpum.GstCtx.r13;
+                            }
+                            else
+                            {
+                                uint32_t u32Disp;
+                                IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                                u64EffAddr += (int32_t)u32Disp;
+                            }
+                            break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                    }
+                    break;
+                }
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+
+            /* Get and add the displacement. */
+            switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+            {
+                case 0:
+                    break;
+                case 1:
+                {
+                    int8_t i8Disp;
+                    IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                    u64EffAddr += i8Disp;
+                    break;
+                }
+                case 2:
+                {
+                    uint32_t u32Disp;
+                    IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                    u64EffAddr += (int32_t)u32Disp;
+                    break;
+                }
+                IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* (caller checked for these) */
+            }
+
+        }
+
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
+            *pGCPtrEff = u64EffAddr;
+        else
+        {
+            Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+            *pGCPtrEff = u64EffAddr & UINT32_MAX;
+        }
+    }
+
+    Log5(("iemOpHlpCalcRmEffAddr: EffAddr=%#010RGv\n", *pGCPtrEff));
+    return VINF_SUCCESS;
+}
+
+
+#ifdef IEM_WITH_SETJMP
+/**
+ * Calculates the effective address of a ModR/M memory operand.
+ *
+ * Meant to be used via IEM_MC_CALC_RM_EFF_ADDR.
+ *
+ * May longjmp on internal error.
+ *
+ * @return  The effective address.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   bRm                 The ModRM byte.
+ * @param   cbImm               The size of any immediate following the
+ *                              effective address opcode bytes. Important for
+ *                              RIP relative addressing.
+ */
+RTGCPTR iemOpHlpCalcRmEffAddrJmp(PVMCPUCC pVCpu, uint8_t bRm, uint8_t cbImm) IEM_NOEXCEPT_MAY_LONGJMP
+{
+    Log5(("iemOpHlpCalcRmEffAddrJmp: bRm=%#x\n", bRm));
+# define SET_SS_DEF() \
+    do \
+    { \
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SEG_MASK)) \
+            pVCpu->iem.s.iEffSeg = X86_SREG_SS; \
+    } while (0)
+
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+    {
+/** @todo Check the effective address size crap! */
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
+        {
+            uint16_t u16EffAddr;
+
+            /* Handle the disp16 form with no registers first. */
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
+                IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);
+            else
+            {
+                /* Get the displacment. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:  u16EffAddr = 0;                             break;
+                    case 1:  IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16EffAddr); break;
+                    case 2:  IEM_OPCODE_GET_NEXT_U16(&u16EffAddr);       break;
+                    default: AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_1)); /* (caller checked for these) */
+                }
+
+                /* Add the base and index registers to the disp. */
+                switch (bRm & X86_MODRM_RM_MASK)
+                {
+                    case 0: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.si; break;
+                    case 1: u16EffAddr += pVCpu->cpum.GstCtx.bx + pVCpu->cpum.GstCtx.di; break;
+                    case 2: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.si; SET_SS_DEF(); break;
+                    case 3: u16EffAddr += pVCpu->cpum.GstCtx.bp + pVCpu->cpum.GstCtx.di; SET_SS_DEF(); break;
+                    case 4: u16EffAddr += pVCpu->cpum.GstCtx.si;            break;
+                    case 5: u16EffAddr += pVCpu->cpum.GstCtx.di;            break;
+                    case 6: u16EffAddr += pVCpu->cpum.GstCtx.bp;            SET_SS_DEF(); break;
+                    case 7: u16EffAddr += pVCpu->cpum.GstCtx.bx;            break;
+                }
+            }
+
+            Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX16\n", u16EffAddr));
+            return u16EffAddr;
+        }
+
+        Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+        uint32_t u32EffAddr;
+
+        /* Handle the disp32 form with no registers first. */
+        if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+            IEM_OPCODE_GET_NEXT_U32(&u32EffAddr);
+        else
+        {
+            /* Get the register (or SIB) value. */
+            switch ((bRm & X86_MODRM_RM_MASK))
+            {
+                case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                case 4: /* SIB */
+                {
+                    uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                    /* Get the index and scale it. */
+                    switch ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK)
+                    {
+                        case 0: u32EffAddr = pVCpu->cpum.GstCtx.eax; break;
+                        case 1: u32EffAddr = pVCpu->cpum.GstCtx.ecx; break;
+                        case 2: u32EffAddr = pVCpu->cpum.GstCtx.edx; break;
+                        case 3: u32EffAddr = pVCpu->cpum.GstCtx.ebx; break;
+                        case 4: u32EffAddr = 0; /*none */ break;
+                        case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; break;
+                        case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                        case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+                    }
+                    u32EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                    /* add base */
+                    switch (bSib & X86_SIB_BASE_MASK)
+                    {
+                        case 0: u32EffAddr += pVCpu->cpum.GstCtx.eax; break;
+                        case 1: u32EffAddr += pVCpu->cpum.GstCtx.ecx; break;
+                        case 2: u32EffAddr += pVCpu->cpum.GstCtx.edx; break;
+                        case 3: u32EffAddr += pVCpu->cpum.GstCtx.ebx; break;
+                        case 4: u32EffAddr += pVCpu->cpum.GstCtx.esp; SET_SS_DEF(); break;
+                        case 5:
+                            if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                            {
+                                u32EffAddr += pVCpu->cpum.GstCtx.ebp;
+                                SET_SS_DEF();
+                            }
+                            else
+                            {
+                                uint32_t u32Disp;
+                                IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                                u32EffAddr += u32Disp;
+                            }
+                            break;
+                        case 6: u32EffAddr += pVCpu->cpum.GstCtx.esi; break;
+                        case 7: u32EffAddr += pVCpu->cpum.GstCtx.edi; break;
+                        IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+                    }
+                    break;
+                }
+                case 5: u32EffAddr = pVCpu->cpum.GstCtx.ebp; SET_SS_DEF(); break;
+                case 6: u32EffAddr = pVCpu->cpum.GstCtx.esi; break;
+                case 7: u32EffAddr = pVCpu->cpum.GstCtx.edi; break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+            }
+
+            /* Get and add the displacement. */
+            switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+            {
+                case 0:
+                    break;
+                case 1:
+                {
+                    int8_t i8Disp; IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                    u32EffAddr += i8Disp;
+                    break;
+                }
+                case 2:
+                {
+                    uint32_t u32Disp; IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                    u32EffAddr += u32Disp;
+                    break;
+                }
+                default:
+                    AssertFailedStmt(IEM_DO_LONGJMP(pVCpu, VERR_IEM_IPE_2)); /* (caller checked for these) */
+            }
+        }
+
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
+        {
+            Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RX32\n", u32EffAddr));
+            return u32EffAddr;
+        }
+        Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT);
+        Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#06RX32\n", u32EffAddr & UINT16_MAX));
+        return u32EffAddr & UINT16_MAX;
+    }
+
+    uint64_t u64EffAddr;
+
+    /* Handle the rip+disp32 form with no registers first. */
+    if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+    {
+        IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64EffAddr);
+        u64EffAddr += pVCpu->cpum.GstCtx.rip + IEM_GET_INSTR_LEN(pVCpu) + cbImm;
+    }
+    else
+    {
+        /* Get the register (or SIB) value. */
+        switch ((bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB)
+        {
+            case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+            case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+            case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+            case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+            case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; SET_SS_DEF(); break;
+            case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+            case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+            case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+            case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+            case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+            case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+            case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+            case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+            case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+            /* SIB */
+            case 4:
+            case 12:
+            {
+                uint8_t bSib; IEM_OPCODE_GET_NEXT_U8(&bSib);
+
+                /* Get the index and scale it. */
+                switch (((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex)
+                {
+                    case  0: u64EffAddr = pVCpu->cpum.GstCtx.rax; break;
+                    case  1: u64EffAddr = pVCpu->cpum.GstCtx.rcx; break;
+                    case  2: u64EffAddr = pVCpu->cpum.GstCtx.rdx; break;
+                    case  3: u64EffAddr = pVCpu->cpum.GstCtx.rbx; break;
+                    case  4: u64EffAddr = 0; /*none */ break;
+                    case  5: u64EffAddr = pVCpu->cpum.GstCtx.rbp; break;
+                    case  6: u64EffAddr = pVCpu->cpum.GstCtx.rsi; break;
+                    case  7: u64EffAddr = pVCpu->cpum.GstCtx.rdi; break;
+                    case  8: u64EffAddr = pVCpu->cpum.GstCtx.r8;  break;
+                    case  9: u64EffAddr = pVCpu->cpum.GstCtx.r9;  break;
+                    case 10: u64EffAddr = pVCpu->cpum.GstCtx.r10; break;
+                    case 11: u64EffAddr = pVCpu->cpum.GstCtx.r11; break;
+                    case 12: u64EffAddr = pVCpu->cpum.GstCtx.r12; break;
+                    case 13: u64EffAddr = pVCpu->cpum.GstCtx.r13; break;
+                    case 14: u64EffAddr = pVCpu->cpum.GstCtx.r14; break;
+                    case 15: u64EffAddr = pVCpu->cpum.GstCtx.r15; break;
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+                }
+                u64EffAddr <<= (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                /* add base */
+                switch ((bSib & X86_SIB_BASE_MASK) | pVCpu->iem.s.uRexB)
+                {
+                    case  0: u64EffAddr += pVCpu->cpum.GstCtx.rax; break;
+                    case  1: u64EffAddr += pVCpu->cpum.GstCtx.rcx; break;
+                    case  2: u64EffAddr += pVCpu->cpum.GstCtx.rdx; break;
+                    case  3: u64EffAddr += pVCpu->cpum.GstCtx.rbx; break;
+                    case  4: u64EffAddr += pVCpu->cpum.GstCtx.rsp; SET_SS_DEF(); break;
+                    case  6: u64EffAddr += pVCpu->cpum.GstCtx.rsi; break;
+                    case  7: u64EffAddr += pVCpu->cpum.GstCtx.rdi; break;
+                    case  8: u64EffAddr += pVCpu->cpum.GstCtx.r8;  break;
+                    case  9: u64EffAddr += pVCpu->cpum.GstCtx.r9;  break;
+                    case 10: u64EffAddr += pVCpu->cpum.GstCtx.r10; break;
+                    case 11: u64EffAddr += pVCpu->cpum.GstCtx.r11; break;
+                    case 12: u64EffAddr += pVCpu->cpum.GstCtx.r12; break;
+                    case 14: u64EffAddr += pVCpu->cpum.GstCtx.r14; break;
+                    case 15: u64EffAddr += pVCpu->cpum.GstCtx.r15; break;
+                    /* complicated encodings */
+                    case 5:
+                    case 13:
+                        if ((bRm & X86_MODRM_MOD_MASK) != 0)
+                        {
+                            if (!pVCpu->iem.s.uRexB)
+                            {
+                                u64EffAddr += pVCpu->cpum.GstCtx.rbp;
+                                SET_SS_DEF();
+                            }
+                            else
+                                u64EffAddr += pVCpu->cpum.GstCtx.r13;
+                        }
+                        else
+                        {
+                            uint32_t u32Disp;
+                            IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                            u64EffAddr += (int32_t)u32Disp;
+                        }
+                        break;
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+                }
+                break;
+            }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX);
+        }
+
+        /* Get and add the displacement. */
+        switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+        {
+            case 0:
+                break;
+            case 1:
+            {
+                int8_t i8Disp;
+                IEM_OPCODE_GET_NEXT_S8(&i8Disp);
+                u64EffAddr += i8Disp;
+                break;
+            }
+            case 2:
+            {
+                uint32_t u32Disp;
+                IEM_OPCODE_GET_NEXT_U32(&u32Disp);
+                u64EffAddr += (int32_t)u32Disp;
+                break;
+            }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET2(RTGCPTR_MAX); /* (caller checked for these) */
+        }
+
+    }
+
+    if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
+    {
+        Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr));
+        return u64EffAddr;
+    }
+    Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT);
+    Log5(("iemOpHlpCalcRmEffAddrJmp: EffAddr=%#010RGv\n", u64EffAddr & UINT32_MAX));
+    return u64EffAddr & UINT32_MAX;
+}
+#endif /* IEM_WITH_SETJMP */
+
+/** @}  */
+
+
+#ifdef LOG_ENABLED
+/**
+ * Logs the current instruction.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   fSameCtx    Set if we have the same context information as the VMM,
+ *                      clear if we may have already executed an instruction in
+ *                      our debug context. When clear, we assume IEMCPU holds
+ *                      valid CPU mode info.
+ *
+ *                      The @a fSameCtx parameter is now misleading and obsolete.
+ * @param   pszFunction The IEM function doing the execution.
+ */
+static void iemLogCurInstr(PVMCPUCC pVCpu, bool fSameCtx, const char *pszFunction) RT_NOEXCEPT
+{
+# ifdef IN_RING3
+    if (LogIs2Enabled())
+    {
+        char     szInstr[256];
+        uint32_t cbInstr = 0;
+        if (fSameCtx)
+            DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, 0, 0,
+                               DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                               szInstr, sizeof(szInstr), &cbInstr);
+        else
+        {
+            uint32_t fFlags = 0;
+            switch (pVCpu->iem.s.enmCpuMode)
+            {
+                case IEMMODE_64BIT: fFlags |= DBGF_DISAS_FLAGS_64BIT_MODE; break;
+                case IEMMODE_32BIT: fFlags |= DBGF_DISAS_FLAGS_32BIT_MODE; break;
+                case IEMMODE_16BIT:
+                    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) || pVCpu->cpum.GstCtx.eflags.Bits.u1VM)
+                        fFlags |= DBGF_DISAS_FLAGS_16BIT_REAL_MODE;
+                    else
+                        fFlags |= DBGF_DISAS_FLAGS_16BIT_MODE;
+                    break;
+            }
+            DBGFR3DisasInstrEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fFlags,
+                               szInstr, sizeof(szInstr), &cbInstr);
+        }
+
+        PCX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+        Log2(("**** %s\n"
+              " eax=%08x ebx=%08x ecx=%08x edx=%08x esi=%08x edi=%08x\n"
+              " eip=%08x esp=%08x ebp=%08x iopl=%d tr=%04x\n"
+              " cs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x efl=%08x\n"
+              " fsw=%04x fcw=%04x ftw=%02x mxcsr=%04x/%04x\n"
+              " %s\n"
+              , pszFunction,
+              pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ebx, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.esi, pVCpu->cpum.GstCtx.edi,
+              pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.esp, pVCpu->cpum.GstCtx.ebp, pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL, pVCpu->cpum.GstCtx.tr.Sel,
+              pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.es.Sel,
+              pVCpu->cpum.GstCtx.fs.Sel, pVCpu->cpum.GstCtx.gs.Sel, pVCpu->cpum.GstCtx.eflags.u,
+              pFpuCtx->FSW, pFpuCtx->FCW, pFpuCtx->FTW, pFpuCtx->MXCSR, pFpuCtx->MXCSR_MASK,
+              szInstr));
+
+        if (LogIs3Enabled())
+            DBGFR3InfoEx(pVCpu->pVMR3->pUVM, pVCpu->idCpu, "cpumguest", "verbose", NULL);
+    }
+    else
+# endif
+        LogFlow(("%s: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x\n", pszFunction, pVCpu->cpum.GstCtx.cs.Sel,
+                 pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u));
+    RT_NOREF_PV(pVCpu); RT_NOREF_PV(fSameCtx);
+}
+#endif /* LOG_ENABLED */
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Deals with VMCPU_FF_VMX_APIC_WRITE, VMCPU_FF_VMX_MTF, VMCPU_FF_VMX_NMI_WINDOW,
+ * VMCPU_FF_VMX_PREEMPT_TIMER and VMCPU_FF_VMX_INT_WINDOW.
+ *
+ * @returns Modified rcStrict.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ * @param   rcStrict    The instruction execution status.
+ */
+static VBOXSTRICTRC iemHandleNestedInstructionBoundaryFFs(PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict) RT_NOEXCEPT
+{
+    Assert(CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)));
+    if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF))
+    {
+        /* VMX preemption timer takes priority over NMI-window exits. */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
+        {
+            rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
+            Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
+        }
+        /*
+         * Check remaining intercepts.
+         *
+         * NMI-window and Interrupt-window VM-exits.
+         * Interrupt shadow (block-by-STI and Mov SS) inhibits interrupts and may also block NMIs.
+         * Event injection during VM-entry takes priority over NMI-window and interrupt-window VM-exits.
+         *
+         * See Intel spec. 26.7.6 "NMI-Window Exiting".
+         * See Intel spec. 26.7.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
+         */
+        else if (   VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW | VMCPU_FF_VMX_INT_WINDOW)
+                 && !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)
+                 && !TRPMHasTrap(pVCpu))
+        {
+            Assert(CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx));
+            if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW)
+                && CPUMIsGuestVmxVirtNmiBlocking(&pVCpu->cpum.GstCtx))
+            {
+                rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_NMI_WINDOW, 0 /* u64ExitQual */);
+                Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
+            }
+            else if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW)
+                     && CPUMIsGuestVmxVirtIntrEnabled(&pVCpu->cpum.GstCtx))
+            {
+                rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_INT_WINDOW, 0 /* u64ExitQual */);
+                Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
+            }
+        }
+    }
+    /* TPR-below threshold/APIC write has the highest priority. */
+    else  if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
+    {
+        rcStrict = iemVmxApicWriteEmulation(pVCpu);
+        Assert(!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx));
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
+    }
+    /* MTF takes priority over VMX-preemption timer. */
+    else
+    {
+        rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* u64ExitQual */);
+        Assert(!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx));
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
+    }
+    return rcStrict;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/** @def IEM_TRY_SETJMP
+ * Wrapper around setjmp / try, hiding all the ugly differences.
+ *
+ * @note Use with extreme care as this is a fragile macro.
+ * @param   a_pVCpu     The cross context virtual CPU structure of the calling EMT.
+ * @param   a_rcTarget  The variable that should receive the status code in case
+ *                      of a longjmp/throw.
+ */
+/** @def IEM_TRY_SETJMP_AGAIN
+ * For when setjmp / try is used again in the same variable scope as a previous
+ * IEM_TRY_SETJMP invocation.
+ */
+/** @def IEM_CATCH_LONGJMP_BEGIN
+ * Start wrapper for catch / setjmp-else.
+ *
+ * This will set up a scope.
+ *
+ * @note Use with extreme care as this is a fragile macro.
+ * @param   a_pVCpu     The cross context virtual CPU structure of the calling EMT.
+ * @param   a_rcTarget  The variable that should receive the status code in case
+ *                      of a longjmp/throw.
+ */
+/** @def IEM_CATCH_LONGJMP_END
+ * End wrapper for catch / setjmp-else.
+ *
+ * This will close the scope set up by IEM_CATCH_LONGJMP_BEGIN and clean up the
+ * state.
+ *
+ * @note Use with extreme care as this is a fragile macro.
+ * @param   a_pVCpu     The cross context virtual CPU structure of the calling EMT.
+ */
+#if defined(IEM_WITH_SETJMP) || defined(DOXYGEN_RUNNING)
+# ifdef IEM_WITH_THROW_CATCH
+#  define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \
+        a_rcTarget = VINF_SUCCESS; \
+        try
+#  define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \
+        IEM_TRY_SETJMP(a_pVCpu, a_rcTarget)
+#  define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \
+        catch (int rcThrown) \
+        { \
+            a_rcTarget = rcThrown
+#  define IEM_CATCH_LONGJMP_END(a_pVCpu) \
+        } \
+        ((void)0)
+# else  /* !IEM_WITH_THROW_CATCH */
+#  define IEM_TRY_SETJMP(a_pVCpu, a_rcTarget) \
+        jmp_buf  JmpBuf; \
+        jmp_buf * volatile pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf); \
+        pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \
+        if ((rcStrict = setjmp(JmpBuf)) == 0)
+#  define IEM_TRY_SETJMP_AGAIN(a_pVCpu, a_rcTarget) \
+        pSavedJmpBuf = pVCpu->iem.s.CTX_SUFF(pJmpBuf); \
+        pVCpu->iem.s.CTX_SUFF(pJmpBuf) = &JmpBuf; \
+        if ((rcStrict = setjmp(JmpBuf)) == 0)
+#  define IEM_CATCH_LONGJMP_BEGIN(a_pVCpu, a_rcTarget) \
+        else \
+        { \
+            ((void)0)
+#  define IEM_CATCH_LONGJMP_END(a_pVCpu) \
+        } \
+        (a_pVCpu)->iem.s.CTX_SUFF(pJmpBuf) = pSavedJmpBuf
+# endif /* !IEM_WITH_THROW_CATCH */
+#endif  /* IEM_WITH_SETJMP */
+
+
+/**
+ * The actual code execution bits of IEMExecOne, IEMExecOneEx, and
+ * IEMExecOneWithPrefetchedByPC.
+ *
+ * Similar code is found in IEMExecLots.
+ *
+ * @return  Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   fExecuteInhibit     If set, execute the instruction following CLI,
+ *                      POP SS and MOV SS,GR.
+ * @param   pszFunction The calling function name.
+ */
+DECLINLINE(VBOXSTRICTRC) iemExecOneInner(PVMCPUCC pVCpu, bool fExecuteInhibit, const char *pszFunction)
+{
+    AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
+    AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
+    AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
+    RT_NOREF_PV(pszFunction);
+
+#ifdef IEM_WITH_SETJMP
+    VBOXSTRICTRC rcStrict;
+    IEM_TRY_SETJMP(pVCpu, rcStrict)
+    {
+        uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
+        rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+    IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
+    {
+        pVCpu->iem.s.cLongJumps++;
+    }
+    IEM_CATCH_LONGJMP_END(pVCpu);
+#else
+    uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
+    VBOXSTRICTRC rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+#endif
+    if (rcStrict == VINF_SUCCESS)
+        pVCpu->iem.s.cInstructions++;
+    if (pVCpu->iem.s.cActiveMappings > 0)
+    {
+        Assert(rcStrict != VINF_SUCCESS);
+        iemMemRollback(pVCpu);
+    }
+    AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
+    AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
+    AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
+
+//#ifdef DEBUG
+//    AssertMsg(IEM_GET_INSTR_LEN(pVCpu) == cbInstr || rcStrict != VINF_SUCCESS, ("%u %u\n", IEM_GET_INSTR_LEN(pVCpu), cbInstr));
+//#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Perform any VMX nested-guest instruction boundary actions.
+     *
+     * If any of these causes a VM-exit, we must skip executing the next
+     * instruction (would run into stale page tables). A VM-exit makes sure
+     * there is no interrupt-inhibition, so that should ensure we don't go
+     * to try execute the next instruction. Clearing fExecuteInhibit is
+     * problematic because of the setjmp/longjmp clobbering above.
+     */
+    if (   !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
+                                     | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW)
+        || rcStrict != VINF_SUCCESS)
+    { /* likely */ }
+    else
+        rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
+#endif
+
+    /* Execute the next instruction as well if a cli, pop ss or
+       mov ss, Gr has just completed successfully. */
+    if (   fExecuteInhibit
+        && rcStrict == VINF_SUCCESS
+        && CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
+    {
+        rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, pVCpu->iem.s.fBypassHandlers, pVCpu->iem.s.fDisregardLock);
+        if (rcStrict == VINF_SUCCESS)
+        {
+#ifdef LOG_ENABLED
+            iemLogCurInstr(pVCpu, false, pszFunction);
+#endif
+#ifdef IEM_WITH_SETJMP
+            IEM_TRY_SETJMP_AGAIN(pVCpu, rcStrict)
+            {
+                uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
+                rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+            }
+            IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
+            {
+                pVCpu->iem.s.cLongJumps++;
+            }
+            IEM_CATCH_LONGJMP_END(pVCpu);
+#else
+            IEM_OPCODE_GET_FIRST_U8(&b);
+            rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+#endif
+            if (rcStrict == VINF_SUCCESS)
+            {
+                pVCpu->iem.s.cInstructions++;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
+                                              | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW))
+                { /* likely */ }
+                else
+                    rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
+#endif
+            }
+            if (pVCpu->iem.s.cActiveMappings > 0)
+            {
+                Assert(rcStrict != VINF_SUCCESS);
+                iemMemRollback(pVCpu);
+            }
+            AssertMsg(pVCpu->iem.s.aMemMappings[0].fAccess == IEM_ACCESS_INVALID, ("0: %#x %RGp\n", pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemBbMappings[0].GCPhysFirst));
+            AssertMsg(pVCpu->iem.s.aMemMappings[1].fAccess == IEM_ACCESS_INVALID, ("1: %#x %RGp\n", pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemBbMappings[1].GCPhysFirst));
+            AssertMsg(pVCpu->iem.s.aMemMappings[2].fAccess == IEM_ACCESS_INVALID, ("2: %#x %RGp\n", pVCpu->iem.s.aMemMappings[2].fAccess, pVCpu->iem.s.aMemBbMappings[2].GCPhysFirst));
+        }
+        else if (pVCpu->iem.s.cActiveMappings > 0)
+            iemMemRollback(pVCpu);
+        /** @todo drop this after we bake this change into RIP advancing. */
+        CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx); /* hope this is correct for all exceptional cases... */
+    }
+
+    /*
+     * Return value fiddling, statistics and sanity assertions.
+     */
+    rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    return rcStrict;
+}
+
+
+/**
+ * Execute one instruction.
+ *
+ * @return  Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(VBOXSTRICTRC) IEMExecOne(PVMCPUCC pVCpu)
+{
+    AssertCompile(sizeof(pVCpu->iem.s) <= sizeof(pVCpu->iem.padding)); /* (tstVMStruct can't do it's job w/o instruction stats) */
+#ifdef LOG_ENABLED
+    iemLogCurInstr(pVCpu, true, "IEMExecOne");
+#endif
+
+    /*
+     * Do the decoding and emulation.
+     */
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOne");
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    if (rcStrict != VINF_SUCCESS)
+        LogFlow(("IEMExecOne: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
+                 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+VMMDECL(VBOXSTRICTRC) IEMExecOneEx(PVMCPUCC pVCpu, uint32_t *pcbWritten)
+{
+    uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneEx");
+        if (pcbWritten)
+            *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
+    }
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+VMMDECL(VBOXSTRICTRC) IEMExecOneWithPrefetchedByPC(PVMCPUCC pVCpu, uint64_t OpcodeBytesPC,
+                                                   const void *pvOpcodeBytes, size_t cbOpcodeBytes)
+{
+    VBOXSTRICTRC rcStrict;
+    if (   cbOpcodeBytes
+        && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
+    {
+        iemInitDecoder(pVCpu, false, false);
+#ifdef IEM_WITH_CODE_TLB
+        pVCpu->iem.s.uInstrBufPc      = OpcodeBytesPC;
+        pVCpu->iem.s.pbInstrBuf       = (uint8_t const *)pvOpcodeBytes;
+        pVCpu->iem.s.cbInstrBufTotal  = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
+        pVCpu->iem.s.offCurInstrStart = 0;
+        pVCpu->iem.s.offInstrNextByte = 0;
+#else
+        pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
+        memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
+#endif
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneWithPrefetchedByPC");
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+VMMDECL(VBOXSTRICTRC) IEMExecOneBypassEx(PVMCPUCC pVCpu, uint32_t *pcbWritten)
+{
+    uint32_t const cbOldWritten = pVCpu->iem.s.cbWritten;
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassEx");
+        if (pcbWritten)
+            *pcbWritten = pVCpu->iem.s.cbWritten - cbOldWritten;
+    }
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+VMMDECL(VBOXSTRICTRC) IEMExecOneBypassWithPrefetchedByPC(PVMCPUCC pVCpu, uint64_t OpcodeBytesPC,
+                                                         const void *pvOpcodeBytes, size_t cbOpcodeBytes)
+{
+    VBOXSTRICTRC rcStrict;
+    if (   cbOpcodeBytes
+        && pVCpu->cpum.GstCtx.rip == OpcodeBytesPC)
+    {
+        iemInitDecoder(pVCpu, true, false);
+#ifdef IEM_WITH_CODE_TLB
+        pVCpu->iem.s.uInstrBufPc      = OpcodeBytesPC;
+        pVCpu->iem.s.pbInstrBuf       = (uint8_t const *)pvOpcodeBytes;
+        pVCpu->iem.s.cbInstrBufTotal  = (uint16_t)RT_MIN(X86_PAGE_SIZE, cbOpcodeBytes);
+        pVCpu->iem.s.offCurInstrStart = 0;
+        pVCpu->iem.s.offInstrNextByte = 0;
+#else
+        pVCpu->iem.s.cbOpcode = (uint8_t)RT_MIN(cbOpcodeBytes, sizeof(pVCpu->iem.s.abOpcode));
+        memcpy(pVCpu->iem.s.abOpcode, pvOpcodeBytes, pVCpu->iem.s.cbOpcode);
+#endif
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, true, false);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemExecOneInner(pVCpu, false, "IEMExecOneBypassWithPrefetchedByPC");
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+/**
+ * For handling split cacheline lock operations when the host has split-lock
+ * detection enabled.
+ *
+ * This will cause the interpreter to disregard the lock prefix and implicit
+ * locking (xchg).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ */
+VMMDECL(VBOXSTRICTRC) IEMExecOneIgnoreLock(PVMCPUCC pVCpu)
+{
+    /*
+     * Do the decoding and emulation.
+     */
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, true /*fDisregardLock*/);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemExecOneInner(pVCpu, true, "IEMExecOneIgnoreLock");
+    else if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    if (rcStrict != VINF_SUCCESS)
+        LogFlow(("IEMExecOneIgnoreLock: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
+                 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+VMMDECL(VBOXSTRICTRC) IEMExecLots(PVMCPUCC pVCpu, uint32_t cMaxInstructions, uint32_t cPollRate, uint32_t *pcInstructions)
+{
+    uint32_t const cInstructionsAtStart = pVCpu->iem.s.cInstructions;
+    AssertMsg(RT_IS_POWER_OF_TWO(cPollRate + 1), ("%#x\n", cPollRate));
+
+    /*
+     * See if there is an interrupt pending in TRPM, inject it if we can.
+     */
+    /** @todo What if we are injecting an exception and not an interrupt? Is that
+     *        possible here? For now we assert it is indeed only an interrupt. */
+    if (!TRPMHasTrap(pVCpu))
+    { /* likely */ }
+    else
+    {
+        if (   !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)
+            && !CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
+        {
+            /** @todo Can we centralize this under CPUMCanInjectInterrupt()? */
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+            bool fIntrEnabled = CPUMGetGuestGif(&pVCpu->cpum.GstCtx);
+            if (fIntrEnabled)
+            {
+                if (!CPUMIsGuestInNestedHwvirtMode(IEM_GET_CTX(pVCpu)))
+                    fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
+                else if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
+                    fIntrEnabled = CPUMIsGuestVmxPhysIntrEnabled(IEM_GET_CTX(pVCpu));
+                else
+                {
+                    Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
+                    fIntrEnabled = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, IEM_GET_CTX(pVCpu));
+                }
+            }
+#else
+            bool fIntrEnabled = pVCpu->cpum.GstCtx.eflags.Bits.u1IF;
+#endif
+            if (fIntrEnabled)
+            {
+                uint8_t     u8TrapNo;
+                TRPMEVENT   enmType;
+                uint32_t    uErrCode;
+                RTGCPTR     uCr2;
+                int rc2 = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, NULL /*pu8InstLen*/, NULL /*fIcebp*/);
+                AssertRC(rc2);
+                Assert(enmType == TRPM_HARDWARE_INT);
+                VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, (uint16_t)uErrCode, uCr2, 0 /*cbInstr*/);
+
+                TRPMResetTrap(pVCpu);
+
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+                /* Injecting an event may cause a VM-exit. */
+                if (   rcStrict != VINF_SUCCESS
+                    && rcStrict != VINF_IEM_RAISED_XCPT)
+                    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+#else
+                NOREF(rcStrict);
+#endif
+            }
+        }
+    }
+
+    /*
+     * Initial decoder init w/ prefetch, then setup setjmp.
+     */
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
+    if (rcStrict == VINF_SUCCESS)
+    {
+#ifdef IEM_WITH_SETJMP
+        pVCpu->iem.s.cActiveMappings = 0; /** @todo wtf? */
+        IEM_TRY_SETJMP(pVCpu, rcStrict)
+#endif
+        {
+            /*
+             * The run loop.  We limit ourselves to 4096 instructions right now.
+             */
+            uint32_t cMaxInstructionsGccStupidity = cMaxInstructions;
+            PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+            for (;;)
+            {
+                /*
+                 * Log the state.
+                 */
+#ifdef LOG_ENABLED
+                iemLogCurInstr(pVCpu, true, "IEMExecLots");
+#endif
+
+                /*
+                 * Do the decoding and emulation.
+                 */
+                uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
+                rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+#ifdef VBOX_STRICT
+                CPUMAssertGuestRFlagsCookie(pVM, pVCpu);
+#endif
+                if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+                {
+                    Assert(pVCpu->iem.s.cActiveMappings == 0);
+                    pVCpu->iem.s.cInstructions++;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                    /* Perform any VMX nested-guest instruction boundary actions. */
+                    uint64_t fCpu = pVCpu->fLocalForcedActions;
+                    if (!(fCpu & (  VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
+                                  | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW)))
+                    { /* likely */ }
+                    else
+                    {
+                        rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
+                        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+                            fCpu = pVCpu->fLocalForcedActions;
+                        else
+                        {
+                            rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+                            break;
+                        }
+                    }
+#endif
+                    if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
+                    {
+#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
+                        uint64_t fCpu = pVCpu->fLocalForcedActions;
+#endif
+                        fCpu &= VMCPU_FF_ALL_MASK & ~(  VMCPU_FF_PGM_SYNC_CR3
+                                                      | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
+                                                      | VMCPU_FF_TLB_FLUSH
+                                                      | VMCPU_FF_UNHALT );
+
+                        if (RT_LIKELY(   (   !fCpu
+                                          || (   !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
+                                              && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF) )
+                                      && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) ))
+                        {
+                            if (cMaxInstructionsGccStupidity-- > 0)
+                            {
+                                /* Poll timers every now an then according to the caller's specs. */
+                                if (   (cMaxInstructionsGccStupidity & cPollRate) != 0
+                                    || !TMTimerPollBool(pVM, pVCpu))
+                                {
+                                    Assert(pVCpu->iem.s.cActiveMappings == 0);
+                                    iemReInitDecoder(pVCpu);
+                                    continue;
+                                }
+                            }
+                        }
+                    }
+                    Assert(pVCpu->iem.s.cActiveMappings == 0);
+                }
+                else if (pVCpu->iem.s.cActiveMappings > 0)
+                    iemMemRollback(pVCpu);
+                rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+                break;
+            }
+        }
+#ifdef IEM_WITH_SETJMP
+        IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
+        {
+            if (pVCpu->iem.s.cActiveMappings > 0)
+                iemMemRollback(pVCpu);
+# if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+            rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+# endif
+            pVCpu->iem.s.cLongJumps++;
+        }
+        IEM_CATCH_LONGJMP_END(pVCpu);
+#endif
+
+        /*
+         * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
+         */
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    }
+    else
+    {
+        if (pVCpu->iem.s.cActiveMappings > 0)
+            iemMemRollback(pVCpu);
+
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+        /*
+         * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
+         * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
+         */
+        rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+#endif
+    }
+
+    /*
+     * Maybe re-enter raw-mode and log.
+     */
+    if (rcStrict != VINF_SUCCESS)
+        LogFlow(("IEMExecLots: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc\n",
+                 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp, pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict)));
+    if (pcInstructions)
+        *pcInstructions = pVCpu->iem.s.cInstructions - cInstructionsAtStart;
+    return rcStrict;
+}
+
+
+/**
+ * Interface used by EMExecuteExec, does exit statistics and limits.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   fWillExit           To be defined.
+ * @param   cMinInstructions    Minimum number of instructions to execute before checking for FFs.
+ * @param   cMaxInstructions    Maximum number of instructions to execute.
+ * @param   cMaxInstructionsWithoutExits
+ *                              The max number of instructions without exits.
+ * @param   pStats              Where to return statistics.
+ */
+VMMDECL(VBOXSTRICTRC) IEMExecForExits(PVMCPUCC pVCpu, uint32_t fWillExit, uint32_t cMinInstructions, uint32_t cMaxInstructions,
+                                      uint32_t cMaxInstructionsWithoutExits, PIEMEXECFOREXITSTATS pStats)
+{
+    NOREF(fWillExit); /** @todo define flexible exit crits */
+
+    /*
+     * Initialize return stats.
+     */
+    pStats->cInstructions    = 0;
+    pStats->cExits           = 0;
+    pStats->cMaxExitDistance = 0;
+    pStats->cReserved        = 0;
+
+    /*
+     * Initial decoder init w/ prefetch, then setup setjmp.
+     */
+    VBOXSTRICTRC rcStrict = iemInitDecoderAndPrefetchOpcodes(pVCpu, false, false);
+    if (rcStrict == VINF_SUCCESS)
+    {
+#ifdef IEM_WITH_SETJMP
+        pVCpu->iem.s.cActiveMappings     = 0; /** @todo wtf?!? */
+        IEM_TRY_SETJMP(pVCpu, rcStrict)
+#endif
+        {
+#ifdef IN_RING0
+            bool const fCheckPreemptionPending   = !RTThreadPreemptIsPossible() || !RTThreadPreemptIsEnabled(NIL_RTTHREAD);
+#endif
+            uint32_t   cInstructionSinceLastExit = 0;
+
+            /*
+             * The run loop.  We limit ourselves to 4096 instructions right now.
+             */
+            PVM pVM = pVCpu->CTX_SUFF(pVM);
+            for (;;)
+            {
+                /*
+                 * Log the state.
+                 */
+#ifdef LOG_ENABLED
+                iemLogCurInstr(pVCpu, true, "IEMExecForExits");
+#endif
+
+                /*
+                 * Do the decoding and emulation.
+                 */
+                uint32_t const cPotentialExits = pVCpu->iem.s.cPotentialExits;
+
+                uint8_t b; IEM_OPCODE_GET_FIRST_U8(&b);
+                rcStrict = FNIEMOP_CALL(g_apfnOneByteMap[b]);
+
+                if (   cPotentialExits != pVCpu->iem.s.cPotentialExits
+                    && cInstructionSinceLastExit > 0 /* don't count the first */ )
+                {
+                    pStats->cExits += 1;
+                    if (cInstructionSinceLastExit > pStats->cMaxExitDistance)
+                        pStats->cMaxExitDistance = cInstructionSinceLastExit;
+                    cInstructionSinceLastExit = 0;
+                }
+
+                if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+                {
+                    Assert(pVCpu->iem.s.cActiveMappings == 0);
+                    pVCpu->iem.s.cInstructions++;
+                    pStats->cInstructions++;
+                    cInstructionSinceLastExit++;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                    /* Perform any VMX nested-guest instruction boundary actions. */
+                    uint64_t fCpu = pVCpu->fLocalForcedActions;
+                    if (!(fCpu & (  VMCPU_FF_VMX_APIC_WRITE | VMCPU_FF_VMX_MTF | VMCPU_FF_VMX_PREEMPT_TIMER
+                                  | VMCPU_FF_VMX_INT_WINDOW | VMCPU_FF_VMX_NMI_WINDOW)))
+                    { /* likely */ }
+                    else
+                    {
+                        rcStrict = iemHandleNestedInstructionBoundaryFFs(pVCpu, rcStrict);
+                        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+                            fCpu = pVCpu->fLocalForcedActions;
+                        else
+                        {
+                            rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+                            break;
+                        }
+                    }
+#endif
+                    if (RT_LIKELY(pVCpu->iem.s.rcPassUp == VINF_SUCCESS))
+                    {
+#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
+                        uint64_t fCpu = pVCpu->fLocalForcedActions;
+#endif
+                        fCpu &= VMCPU_FF_ALL_MASK & ~(  VMCPU_FF_PGM_SYNC_CR3
+                                                      | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
+                                                      | VMCPU_FF_TLB_FLUSH
+                                                      | VMCPU_FF_UNHALT );
+                        if (RT_LIKELY(   (   (   !fCpu
+                                              || (   !(fCpu & ~(VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
+                                                  && !pVCpu->cpum.GstCtx.rflags.Bits.u1IF))
+                                          && !VM_FF_IS_ANY_SET(pVM, VM_FF_ALL_MASK) )
+                                      || pStats->cInstructions < cMinInstructions))
+                        {
+                            if (pStats->cInstructions < cMaxInstructions)
+                            {
+                                if (cInstructionSinceLastExit <= cMaxInstructionsWithoutExits)
+                                {
+#ifdef IN_RING0
+                                    if (   !fCheckPreemptionPending
+                                        || !RTThreadPreemptIsPending(NIL_RTTHREAD))
+#endif
+                                    {
+                                        Assert(pVCpu->iem.s.cActiveMappings == 0);
+                                        iemReInitDecoder(pVCpu);
+                                        continue;
+                                    }
+#ifdef IN_RING0
+                                    rcStrict = VINF_EM_RAW_INTERRUPT;
+                                    break;
+#endif
+                                }
+                            }
+                        }
+                        Assert(!(fCpu & VMCPU_FF_IEM));
+                    }
+                    Assert(pVCpu->iem.s.cActiveMappings == 0);
+                }
+                else if (pVCpu->iem.s.cActiveMappings > 0)
+                        iemMemRollback(pVCpu);
+                rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+                break;
+            }
+        }
+#ifdef IEM_WITH_SETJMP
+        IEM_CATCH_LONGJMP_BEGIN(pVCpu, rcStrict);
+        {
+            if (pVCpu->iem.s.cActiveMappings > 0)
+                iemMemRollback(pVCpu);
+            pVCpu->iem.s.cLongJumps++;
+        }
+        IEM_CATCH_LONGJMP_END(pVCpu);
+#endif
+
+        /*
+         * Assert hidden register sanity (also done in iemInitDecoder and iemReInitDecoder).
+         */
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+    }
+    else
+    {
+        if (pVCpu->iem.s.cActiveMappings > 0)
+            iemMemRollback(pVCpu);
+
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+        /*
+         * When a nested-guest causes an exception intercept (e.g. #PF) when fetching
+         * code as part of instruction execution, we need this to fix-up VINF_SVM_VMEXIT.
+         */
+        rcStrict = iemExecStatusCodeFiddling(pVCpu, rcStrict);
+#endif
+    }
+
+    /*
+     * Maybe re-enter raw-mode and log.
+     */
+    if (rcStrict != VINF_SUCCESS)
+        LogFlow(("IEMExecForExits: cs:rip=%04x:%08RX64 ss:rsp=%04x:%08RX64 EFL=%06x - rcStrict=%Rrc; ins=%u exits=%u maxdist=%u\n",
+                 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp,
+                 pVCpu->cpum.GstCtx.eflags.u, VBOXSTRICTRC_VAL(rcStrict), pStats->cInstructions, pStats->cExits, pStats->cMaxExitDistance));
+    return rcStrict;
+}
+
+
+/**
+ * Injects a trap, fault, abort, software interrupt or external interrupt.
+ *
+ * The parameter list matches TRPMQueryTrapAll pretty closely.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   u8TrapNo            The trap number.
+ * @param   enmType             What type is it (trap/fault/abort), software
+ *                              interrupt or hardware interrupt.
+ * @param   uErrCode            The error code if applicable.
+ * @param   uCr2                The CR2 value if applicable.
+ * @param   cbInstr             The instruction length (only relevant for
+ *                              software interrupts).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMInjectTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType, uint16_t uErrCode, RTGCPTR uCr2,
+                                         uint8_t cbInstr)
+{
+    iemInitDecoder(pVCpu, false, false);
+#ifdef DBGFTRACE_ENABLED
+    RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "IEMInjectTrap: %x %d %x %llx",
+                      u8TrapNo, enmType, uErrCode, uCr2);
+#endif
+
+    uint32_t fFlags;
+    switch (enmType)
+    {
+        case TRPM_HARDWARE_INT:
+            Log(("IEMInjectTrap: %#4x ext\n", u8TrapNo));
+            fFlags = IEM_XCPT_FLAGS_T_EXT_INT;
+            uErrCode = uCr2 = 0;
+            break;
+
+        case TRPM_SOFTWARE_INT:
+            Log(("IEMInjectTrap: %#4x soft\n", u8TrapNo));
+            fFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
+            uErrCode = uCr2 = 0;
+            break;
+
+        case TRPM_TRAP:
+            Log(("IEMInjectTrap: %#4x trap err=%#x cr2=%#RGv\n", u8TrapNo, uErrCode, uCr2));
+            fFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
+            if (u8TrapNo == X86_XCPT_PF)
+                fFlags |= IEM_XCPT_FLAGS_CR2;
+            switch (u8TrapNo)
+            {
+                case X86_XCPT_DF:
+                case X86_XCPT_TS:
+                case X86_XCPT_NP:
+                case X86_XCPT_SS:
+                case X86_XCPT_PF:
+                case X86_XCPT_AC:
+                case X86_XCPT_GP:
+                    fFlags |= IEM_XCPT_FLAGS_ERR;
+                    break;
+            }
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    VBOXSTRICTRC rcStrict = iemRaiseXcptOrInt(pVCpu, cbInstr, u8TrapNo, fFlags, uErrCode, uCr2);
+
+    if (pVCpu->iem.s.cActiveMappings > 0)
+        iemMemRollback(pVCpu);
+
+    return rcStrict;
+}
+
+
+/**
+ * Injects the active TRPM event.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMDECL(VBOXSTRICTRC) IEMInjectTrpmEvent(PVMCPUCC pVCpu)
+{
+#ifndef IEM_IMPLEMENTS_TASKSWITCH
+    IEM_RETURN_ASPECT_NOT_IMPLEMENTED_LOG(("Event injection\n"));
+#else
+    uint8_t     u8TrapNo;
+    TRPMEVENT   enmType;
+    uint32_t    uErrCode;
+    RTGCUINTPTR uCr2;
+    uint8_t     cbInstr;
+    int rc = TRPMQueryTrapAll(pVCpu, &u8TrapNo, &enmType, &uErrCode, &uCr2, &cbInstr, NULL /* fIcebp */);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /** @todo r=ramshankar: Pass ICEBP info. to IEMInjectTrap() below and handle
+     *        ICEBP \#DB injection as a special case. */
+    VBOXSTRICTRC rcStrict = IEMInjectTrap(pVCpu, u8TrapNo, enmType, uErrCode, uCr2, cbInstr);
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (rcStrict == VINF_SVM_VMEXIT)
+        rcStrict = VINF_SUCCESS;
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (rcStrict == VINF_VMX_VMEXIT)
+        rcStrict = VINF_SUCCESS;
+#endif
+    /** @todo Are there any other codes that imply the event was successfully
+     *        delivered to the guest? See @bugref{6607}.  */
+    if (   rcStrict == VINF_SUCCESS
+        || rcStrict == VINF_IEM_RAISED_XCPT)
+        TRPMResetTrap(pVCpu);
+
+    return rcStrict;
+#endif
+}
+
+
+VMM_INT_DECL(int) IEMBreakpointSet(PVM pVM, RTGCPTR GCPtrBp)
+{
+    RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+VMM_INT_DECL(int) IEMBreakpointClear(PVM pVM, RTGCPTR GCPtrBp)
+{
+    RT_NOREF_PV(pVM); RT_NOREF_PV(GCPtrBp);
+    return VERR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Interface for HM and EM for executing string I/O OUT (write) instructions.
+ *
+ * This API ASSUMES that the caller has already verified that the guest code is
+ * allowed to access the I/O port.  (The I/O port is in the DX register in the
+ * guest state.)
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbValue             The size of the I/O port access (1, 2, or 4).
+ * @param   enmAddrMode         The addressing mode.
+ * @param   fRepPrefix          Indicates whether a repeat prefix is used
+ *                              (doesn't matter which for this instruction).
+ * @param   cbInstr             The instruction length in bytes.
+ * @param   iEffSeg             The effective segment address.
+ * @param   fIoChecked          Whether the access to the I/O port has been
+ *                              checked or not.  It's typically checked in the
+ *                              HM scenario.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoWrite(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
+                                                bool fRepPrefix, uint8_t cbInstr, uint8_t iEffSeg, bool fIoChecked)
+{
+    AssertMsgReturn(iEffSeg < X86_SREG_COUNT, ("%#x\n", iEffSeg), VERR_IEM_INVALID_EFF_SEG);
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
+
+    /*
+     * State init.
+     */
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    /*
+     * Switch orgy for getting to the right handler.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (fRepPrefix)
+    {
+        switch (enmAddrMode)
+        {
+            case IEMMODE_16BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_32BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_64BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            default:
+                AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
+        }
+    }
+    else
+    {
+        switch (enmAddrMode)
+        {
+            case IEMMODE_16BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_outs_op8_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_outs_op16_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_outs_op32_addr16(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_32BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_outs_op8_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_outs_op16_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_outs_op32_addr32(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_64BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_outs_op8_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_outs_op16_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_outs_op32_addr64(pVCpu, cbInstr, iEffSeg, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            default:
+                AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
+        }
+    }
+
+    if (pVCpu->iem.s.cActiveMappings)
+        iemMemRollback(pVCpu);
+
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM for executing string I/O IN (read) instructions.
+ *
+ * This API ASSUMES that the caller has already verified that the guest code is
+ * allowed to access the I/O port.  (The I/O port is in the DX register in the
+ * guest state.)
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbValue             The size of the I/O port access (1, 2, or 4).
+ * @param   enmAddrMode         The addressing mode.
+ * @param   fRepPrefix          Indicates whether a repeat prefix is used
+ *                              (doesn't matter which for this instruction).
+ * @param   cbInstr             The instruction length in bytes.
+ * @param   fIoChecked          Whether the access to the I/O port has been
+ *                              checked or not.  It's typically checked in the
+ *                              HM scenario.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecStringIoRead(PVMCPUCC pVCpu, uint8_t cbValue, IEMMODE enmAddrMode,
+                                               bool fRepPrefix, uint8_t cbInstr, bool fIoChecked)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
+
+    /*
+     * State init.
+     */
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    /*
+     * Switch orgy for getting to the right handler.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (fRepPrefix)
+    {
+        switch (enmAddrMode)
+        {
+            case IEMMODE_16BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_32BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_64BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_rep_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_rep_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_rep_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            default:
+                AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
+        }
+    }
+    else
+    {
+        switch (enmAddrMode)
+        {
+            case IEMMODE_16BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_ins_op8_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_ins_op16_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_ins_op32_addr16(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_32BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_ins_op8_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_ins_op16_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_ins_op32_addr32(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            case IEMMODE_64BIT:
+                switch (cbValue)
+                {
+                    case 1: rcStrict = iemCImpl_ins_op8_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    case 2: rcStrict = iemCImpl_ins_op16_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    case 4: rcStrict = iemCImpl_ins_op32_addr64(pVCpu, cbInstr, fIoChecked); break;
+                    default:
+                        AssertMsgFailedReturn(("cbValue=%#x\n", cbValue), VERR_IEM_INVALID_OPERAND_SIZE);
+                }
+                break;
+
+            default:
+                AssertMsgFailedReturn(("enmAddrMode=%d\n", enmAddrMode), VERR_IEM_INVALID_ADDRESS_MODE);
+        }
+    }
+
+    if (   pVCpu->iem.s.cActiveMappings == 0
+        || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_IEM))
+    { /* likely */ }
+    else
+    {
+        AssertMsg(!IOM_SUCCESS(rcStrict), ("%#x\n", VBOXSTRICTRC_VAL(rcStrict)));
+        iemMemRollback(pVCpu);
+    }
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for rawmode to write execute an OUT instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   u16Port     The port to read.
+ * @param   fImm        Whether the port is specified using an immediate operand or
+ *                      using the implicit DX register.
+ * @param   cbReg       The register size.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedOut(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
+    Assert(cbReg <= 4 && cbReg != 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_out, u16Port, fImm, cbReg);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for rawmode to write execute an IN instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   u16Port     The port to read.
+ * @param   fImm        Whether the port is specified using an immediate operand or
+ *                      using the implicit DX.
+ * @param   cbReg       The register size.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedIn(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t u16Port, bool fImm, uint8_t cbReg)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
+    Assert(cbReg <= 4 && cbReg != 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_in, u16Port, fImm, cbReg);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to write to a CRx register.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iCrReg      The control register number (destination).
+ * @param   iGReg       The general purpose register number (source).
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iCrReg, uint8_t iGReg)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    Assert(iCrReg < 16);
+    Assert(iGReg < 16);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Cd_Rd, iCrReg, iGReg);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to read from a CRx register.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iGReg       The general purpose register number (destination).
+ * @param   iCrReg      The control register number (source).
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovCRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4
+                        | CPUMCTX_EXTRN_APIC_TPR);
+    Assert(iCrReg < 16);
+    Assert(iGReg < 16);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Cd, iGReg, iCrReg);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to write to a DRx register.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iDrReg      The debug register number (destination).
+ * @param   iGReg       The general purpose register number (source).
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovDRxWrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iDrReg, uint8_t iGReg)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_DR7);
+    Assert(iDrReg < 8);
+    Assert(iGReg < 16);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Dd_Rd, iDrReg, iGReg);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to read from a DRx register.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iGReg       The general purpose register number (destination).
+ * @param   iDrReg      The debug register number (source).
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMovDRxRead(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iDrReg)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_DR7);
+    Assert(iDrReg < 8);
+    Assert(iGReg < 16);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_mov_Rd_Dd, iGReg, iDrReg);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to clear the CR0[TS] bit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClts(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clts);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the LMSW instruction (loads CR0).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   uValue          The value to load into CR0.
+ * @param   GCPtrEffDst     The guest-linear address if the LMSW instruction has a
+ *                          memory operand. Otherwise pass NIL_RTGCPTR.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedLmsw(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uValue, RTGCPTR GCPtrEffDst)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_2(iemCImpl_lmsw, uValue, GCPtrEffDst);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the XSETBV instruction (loads XCRx).
+ *
+ * Takes input values in ecx and edx:eax of the CPU context of the calling EMT.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedXsetbv(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_xsetbv);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the WBINVD instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWbinvd(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wbinvd);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVD instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvd(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invd);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVLPG instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_PGM_SYNC_CR3
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   GCPtrPage   The effective address of the page to invalidate.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpg(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPTR GCPtrPage)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_invlpg, GCPtrPage);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVPCID instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_PGM_SYNC_CR3
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iEffSeg     The effective segment register.
+ * @param   GCPtrDesc   The effective address of the INVPCID descriptor.
+ * @param   uType       The invalidation type.
+ *
+ * @remarks In ring-0 not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvpcid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDesc,
+                                                 uint64_t uType)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 4);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_3(iemCImpl_invpcid, iEffSeg, GCPtrDesc, uType);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the CPUID instruction.
+ *
+ * @returns Strict VBox status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in, the usual pluss RAX and RCX.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedCpuid(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_cpuid);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the RDPMC instruction.
+ *
+ * @returns Strict VBox status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdpmc(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdpmc);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the RDTSC instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtsc(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtsc);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the RDTSCP instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.  Recommended
+ *          to include CPUMCTX_EXTRN_TSC_AUX, to avoid extra fetch call.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdtscp(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_TSC_AUX);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdtscp);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the RDMSR instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.  Requires RCX and
+ *          (currently) all MSRs.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedRdmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_ALL_MSRS);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_rdmsr);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the WRMSR instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.  Requires RCX, RAX, RDX,
+ *          and (currently) all MSRs.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedWrmsr(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 2);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
+                        | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_ALL_MSRS);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_wrmsr);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the MONITOR instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.
+ * @remarks ASSUMES the default segment of DS and no segment override prefixes
+ *          are used.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMonitor(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_1(iemCImpl_monitor, X86_SREG_DS);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the MWAIT instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedMwait(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RAX);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_mwait);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the HLT instruction.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_IEM_RAISED_XCPT (VINF_EM_RESCHEDULE) if exception is raised.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ *
+ * @remarks Not all of the state needs to be synced in.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedHlt(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 1);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_hlt);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Checks if IEM is in the process of delivering an event (interrupt or
+ * exception).
+ *
+ * @returns true if we're in the process of raising an interrupt or exception,
+ *          false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   puVector        Where to store the vector associated with the
+ *                          currently delivered event, optional.
+ * @param   pfFlags         Where to store th event delivery flags (see
+ *                          IEM_XCPT_FLAGS_XXX), optional.
+ * @param   puErr           Where to store the error code associated with the
+ *                          event, optional.
+ * @param   puCr2           Where to store the CR2 associated with the event,
+ *                          optional.
+ * @remarks The caller should check the flags to determine if the error code and
+ *          CR2 are valid for the event.
+ */
+VMM_INT_DECL(bool) IEMGetCurrentXcpt(PVMCPUCC pVCpu, uint8_t *puVector, uint32_t *pfFlags, uint32_t *puErr, uint64_t *puCr2)
+{
+    bool const fRaisingXcpt = pVCpu->iem.s.cXcptRecursions > 0;
+    if (fRaisingXcpt)
+    {
+        if (puVector)
+            *puVector = pVCpu->iem.s.uCurXcpt;
+        if (pfFlags)
+            *pfFlags = pVCpu->iem.s.fCurXcpt;
+        if (puErr)
+            *puErr = pVCpu->iem.s.uCurXcptErr;
+        if (puCr2)
+            *puCr2 = pVCpu->iem.s.uCurXcptCr2;
+    }
+    return fRaisingXcpt;
+}
+
+#ifdef IN_RING3
+
+/**
+ * Handles the unlikely and probably fatal merge cases.
+ *
+ * @returns Merged status code.
+ * @param   rcStrict        Current EM status code.
+ * @param   rcStrictCommit  The IOM I/O or MMIO write commit status to merge
+ *                          with @a rcStrict.
+ * @param   iMemMap         The memory mapping index. For error reporting only.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          thread, for error reporting only.
+ */
+DECL_NO_INLINE(static, VBOXSTRICTRC) iemR3MergeStatusSlow(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit,
+                                                          unsigned iMemMap, PVMCPUCC pVCpu)
+{
+    if (RT_FAILURE_NP(rcStrict))
+        return rcStrict;
+
+    if (RT_FAILURE_NP(rcStrictCommit))
+        return rcStrictCommit;
+
+    if (rcStrict == rcStrictCommit)
+        return rcStrictCommit;
+
+    AssertLogRelMsgFailed(("rcStrictCommit=%Rrc rcStrict=%Rrc iMemMap=%u fAccess=%#x FirstPg=%RGp LB %u SecondPg=%RGp LB %u\n",
+                           VBOXSTRICTRC_VAL(rcStrictCommit), VBOXSTRICTRC_VAL(rcStrict), iMemMap,
+                           pVCpu->iem.s.aMemMappings[iMemMap].fAccess,
+                           pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst,
+                           pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond));
+    return VERR_IOM_FF_STATUS_IPE;
+}
+
+
+/**
+ * Helper for IOMR3ProcessForceFlag.
+ *
+ * @returns Merged status code.
+ * @param   rcStrict        Current EM status code.
+ * @param   rcStrictCommit  The IOM I/O or MMIO write commit status to merge
+ *                          with @a rcStrict.
+ * @param   iMemMap         The memory mapping index. For error reporting only.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling
+ *                          thread, for error reporting only.
+ */
+DECLINLINE(VBOXSTRICTRC) iemR3MergeStatus(VBOXSTRICTRC rcStrict, VBOXSTRICTRC rcStrictCommit, unsigned iMemMap, PVMCPUCC pVCpu)
+{
+    /* Simple. */
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RAW_TO_R3))
+        return rcStrictCommit;
+
+    if (RT_LIKELY(rcStrictCommit == VINF_SUCCESS))
+        return rcStrict;
+
+    /* EM scheduling status codes. */
+    if (RT_LIKELY(   rcStrict >= VINF_EM_FIRST
+                  && rcStrict <= VINF_EM_LAST))
+    {
+        if (RT_LIKELY(   rcStrictCommit >= VINF_EM_FIRST
+                      && rcStrictCommit <= VINF_EM_LAST))
+            return rcStrict < rcStrictCommit ? rcStrict : rcStrictCommit;
+    }
+
+    /* Unlikely */
+    return iemR3MergeStatusSlow(rcStrict, rcStrictCommit, iMemMap, pVCpu);
+}
+
+
+/**
+ * Called by force-flag handling code when VMCPU_FF_IEM is set.
+ *
+ * @returns Merge between @a rcStrict and what the commit operation returned.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   rcStrict    The status code returned by ring-0 or raw-mode.
+ */
+VMMR3_INT_DECL(VBOXSTRICTRC) IEMR3ProcessForceFlag(PVM pVM, PVMCPUCC pVCpu, VBOXSTRICTRC rcStrict)
+{
+    /*
+     * Reset the pending commit.
+     */
+    AssertMsg(  (pVCpu->iem.s.aMemMappings[0].fAccess | pVCpu->iem.s.aMemMappings[1].fAccess | pVCpu->iem.s.aMemMappings[2].fAccess)
+              & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND),
+              ("%#x %#x %#x\n",
+               pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_IEM);
+
+    /*
+     * Commit the pending bounce buffers (usually just one).
+     */
+    unsigned cBufs = 0;
+    unsigned iMemMap = RT_ELEMENTS(pVCpu->iem.s.aMemMappings);
+    while (iMemMap-- > 0)
+        if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & (IEM_ACCESS_PENDING_R3_WRITE_1ST | IEM_ACCESS_PENDING_R3_WRITE_2ND))
+        {
+            Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_TYPE_WRITE);
+            Assert(pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_BOUNCE_BUFFERED);
+            Assert(!pVCpu->iem.s.aMemBbMappings[iMemMap].fUnassigned);
+
+            uint16_t const  cbFirst  = pVCpu->iem.s.aMemBbMappings[iMemMap].cbFirst;
+            uint16_t const  cbSecond = pVCpu->iem.s.aMemBbMappings[iMemMap].cbSecond;
+            uint8_t const  *pbBuf    = &pVCpu->iem.s.aBounceBuffers[iMemMap].ab[0];
+
+            if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_1ST)
+            {
+                VBOXSTRICTRC rcStrictCommit1 = PGMPhysWrite(pVM,
+                                                            pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst,
+                                                            pbBuf,
+                                                            cbFirst,
+                                                            PGMACCESSORIGIN_IEM);
+                rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit1, iMemMap, pVCpu);
+                Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysFirst=%RGp LB %#x %Rrc => %Rrc\n",
+                     iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysFirst, cbFirst,
+                     VBOXSTRICTRC_VAL(rcStrictCommit1), VBOXSTRICTRC_VAL(rcStrict)));
+            }
+
+            if (pVCpu->iem.s.aMemMappings[iMemMap].fAccess & IEM_ACCESS_PENDING_R3_WRITE_2ND)
+            {
+                VBOXSTRICTRC rcStrictCommit2 = PGMPhysWrite(pVM,
+                                                            pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond,
+                                                            pbBuf + cbFirst,
+                                                            cbSecond,
+                                                            PGMACCESSORIGIN_IEM);
+                rcStrict = iemR3MergeStatus(rcStrict, rcStrictCommit2, iMemMap, pVCpu);
+                Log(("IEMR3ProcessForceFlag: iMemMap=%u GCPhysSecond=%RGp LB %#x %Rrc => %Rrc\n",
+                     iMemMap, pVCpu->iem.s.aMemBbMappings[iMemMap].GCPhysSecond, cbSecond,
+                     VBOXSTRICTRC_VAL(rcStrictCommit2), VBOXSTRICTRC_VAL(rcStrict)));
+            }
+            cBufs++;
+            pVCpu->iem.s.aMemMappings[iMemMap].fAccess = IEM_ACCESS_INVALID;
+        }
+
+    AssertMsg(cBufs > 0 && cBufs == pVCpu->iem.s.cActiveMappings,
+              ("cBufs=%u cActiveMappings=%u - %#x %#x %#x\n", cBufs, pVCpu->iem.s.cActiveMappings,
+               pVCpu->iem.s.aMemMappings[0].fAccess, pVCpu->iem.s.aMemMappings[1].fAccess, pVCpu->iem.s.aMemMappings[2].fAccess));
+    pVCpu->iem.s.cActiveMappings = 0;
+    return rcStrict;
+}
+
+#endif /* IN_RING3 */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllAImpl.asm b/src/VBox/VMM/VMMAll/IEMAllAImpl.asm
new file mode 100644
index 00000000..0d6e8517
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllAImpl.asm
@@ -0,0 +1,6458 @@
+; $Id: IEMAllAImpl.asm $
+;; @file
+; IEM - Instruction Implementation in Assembly.
+;
+
+;
+; Copyright (C) 2011-2023 Oracle and/or its affiliates.
+;
+; This file is part of VirtualBox base platform packages, as
+; available from https://www.virtualbox.org.
+;
+; This program is free software; you can redistribute it and/or
+; modify it under the terms of the GNU General Public License
+; as published by the Free Software Foundation, in version 3 of the
+; License.
+;
+; This program is distributed in the hope that it will be useful, but
+; WITHOUT ANY WARRANTY; without even the implied warranty of
+; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+; General Public License for more details.
+;
+; You should have received a copy of the GNU General Public License
+; along with this program; if not, see <https://www.gnu.org/licenses>.
+;
+; SPDX-License-Identifier: GPL-3.0-only
+;
+
+
+;*********************************************************************************************************************************
+;*  Header Files                                                                                                                 *
+;*********************************************************************************************************************************
+%include "VBox/asmdefs.mac"
+%include "VBox/err.mac"
+%include "iprt/x86.mac"
+
+
+;*********************************************************************************************************************************
+;*  Defined Constants And Macros                                                                                                 *
+;*********************************************************************************************************************************
+
+;;
+; RET XX / RET wrapper for fastcall.
+;
+%macro RET_FASTCALL 1
+%ifdef RT_ARCH_X86
+ %ifdef RT_OS_WINDOWS
+    ret %1
+ %else
+    ret
+ %endif
+%else
+    ret
+%endif
+%endmacro
+
+;;
+; NAME for fastcall functions.
+;
+;; @todo 'global @fastcall@12' is still broken in yasm and requires dollar
+;         escaping (or whatever the dollar is good for here).  Thus the ugly
+;         prefix argument.
+;
+%define NAME_FASTCALL(a_Name, a_cbArgs, a_Prefix)   NAME(a_Name)
+%ifdef RT_ARCH_X86
+ %ifdef RT_OS_WINDOWS
+  %undef NAME_FASTCALL
+  %define NAME_FASTCALL(a_Name, a_cbArgs, a_Prefix) a_Prefix %+ a_Name %+ @ %+ a_cbArgs
+ %endif
+%endif
+
+;;
+; BEGINPROC for fastcall functions.
+;
+; @param        1       The function name (C).
+; @param        2       The argument size on x86.
+;
+%macro BEGINPROC_FASTCALL 2
+ %ifdef ASM_FORMAT_PE
+  export %1=NAME_FASTCALL(%1,%2,$@)
+ %endif
+ %ifdef __NASM__
+  %ifdef ASM_FORMAT_OMF
+   export NAME(%1) NAME_FASTCALL(%1,%2,$@)
+  %endif
+ %endif
+ %ifndef ASM_FORMAT_BIN
+  global NAME_FASTCALL(%1,%2,$@)
+ %endif
+NAME_FASTCALL(%1,%2,@):
+        IBT_ENDBRxx
+%endmacro
+
+
+;
+; We employ some macro assembly here to hid the calling convention differences.
+;
+%ifdef RT_ARCH_AMD64
+ %macro PROLOGUE_1_ARGS 0
+ %endmacro
+ %macro EPILOGUE_1_ARGS 0
+        ret
+ %endmacro
+ %macro EPILOGUE_1_ARGS_EX 0
+        ret
+ %endmacro
+
+ %macro PROLOGUE_2_ARGS 0
+ %endmacro
+ %macro EPILOGUE_2_ARGS 0
+        ret
+ %endmacro
+ %macro EPILOGUE_2_ARGS_EX 1
+        ret
+ %endmacro
+
+ %macro PROLOGUE_3_ARGS 0
+ %endmacro
+ %macro EPILOGUE_3_ARGS 0
+        ret
+ %endmacro
+ %macro EPILOGUE_3_ARGS_EX 1
+        ret
+ %endmacro
+
+ %macro PROLOGUE_4_ARGS 0
+ %endmacro
+ %macro EPILOGUE_4_ARGS 0
+        ret
+ %endmacro
+ %macro EPILOGUE_4_ARGS_EX 1
+        ret
+ %endmacro
+
+ %ifdef ASM_CALL64_GCC
+  %define A0        rdi
+  %define A0_32     edi
+  %define A0_16     di
+  %define A0_8      dil
+
+  %define A1        rsi
+  %define A1_32     esi
+  %define A1_16     si
+  %define A1_8      sil
+
+  %define A2        rdx
+  %define A2_32     edx
+  %define A2_16     dx
+  %define A2_8      dl
+
+  %define A3        rcx
+  %define A3_32     ecx
+  %define A3_16     cx
+ %endif
+
+ %ifdef ASM_CALL64_MSC
+  %define A0        rcx
+  %define A0_32     ecx
+  %define A0_16     cx
+  %define A0_8      cl
+
+  %define A1        rdx
+  %define A1_32     edx
+  %define A1_16     dx
+  %define A1_8      dl
+
+  %define A2        r8
+  %define A2_32     r8d
+  %define A2_16     r8w
+  %define A2_8      r8b
+
+  %define A3        r9
+  %define A3_32     r9d
+  %define A3_16     r9w
+ %endif
+
+ %define T0         rax
+ %define T0_32      eax
+ %define T0_16      ax
+ %define T0_8       al
+
+ %define T1         r11
+ %define T1_32      r11d
+ %define T1_16      r11w
+ %define T1_8       r11b
+
+ %define T2         r10                 ; only AMD64
+ %define T2_32      r10d
+ %define T2_16      r10w
+ %define T2_8       r10b
+
+%else
+ ; x86
+ %macro PROLOGUE_1_ARGS 0
+        push    edi
+ %endmacro
+ %macro EPILOGUE_1_ARGS 0
+        pop     edi
+        ret     0
+ %endmacro
+ %macro EPILOGUE_1_ARGS_EX 1
+        pop     edi
+        ret     %1
+ %endmacro
+
+ %macro PROLOGUE_2_ARGS 0
+        push    edi
+ %endmacro
+ %macro EPILOGUE_2_ARGS 0
+        pop     edi
+        ret     0
+ %endmacro
+ %macro EPILOGUE_2_ARGS_EX 1
+        pop     edi
+        ret     %1
+ %endmacro
+
+ %macro PROLOGUE_3_ARGS 0
+        push    ebx
+        mov     ebx, [esp + 4 + 4]
+        push    edi
+ %endmacro
+ %macro EPILOGUE_3_ARGS_EX 1
+  %if (%1) < 4
+   %error "With three args, at least 4 bytes must be remove from the stack upon return (32-bit)."
+  %endif
+        pop     edi
+        pop     ebx
+        ret     %1
+ %endmacro
+ %macro EPILOGUE_3_ARGS 0
+        EPILOGUE_3_ARGS_EX 4
+ %endmacro
+
+ %macro PROLOGUE_4_ARGS 0
+        push    ebx
+        push    edi
+        push    esi
+        mov     ebx, [esp + 12 + 4 + 0]
+        mov     esi, [esp + 12 + 4 + 4]
+ %endmacro
+ %macro EPILOGUE_4_ARGS_EX 1
+  %if (%1) < 8
+   %error "With four args, at least 8 bytes must be remove from the stack upon return (32-bit)."
+  %endif
+        pop     esi
+        pop     edi
+        pop     ebx
+        ret     %1
+ %endmacro
+ %macro EPILOGUE_4_ARGS 0
+        EPILOGUE_4_ARGS_EX 8
+ %endmacro
+
+ %define A0         ecx
+ %define A0_32      ecx
+ %define A0_16       cx
+ %define A0_8        cl
+
+ %define A1         edx
+ %define A1_32      edx
+ %define A1_16      dx
+ %define A1_8       dl
+
+ %define A2         ebx
+ %define A2_32      ebx
+ %define A2_16      bx
+ %define A2_8       bl
+
+ %define A3         esi
+ %define A3_32      esi
+ %define A3_16      si
+
+ %define T0         eax
+ %define T0_32      eax
+ %define T0_16      ax
+ %define T0_8       al
+
+ %define T1         edi
+ %define T1_32      edi
+ %define T1_16      di
+%endif
+
+
+;;
+; Load the relevant flags from [%1] if there are undefined flags (%3).
+;
+; @remarks      Clobbers T0, stack. Changes EFLAGS.
+; @param        A2      The register pointing to the flags.
+; @param        1       The parameter (A0..A3) pointing to the eflags.
+; @param        2       The set of modified flags.
+; @param        3       The set of undefined flags.
+;
+%macro IEM_MAYBE_LOAD_FLAGS 3
+ ;%if (%3) != 0
+        pushf                           ; store current flags
+        mov     T0_32, [%1]             ; load the guest flags
+        and     dword [xSP], ~(%2 | %3) ; mask out the modified and undefined flags
+        and     T0_32, (%2 | %3)        ; select the modified and undefined flags.
+        or      [xSP], T0               ; merge guest flags with host flags.
+        popf                            ; load the mixed flags.
+ ;%endif
+%endmacro
+
+;;
+; Update the flag.
+;
+; @remarks  Clobbers T0, T1, stack.
+; @param        1       The register pointing to the EFLAGS.
+; @param        2       The mask of modified flags to save.
+; @param        3       The mask of undefined flags to (maybe) save.
+;
+%macro IEM_SAVE_FLAGS 3
+ %if (%2 | %3) != 0
+        pushf
+        pop     T1
+        mov     T0_32, [%1]             ; flags
+        and     T0_32, ~(%2 | %3)       ; clear the modified & undefined flags.
+        and     T1_32, (%2 | %3)        ; select the modified and undefined flags.
+        or      T0_32, T1_32            ; combine the flags.
+        mov     [%1], T0_32             ; save the flags.
+ %endif
+%endmacro
+
+;;
+; Calculates the new EFLAGS based on the CPU EFLAGS and fixed clear and set bit masks.
+;
+; @remarks  Clobbers T0, T1, stack.
+; @param        1       The register pointing to the EFLAGS.
+; @param        2       The mask of modified flags to save.
+; @param        3       Mask of additional flags to always clear
+; @param        4       Mask of additional flags to always set.
+;
+%macro IEM_SAVE_AND_ADJUST_FLAGS 4
+ %if (%2 | %3 | %4) != 0
+        pushf
+        pop     T1
+        mov     T0_32, [%1]             ; load flags.
+        and     T0_32, ~(%2 | %3)       ; clear the modified and always cleared flags.
+        and     T1_32, (%2)             ; select the modified flags.
+        or      T0_32, T1_32            ; combine the flags.
+  %if (%4) != 0
+        or      T0_32, %4               ; add the always set flags.
+  %endif
+        mov     [%1], T0_32             ; save the result.
+ %endif
+%endmacro
+
+;;
+; Calculates the new EFLAGS based on the CPU EFLAGS (%2), a clear mask (%3),
+; signed input (%4[%5]) and parity index (%6).
+;
+; This is used by MUL and IMUL, where we got result (%4 & %6) in xAX which is
+; also T0. So, we have to use T1 for the EFLAGS calculation and save T0/xAX
+; while we extract the %2 flags from the CPU EFLAGS or use T2 (only AMD64).
+;
+; @remarks  Clobbers T0, T1, stack, %6, EFLAGS.
+; @param        1       The register pointing to the EFLAGS.
+; @param        2       The mask of modified flags to save.
+; @param        3       Mask of additional flags to always clear
+; @param        4       The result register to set SF by.
+; @param        5       The width of the %4 register in bits (8, 16, 32, or 64).
+; @param        6       The (full) register containing the parity table index. Will be modified!
+
+%macro IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF 6
+ %ifdef RT_ARCH_AMD64
+        pushf
+        pop     T2
+ %else
+        push    T0
+        pushf
+        pop     T0
+ %endif
+        mov     T1_32, [%1]             ; load flags.
+        and     T1_32, ~(%2 | %3 | X86_EFL_PF | X86_EFL_SF)  ; clear the modified, always cleared flags and the two flags we calc.
+ %ifdef RT_ARCH_AMD64
+        and     T2_32, (%2)             ; select the modified flags.
+        or      T1_32, T2_32            ; combine the flags.
+ %else
+        and     T0_32, (%2)             ; select the modified flags.
+        or      T1_32, T0_32            ; combine the flags.
+        pop     T0
+ %endif
+
+        ; First calculate SF as it's likely to be refereing to the same register as %6 does.
+        bt      %4, %5 - 1
+        jnc     %%sf_clear
+        or      T1_32, X86_EFL_SF
+ %%sf_clear:
+
+        ; Parity last.
+        and     %6, 0xff
+ %ifdef RT_ARCH_AMD64
+        lea     T2, [NAME(g_afParity) xWrtRIP]
+        or      T1_8, [T2 + %6]
+ %else
+        or      T1_8, [NAME(g_afParity) + %6]
+ %endif
+
+        mov     [%1], T1_32             ; save the result.
+%endmacro
+
+;;
+; Calculates the new EFLAGS using fixed clear and set bit masks.
+;
+; @remarks  Clobbers T0.
+; @param        1       The register pointing to the EFLAGS.
+; @param        2       Mask of additional flags to always clear
+; @param        3       Mask of additional flags to always set.
+;
+%macro IEM_ADJUST_FLAGS 3
+ %if (%2 | %3) != 0
+        mov     T0_32, [%1]             ; Load flags.
+  %if (%2) != 0
+        and     T0_32, ~(%2)            ; Remove the always cleared flags.
+  %endif
+  %if (%3) != 0
+        or      T0_32, %3               ; Add the always set flags.
+  %endif
+        mov     [%1], T0_32             ; Save the result.
+ %endif
+%endmacro
+
+;;
+; Calculates the new EFLAGS using fixed clear and set bit masks.
+;
+; @remarks  Clobbers T0, %4, EFLAGS.
+; @param        1       The register pointing to the EFLAGS.
+; @param        2       Mask of additional flags to always clear
+; @param        3       Mask of additional flags to always set.
+; @param        4       The (full) register containing the parity table index. Will be modified!
+;
+%macro IEM_ADJUST_FLAGS_WITH_PARITY 4
+        mov     T0_32, [%1]                 ; Load flags.
+        and     T0_32, ~(%2 | X86_EFL_PF)   ; Remove PF and the always cleared flags.
+ %if (%3) != 0
+        or      T0_32, %3                   ; Add the always set flags.
+ %endif
+        and     %4, 0xff
+ %ifdef RT_ARCH_AMD64
+        lea     T2, [NAME(g_afParity) xWrtRIP]
+        or      T0_8, [T2 + %4]
+ %else
+        or      T0_8, [NAME(g_afParity) + %4]
+ %endif
+        mov     [%1], T0_32             ; Save the result.
+%endmacro
+
+
+;;
+; Checks that the size expression %1 matches %2 adjusted according to
+; RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK and for 256 entries.
+; @param 1  The jump array size assembly expression.
+; @param 2  The size without accounting for the IBT_ENDBRxx_WITHOUT_NOTRACK instruction.
+;
+%macro IEMCHECK_256_JUMP_ARRAY_SIZE 2
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        dw      (0xffff - %2 - 256*4) + %1  ; will cause warning if entries are too big.
+        dw      (0xffff + %2 + 256*4) - %1  ; will cause warning if entries are too small.
+ %else
+        dw      (0xffff - %2)         + %1  ; will cause warning if entries are too big.
+        dw      (0xffff + %2)         - %1  ; will cause warning if entries are too small.
+ %endif
+%endmacro
+
+
+;*********************************************************************************************************************************
+;*  External Symbols                                                                                                             *
+;*********************************************************************************************************************************
+extern NAME(g_afParity)
+
+
+;;
+; Macro for implementing a binary operator.
+;
+; This will generate code for the 8, 16, 32 and 64 bit accesses with locked
+; variants, except on 32-bit system where the 64-bit accesses requires hand
+; coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the source register operand in A1 and a pointer to eflags in A2.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       Non-zero if there should be a locked version.
+; @param        3       The modified flags.
+; @param        4       The undefined flags.
+;
+%macro IEMIMPL_BIN_OP 4
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      byte [A0], A1_8
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      word [A0], A1_16
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      dword [A0], A1_32
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      qword [A0], A1
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+  %endif ; RT_ARCH_AMD64
+
+ %if %2 != 0 ; locked versions requested?
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 byte [A0], A1_8
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8_locked
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 word [A0], A1_16
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16_locked
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 dword [A0], A1_32
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32_locked
+
+  %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 qword [A0], A1
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
+  %endif ; RT_ARCH_AMD64
+ %endif ; locked
+%endmacro
+
+;            instr,lock, modified-flags,                                                               undefined flags
+IEMIMPL_BIN_OP add,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_BIN_OP adc,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_BIN_OP sub,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_BIN_OP sbb,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_BIN_OP or,   1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF
+IEMIMPL_BIN_OP xor,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF
+IEMIMPL_BIN_OP and,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF
+IEMIMPL_BIN_OP cmp,  0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_BIN_OP test, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF
+
+
+;;
+; Macro for implementing a binary operator, VEX variant with separate input/output.
+;
+; This will generate code for the 32 and 64 bit accesses, except on 32-bit system
+; where the 64-bit accesses requires hand coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the first source register operand in A1, the second source register operand
+; in A2 and a pointer to eflags in A3.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+%macro IEMIMPL_VEX_BIN_OP 3
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A3, %2, %3
+        %1      T0_32, A1_32, A2_32
+        mov     [A0], T0_32
+        IEM_SAVE_FLAGS                 A3, %2, %3
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A3, %2, %3
+        %1      T0, A1, A2
+        mov     [A0], T0
+        IEM_SAVE_FLAGS                 A3, %2, %3
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif ; RT_ARCH_AMD64
+%endmacro
+
+;                 instr,  modified-flags,                                                                undefined-flags
+IEMIMPL_VEX_BIN_OP andn,  (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF),                           (X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_VEX_BIN_OP bextr, (X86_EFL_OF | X86_EFL_ZF | X86_EFL_CF),                                        (X86_EFL_SF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_VEX_BIN_OP bzhi,  (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF),                           (X86_EFL_AF | X86_EFL_PF)
+
+;;
+; Macro for implementing BLSR, BLCMSK and BLSI (fallbacks implemented in C).
+;
+; This will generate code for the 32 and 64 bit accesses, except on 32-bit system
+; where the 64-bit accesses requires hand coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the source register operand in A1 and a pointer to eflags in A2.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+%macro IEMIMPL_VEX_BIN_OP_2 3
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        mov     T0_32, [A0]
+        %1      T0_32, A1_32
+        mov     [A0], T0_32
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        mov     T0, [A0]
+        %1      T0, A1
+        mov     [A0], T0
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif ; RT_ARCH_AMD64
+%endmacro
+
+;                  instr,  modified-flags,                                      undefined-flags
+IEMIMPL_VEX_BIN_OP_2 blsr,   (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_VEX_BIN_OP_2 blsmsk, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_VEX_BIN_OP_2 blsi,   (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF)
+
+
+;;
+; Macro for implementing a binary operator w/o flags, VEX variant with separate input/output.
+;
+; This will generate code for the 32 and 64 bit accesses, except on 32-bit system
+; where the 64-bit accesses requires hand coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the first source register operand in A1, the second source register operand
+; in A2 and a pointer to eflags in A3.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       Fallback instruction if applicable.
+; @param        3       Whether to emit fallback or not.
+;
+%macro IEMIMPL_VEX_BIN_OP_NOEFL 3
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_3_ARGS
+        %1      T0_32, A1_32, A2_32
+        mov     [A0], T0_32
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %if %3
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_fallback, 12
+        PROLOGUE_3_ARGS
+  %ifdef ASM_CALL64_GCC
+        mov     cl, A2_8
+        %2      A1_32, cl
+        mov     [A0], A1_32
+  %else
+        xchg    A2, A0
+        %2      A1_32, cl
+        mov     [A2], A1_32
+  %endif
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32_fallback
+ %endif
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
+        PROLOGUE_3_ARGS
+        %1      T0, A1, A2
+        mov     [A0], T0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+
+ %if %3
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_fallback, 12
+        PROLOGUE_3_ARGS
+  %ifdef ASM_CALL64_GCC
+        mov     cl, A2_8
+        %2      A1, cl
+        mov     [A0], A1_32
+  %else
+        xchg    A2, A0
+        %2      A1, cl
+        mov     [A2], A1_32
+  %endif
+        mov     [A0], A1
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64_fallback
+  %endif
+ %endif ; RT_ARCH_AMD64
+%endmacro
+
+;                           instr, fallback instr, emit fallback
+IEMIMPL_VEX_BIN_OP_NOEFL    sarx,  sar,            1
+IEMIMPL_VEX_BIN_OP_NOEFL    shlx,  shl,            1
+IEMIMPL_VEX_BIN_OP_NOEFL    shrx,  shr,            1
+IEMIMPL_VEX_BIN_OP_NOEFL    pdep,  nop,            0
+IEMIMPL_VEX_BIN_OP_NOEFL    pext,  nop,            0
+
+
+;
+; RORX uses a immediate byte for the shift count, so we only do
+; fallback implementation of that one.
+;
+BEGINPROC_FASTCALL iemAImpl_rorx_u32, 12
+        PROLOGUE_3_ARGS
+ %ifdef ASM_CALL64_GCC
+        mov     cl, A2_8
+        ror     A1_32, cl
+        mov     [A0], A1_32
+ %else
+        xchg    A2, A0
+        ror     A1_32, cl
+        mov     [A2], A1_32
+ %endif
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_rorx_u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_rorx_u64, 12
+        PROLOGUE_3_ARGS
+ %ifdef ASM_CALL64_GCC
+        mov     cl, A2_8
+        ror     A1, cl
+        mov     [A0], A1_32
+ %else
+        xchg    A2, A0
+        ror     A1, cl
+        mov     [A2], A1_32
+ %endif
+        mov     [A0], A1
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_rorx_u64
+ %endif ; RT_ARCH_AMD64
+
+
+;
+; MULX
+;
+BEGINPROC_FASTCALL iemAImpl_mulx_u32, 16
+        PROLOGUE_4_ARGS
+%ifdef ASM_CALL64_GCC
+        ; A2_32 is EDX - prefect
+        mulx    T0_32, T1_32, A3_32
+        mov     [A1], T1_32 ; Low value first, as we should return the high part if same destination registers.
+        mov     [A0], T0_32
+%else
+        ; A1 is xDX - must switch A1 and A2, so EDX=uSrc1
+        xchg    A1, A2
+        mulx    T0_32, T1_32, A3_32
+        mov     [A2], T1_32 ; Low value first, as we should return the high part if same destination registers.
+        mov     [A0], T0_32
+%endif
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_mulx_u32
+
+
+BEGINPROC_FASTCALL iemAImpl_mulx_u32_fallback, 16
+        PROLOGUE_4_ARGS
+%ifdef ASM_CALL64_GCC
+        ; A2_32 is EDX, T0_32 is EAX
+        mov     eax, A3_32
+        mul     A2_32
+        mov     [A1], eax ; Low value first, as we should return the high part if same destination registers.
+        mov     [A0], edx
+%else
+        ; A1 is xDX, T0_32 is EAX - must switch A1 and A2, so EDX=uSrc1
+        xchg    A1, A2
+        mov     eax, A3_32
+        mul     A2_32
+        mov     [A2], eax ; Low value first, as we should return the high part if same destination registers.
+        mov     [A0], edx
+%endif
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_mulx_u32_fallback
+
+%ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_mulx_u64, 16
+        PROLOGUE_4_ARGS
+%ifdef ASM_CALL64_GCC
+        ; A2 is RDX - prefect
+        mulx    T0, T1, A3
+        mov     [A1], T1 ; Low value first, as we should return the high part if same destination registers.
+        mov     [A0], T0
+%else
+        ; A1 is xDX - must switch A1 and A2, so RDX=uSrc1
+        xchg    A1, A2
+        mulx    T0, T1, A3
+        mov     [A2], T1 ; Low value first, as we should return the high part if same destination registers.
+        mov     [A0], T0
+%endif
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_mulx_u64
+
+
+BEGINPROC_FASTCALL iemAImpl_mulx_u64_fallback, 16
+        PROLOGUE_4_ARGS
+%ifdef ASM_CALL64_GCC
+        ; A2 is RDX, T0 is RAX
+        mov     rax, A3
+        mul     A2
+        mov     [A1], rax ; Low value first, as we should return the high part if same destination registers.
+        mov     [A0], rdx
+%else
+        ; A1 is xDX, T0 is RAX - must switch A1 and A2, so RDX=uSrc1
+        xchg    A1, A2
+        mov     rax, A3
+        mul     A2
+        mov     [A2], rax ; Low value first, as we should return the high part if same destination registers.
+        mov     [A0], rdx
+%endif
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_mulx_u64_fallback
+
+%endif
+
+
+;;
+; Macro for implementing a bit operator.
+;
+; This will generate code for the 16, 32 and 64 bit accesses with locked
+; variants, except on 32-bit system where the 64-bit accesses requires hand
+; coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the source register operand in A1 and a pointer to eflags in A2.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       Non-zero if there should be a locked version.
+; @param        3       The modified flags.
+; @param        4       The undefined flags.
+;
+%macro IEMIMPL_BIT_OP 4
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      word [A0], A1_16
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      dword [A0], A1_32
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        %1      qword [A0], A1
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+  %endif ; RT_ARCH_AMD64
+
+ %if %2 != 0 ; locked versions requested?
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 word [A0], A1_16
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16_locked
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 dword [A0], A1_32
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32_locked
+
+  %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        lock %1 qword [A0], A1
+        IEM_SAVE_FLAGS                 A2, %3, %4
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
+  %endif ; RT_ARCH_AMD64
+ %endif ; locked
+%endmacro
+IEMIMPL_BIT_OP bt,  0, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_BIT_OP btc, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_BIT_OP bts, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_BIT_OP btr, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+
+;;
+; Macro for implementing a bit search operator.
+;
+; This will generate code for the 16, 32 and 64 bit accesses, except on 32-bit
+; system where the 64-bit accesses requires hand coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the source register operand in A1 and a pointer to eflags in A2.
+;
+; In the ZF case the destination register is 'undefined', however it seems that
+; both AMD and Intel just leaves it as is.  The undefined EFLAGS differs between
+; AMD and Intel and accoridng to https://www.sandpile.org/x86/flags.htm between
+; Intel microarchitectures.  We only implement 'intel' and 'amd' variation with
+; the behaviour of more recent CPUs (Intel 10980X and AMD 3990X).
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+; @param        4       Non-zero if destination isn't written when ZF=1.  Zero if always written.
+;
+%macro IEMIMPL_BIT_OP2 4
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        %1      T0_16, A1_16
+%if %4 != 0
+        jz      .unchanged_dst
+%endif
+        mov     [A0], T0_16
+.unchanged_dst:
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ _intel, 12
+        PROLOGUE_3_ARGS
+        %1      T1_16, A1_16
+%if %4 != 0
+        jz      .unchanged_dst
+%endif
+        mov     [A0], T1_16
+        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
+        EPILOGUE_3_ARGS
+.unchanged_dst:
+        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16_intel
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ _amd, 12
+        PROLOGUE_3_ARGS
+        %1      T0_16, A1_16
+%if %4 != 0
+        jz      .unchanged_dst
+%endif
+        mov     [A0], T0_16
+.unchanged_dst:
+        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16_amd
+
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        %1      T0_32, A1_32
+%if %4 != 0
+        jz      .unchanged_dst
+%endif
+        mov     [A0], T0_32
+.unchanged_dst:
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ _intel, 12
+        PROLOGUE_3_ARGS
+        %1      T1_32, A1_32
+%if %4 != 0
+        jz      .unchanged_dst
+%endif
+        mov     [A0], T1_32
+        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
+        EPILOGUE_3_ARGS
+.unchanged_dst:
+        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32_intel
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ _amd, 12
+        PROLOGUE_3_ARGS
+        %1      T0_32, A1_32
+%if %4 != 0
+        jz      .unchanged_dst
+%endif
+        mov     [A0], T0_32
+.unchanged_dst:
+        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32_amd
+
+
+ %ifdef RT_ARCH_AMD64
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        %1      T0, A1
+%if %4 != 0
+        jz      .unchanged_dst
+%endif
+        mov     [A0], T0
+.unchanged_dst:
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ _intel, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        %1      T1, A1
+%if %4 != 0
+        jz      .unchanged_dst
+%endif
+        mov     [A0], T1
+        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
+        EPILOGUE_3_ARGS
+.unchanged_dst:
+        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64_intel
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ _amd, 16
+        PROLOGUE_3_ARGS
+        %1      T0, A1
+%if %4 != 0
+        jz      .unchanged_dst
+%endif
+        mov     [A0], T0
+.unchanged_dst:
+        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64_amd
+
+ %endif ; RT_ARCH_AMD64
+%endmacro
+
+IEMIMPL_BIT_OP2 bsf,   (X86_EFL_ZF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 1
+IEMIMPL_BIT_OP2 bsr,   (X86_EFL_ZF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 1
+IEMIMPL_BIT_OP2 tzcnt, (X86_EFL_ZF | X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF), 0
+IEMIMPL_BIT_OP2 lzcnt, (X86_EFL_ZF | X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF), 0
+
+
+;;
+; Macro for implementing POPCNT.
+;
+; This will generate code for the 16, 32 and 64 bit accesses, except on 32-bit
+; system where the 64-bit accesses requires hand coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the source register operand in A1 and a pointer to eflags in A2.
+;
+; ASSUMES Intel and AMD set EFLAGS the same way.
+;
+; ASSUMES the instruction does not support memory destination.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+%macro IEMIMPL_BIT_OP3 3
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        %1      T0_16, A1_16
+        mov     [A0], T0_16
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        %1      T0_32, A1_32
+        mov     [A0], T0_32
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        %1      T0, A1
+        mov     [A0], T0
+        IEM_SAVE_FLAGS                 A2, %2, %3
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif ; RT_ARCH_AMD64
+%endmacro
+IEMIMPL_BIT_OP3 popcnt, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF), 0
+
+
+;
+; IMUL is also a similar but yet different case (no lock, no mem dst).
+; The rDX:rAX variant of imul is handled together with mul further down.
+;
+BEGINCODE
+; @param        1       EFLAGS that are modified.
+; @param        2       Undefined EFLAGS.
+; @param        3       Function suffix.
+; @param        4       EFLAGS variation: 0 for native, 1 for intel (ignored),
+;                       2 for AMD (set AF, clear PF, ZF and SF).
+%macro IEMIMPL_IMUL_TWO 4
+BEGINPROC_FASTCALL iemAImpl_imul_two_u16 %+ %3, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2
+        imul    A1_16, word [A0]
+        mov     [A0], A1_16
+ %if %4 != 1
+        IEM_SAVE_FLAGS                          A2, %1, %2
+ %else
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1_16, 16, A1
+ %endif
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_imul_two_u16 %+ %3
+
+BEGINPROC_FASTCALL iemAImpl_imul_two_u32 %+ %3, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2
+        imul    A1_32, dword [A0]
+        mov     [A0], A1_32
+ %if %4 != 1
+        IEM_SAVE_FLAGS                          A2, %1, %2
+ %else
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1_32, 32, A1
+ %endif
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_imul_two_u32 %+ %3
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_imul_two_u64 %+ %3, 16
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2
+        imul    A1, qword [A0]
+        mov     [A0], A1
+ %if %4 != 1
+        IEM_SAVE_FLAGS                          A2, %1, %2
+ %else
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1, 64, A1
+ %endif
+        EPILOGUE_3_ARGS_EX 8
+ENDPROC iemAImpl_imul_two_u64 %+ %3
+ %endif ; RT_ARCH_AMD64
+%endmacro
+IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF,       , 0
+IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, 0,                                                 _intel, 1
+IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, 0,                                                 _amd,   2
+
+
+;
+; XCHG for memory operands.  This implies locking.  No flag changes.
+;
+; Each function takes two arguments, first the pointer to the memory,
+; then the pointer to the register.  They all return void.
+;
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_xchg_u8_locked, 8
+        PROLOGUE_2_ARGS
+        mov     T0_8, [A1]
+        xchg    [A0], T0_8
+        mov     [A1], T0_8
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u8_locked
+
+BEGINPROC_FASTCALL iemAImpl_xchg_u16_locked, 8
+        PROLOGUE_2_ARGS
+        mov     T0_16, [A1]
+        xchg    [A0], T0_16
+        mov     [A1], T0_16
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u16_locked
+
+BEGINPROC_FASTCALL iemAImpl_xchg_u32_locked, 8
+        PROLOGUE_2_ARGS
+        mov     T0_32, [A1]
+        xchg    [A0], T0_32
+        mov     [A1], T0_32
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u32_locked
+
+%ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_xchg_u64_locked, 8
+        PROLOGUE_2_ARGS
+        mov     T0, [A1]
+        xchg    [A0], T0
+        mov     [A1], T0
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u64_locked
+%endif
+
+; Unlocked variants for fDisregardLock mode.
+
+BEGINPROC_FASTCALL iemAImpl_xchg_u8_unlocked, 8
+        PROLOGUE_2_ARGS
+        mov     T0_8, [A1]
+        mov     T1_8, [A0]
+        mov     [A0], T0_8
+        mov     [A1], T1_8
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u8_unlocked
+
+BEGINPROC_FASTCALL iemAImpl_xchg_u16_unlocked, 8
+        PROLOGUE_2_ARGS
+        mov     T0_16, [A1]
+        mov     T1_16, [A0]
+        mov     [A0], T0_16
+        mov     [A1], T1_16
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u16_unlocked
+
+BEGINPROC_FASTCALL iemAImpl_xchg_u32_unlocked, 8
+        PROLOGUE_2_ARGS
+        mov     T0_32, [A1]
+        mov     T1_32, [A0]
+        mov     [A0], T0_32
+        mov     [A1], T1_32
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u32_unlocked
+
+%ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_xchg_u64_unlocked, 8
+        PROLOGUE_2_ARGS
+        mov     T0, [A1]
+        mov     T1, [A0]
+        mov     [A0], T0
+        mov     [A1], T1
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_xchg_u64_unlocked
+%endif
+
+
+;
+; XADD for memory operands.
+;
+; Each function takes three arguments, first the pointer to the
+; memory/register, then the pointer to the register, and finally a pointer to
+; eflags.  They all return void.
+;
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_xadd_u8, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_8, [A1]
+        xadd    [A0], T0_8
+        mov     [A1], T0_8
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u8
+
+BEGINPROC_FASTCALL iemAImpl_xadd_u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_16, [A1]
+        xadd    [A0], T0_16
+        mov     [A1], T0_16
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u16
+
+BEGINPROC_FASTCALL iemAImpl_xadd_u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_32, [A1]
+        xadd    [A0], T0_32
+        mov     [A1], T0_32
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u32
+
+%ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_xadd_u64, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0, [A1]
+        xadd    [A0], T0
+        mov     [A1], T0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u64
+%endif ; RT_ARCH_AMD64
+
+BEGINPROC_FASTCALL iemAImpl_xadd_u8_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_8, [A1]
+        lock xadd [A0], T0_8
+        mov     [A1], T0_8
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u8_locked
+
+BEGINPROC_FASTCALL iemAImpl_xadd_u16_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_16, [A1]
+        lock xadd [A0], T0_16
+        mov     [A1], T0_16
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u16_locked
+
+BEGINPROC_FASTCALL iemAImpl_xadd_u32_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0_32, [A1]
+        lock xadd [A0], T0_32
+        mov     [A1], T0_32
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u32_locked
+
+%ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_xadd_u64_locked, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        mov     T0, [A1]
+        lock xadd [A0], T0
+        mov     [A1], T0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_xadd_u64_locked
+%endif ; RT_ARCH_AMD64
+
+
+;
+; CMPXCHG8B.
+;
+; These are tricky register wise, so the code is duplicated for each calling
+; convention.
+;
+; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to!
+;
+; C-proto:
+; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
+;                                             uint32_t *pEFlags));
+;
+; Note! Identical to iemAImpl_cmpxchg16b.
+;
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_cmpxchg8b, 16
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_MSC
+        push    rbx
+
+        mov     r11, rdx                ; pu64EaxEdx (is also T1)
+        mov     r10, rcx                ; pu64Dst
+
+        mov     ebx, [r8]
+        mov     ecx, [r8 + 4]
+        IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     eax, [r11]
+        mov     edx, [r11 + 4]
+
+        cmpxchg8b [r10]
+
+        mov     [r11], eax
+        mov     [r11 + 4], edx
+        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+ %else
+        push    rbx
+
+        mov     r10, rcx                ; pEFlags
+        mov     r11, rdx                ; pu64EbxEcx (is also T1)
+
+        mov     ebx, [r11]
+        mov     ecx, [r11 + 4]
+        IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     eax, [rsi]
+        mov     edx, [rsi + 4]
+
+        cmpxchg8b [rdi]
+
+        mov     [rsi], eax
+        mov     [rsi + 4], edx
+        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+
+ %endif
+%else
+        push    esi
+        push    edi
+        push    ebx
+        push    ebp
+
+        mov     edi, ecx                ; pu64Dst
+        mov     esi, edx                ; pu64EaxEdx
+        mov     ecx, [esp + 16 + 4 + 0] ; pu64EbxEcx
+        mov     ebp, [esp + 16 + 4 + 4] ; pEFlags
+
+        mov     ebx, [ecx]
+        mov     ecx, [ecx + 4]
+        IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF), 0  ; clobbers T0 (eax)
+        mov     eax, [esi]
+        mov     edx, [esi + 4]
+
+        cmpxchg8b [edi]
+
+        mov     [esi], eax
+        mov     [esi + 4], edx
+        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, edi)
+
+        pop     ebp
+        pop     ebx
+        pop     edi
+        pop     esi
+        ret     8
+%endif
+ENDPROC iemAImpl_cmpxchg8b
+
+BEGINPROC_FASTCALL iemAImpl_cmpxchg8b_locked, 16
+%ifdef RT_ARCH_AMD64
+ %ifdef ASM_CALL64_MSC
+        push    rbx
+
+        mov     r11, rdx                ; pu64EaxEdx (is also T1)
+        mov     r10, rcx                ; pu64Dst
+
+        mov     ebx, [r8]
+        mov     ecx, [r8 + 4]
+        IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     eax, [r11]
+        mov     edx, [r11 + 4]
+
+        lock cmpxchg8b [r10]
+
+        mov     [r11], eax
+        mov     [r11 + 4], edx
+        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+ %else
+        push    rbx
+
+        mov     r10, rcx                ; pEFlags
+        mov     r11, rdx                ; pu64EbxEcx (is also T1)
+
+        mov     ebx, [r11]
+        mov     ecx, [r11 + 4]
+        IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     eax, [rsi]
+        mov     edx, [rsi + 4]
+
+        lock cmpxchg8b [rdi]
+
+        mov     [rsi], eax
+        mov     [rsi + 4], edx
+        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+
+ %endif
+%else
+        push    esi
+        push    edi
+        push    ebx
+        push    ebp
+
+        mov     edi, ecx                ; pu64Dst
+        mov     esi, edx                ; pu64EaxEdx
+        mov     ecx, [esp + 16 + 4 + 0] ; pu64EbxEcx
+        mov     ebp, [esp + 16 + 4 + 4] ; pEFlags
+
+        mov     ebx, [ecx]
+        mov     ecx, [ecx + 4]
+        IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF), 0  ; clobbers T0 (eax)
+        mov     eax, [esi]
+        mov     edx, [esi + 4]
+
+        lock cmpxchg8b [edi]
+
+        mov     [esi], eax
+        mov     [esi + 4], edx
+        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, edi)
+
+        pop     ebp
+        pop     ebx
+        pop     edi
+        pop     esi
+        ret     8
+%endif
+ENDPROC iemAImpl_cmpxchg8b_locked
+
+%ifdef RT_ARCH_AMD64
+
+;
+; CMPXCHG16B.
+;
+; These are tricky register wise, so the code is duplicated for each calling
+; convention.
+;
+; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to!
+;
+; C-proto:
+; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu1284RaxRdx, PRTUINT128U pu128RbxRcx,
+;                                              uint32_t *pEFlags));
+;
+; Note! Identical to iemAImpl_cmpxchg8b.
+;
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_cmpxchg16b, 16
+ %ifdef ASM_CALL64_MSC
+        push    rbx
+
+        mov     r11, rdx                ; pu64RaxRdx (is also T1)
+        mov     r10, rcx                ; pu64Dst
+
+        mov     rbx, [r8]
+        mov     rcx, [r8 + 8]
+        IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     rax, [r11]
+        mov     rdx, [r11 + 8]
+
+        cmpxchg16b [r10]
+
+        mov     [r11], rax
+        mov     [r11 + 8], rdx
+        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+ %else
+        push    rbx
+
+        mov     r10, rcx                ; pEFlags
+        mov     r11, rdx                ; pu64RbxRcx (is also T1)
+
+        mov     rbx, [r11]
+        mov     rcx, [r11 + 8]
+        IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     rax, [rsi]
+        mov     rdx, [rsi + 8]
+
+        cmpxchg16b [rdi]
+
+        mov     [rsi], rax
+        mov     [rsi + 8], rdx
+        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+
+ %endif
+ENDPROC iemAImpl_cmpxchg16b
+
+BEGINPROC_FASTCALL iemAImpl_cmpxchg16b_locked, 16
+ %ifdef ASM_CALL64_MSC
+        push    rbx
+
+        mov     r11, rdx                ; pu64RaxRdx (is also T1)
+        mov     r10, rcx                ; pu64Dst
+
+        mov     rbx, [r8]
+        mov     rcx, [r8 + 8]
+        IEM_MAYBE_LOAD_FLAGS r9, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     rax, [r11]
+        mov     rdx, [r11 + 8]
+
+        lock cmpxchg16b [r10]
+
+        mov     [r11], rax
+        mov     [r11 + 8], rdx
+        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+ %else
+        push    rbx
+
+        mov     r10, rcx                ; pEFlags
+        mov     r11, rdx                ; pu64RbxRcx (is also T1)
+
+        mov     rbx, [r11]
+        mov     rcx, [r11 + 8]
+        IEM_MAYBE_LOAD_FLAGS r10, (X86_EFL_ZF), 0 ; clobbers T0 (eax)
+        mov     rax, [rsi]
+        mov     rdx, [rsi + 8]
+
+        lock cmpxchg16b [rdi]
+
+        mov     [rsi], rax
+        mov     [rsi + 8], rdx
+        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+
+        pop     rbx
+        ret
+
+ %endif
+ENDPROC iemAImpl_cmpxchg16b_locked
+
+%endif ; RT_ARCH_AMD64
+
+
+;
+; CMPXCHG.
+;
+; WARNING! This code make ASSUMPTIONS about which registers T1 and T0 are mapped to!
+;
+; C-proto:
+; IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg,(uintX_t *puXDst, uintX_t puEax, uintX_t uReg, uint32_t *pEFlags));
+;
+BEGINCODE
+%macro IEMIMPL_CMPXCHG 2
+BEGINPROC_FASTCALL iemAImpl_cmpxchg_u8 %+ %2, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
+        mov     al, [A1]
+        %1 cmpxchg [A0], A2_8
+        mov     [A1], al
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cmpxchg_u8 %+ %2
+
+BEGINPROC_FASTCALL iemAImpl_cmpxchg_u16 %+ %2, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
+        mov     ax, [A1]
+        %1 cmpxchg [A0], A2_16
+        mov     [A1], ax
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cmpxchg_u16 %+ %2
+
+BEGINPROC_FASTCALL iemAImpl_cmpxchg_u32 %+ %2, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
+        mov     eax, [A1]
+        %1 cmpxchg [A0], A2_32
+        mov     [A1], eax
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cmpxchg_u32 %+ %2
+
+BEGINPROC_FASTCALL iemAImpl_cmpxchg_u64 %+ %2, 16
+%ifdef RT_ARCH_AMD64
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0 (eax)
+        mov     rax, [A1]
+        %1 cmpxchg [A0], A2
+        mov     [A1], rax
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        EPILOGUE_4_ARGS
+%else
+        ;
+        ; Must use cmpxchg8b here. See also iemAImpl_cmpxchg8b.
+        ;
+        push    esi
+        push    edi
+        push    ebx
+        push    ebp
+
+        mov     edi, ecx                ; pu64Dst
+        mov     esi, edx                ; pu64Rax
+        mov     ecx, [esp + 16 + 4 + 0] ; pu64Reg - Note! Pointer on 32-bit hosts!
+        mov     ebp, [esp + 16 + 4 + 4] ; pEFlags
+
+        mov     ebx, [ecx]
+        mov     ecx, [ecx + 4]
+        IEM_MAYBE_LOAD_FLAGS ebp, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0  ; clobbers T0 (eax)
+        mov     eax, [esi]
+        mov     edx, [esi + 4]
+
+        lock cmpxchg8b [edi]
+
+        ; cmpxchg8b doesn't set CF, PF, AF, SF and OF, so we have to do that.
+        jz      .cmpxchg8b_not_equal
+        cmp     eax, eax                ; just set the other flags.
+.store:
+        mov     [esi], eax
+        mov     [esi + 4], edx
+        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, edi)
+
+        pop     ebp
+        pop     ebx
+        pop     edi
+        pop     esi
+        ret     8
+
+.cmpxchg8b_not_equal:
+        cmp     [esi + 4], edx          ;; @todo FIXME - verify 64-bit compare implementation
+        jne     .store
+        cmp     [esi], eax
+        jmp     .store
+
+%endif
+ENDPROC iemAImpl_cmpxchg_u64 %+ %2
+%endmacro ; IEMIMPL_CMPXCHG
+
+IEMIMPL_CMPXCHG , ,
+IEMIMPL_CMPXCHG lock, _locked
+
+;;
+; Macro for implementing a unary operator.
+;
+; This will generate code for the 8, 16, 32 and 64 bit accesses with locked
+; variants, except on 32-bit system where the 64-bit accesses requires hand
+; coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the source register operand in A1 and a pointer to eflags in A2.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+%macro IEMIMPL_UNARY_OP 3
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        %1      byte [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8_locked, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        lock %1 byte [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8_locked
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        %1      word [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        lock %1 word [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16_locked
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        %1      dword [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        lock %1 dword [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32_locked
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        %1      qword [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 8
+        PROLOGUE_2_ARGS
+        IEM_MAYBE_LOAD_FLAGS A1, %2, %3
+        lock %1 qword [A0]
+        IEM_SAVE_FLAGS       A1, %2, %3
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
+ %endif ; RT_ARCH_AMD64
+
+%endmacro
+
+IEMIMPL_UNARY_OP inc, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF), 0
+IEMIMPL_UNARY_OP dec, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF), 0
+IEMIMPL_UNARY_OP neg, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+IEMIMPL_UNARY_OP not, 0, 0
+
+
+;
+; BSWAP. No flag changes.
+;
+; Each function takes one argument, pointer to the value to bswap
+; (input/output). They all return void.
+;
+BEGINPROC_FASTCALL iemAImpl_bswap_u16, 4
+        PROLOGUE_1_ARGS
+        mov     T0_32, [A0]             ; just in case any of the upper bits are used.
+        db 66h
+        bswap   T0_32
+        mov     [A0], T0_32
+        EPILOGUE_1_ARGS
+ENDPROC iemAImpl_bswap_u16
+
+BEGINPROC_FASTCALL iemAImpl_bswap_u32, 4
+        PROLOGUE_1_ARGS
+        mov     T0_32, [A0]
+        bswap   T0_32
+        mov     [A0], T0_32
+        EPILOGUE_1_ARGS
+ENDPROC iemAImpl_bswap_u32
+
+BEGINPROC_FASTCALL iemAImpl_bswap_u64, 4
+%ifdef RT_ARCH_AMD64
+        PROLOGUE_1_ARGS
+        mov     T0, [A0]
+        bswap   T0
+        mov     [A0], T0
+        EPILOGUE_1_ARGS
+%else
+        PROLOGUE_1_ARGS
+        mov     T0, [A0]
+        mov     T1, [A0 + 4]
+        bswap   T0
+        bswap   T1
+        mov     [A0 + 4], T0
+        mov     [A0], T1
+        EPILOGUE_1_ARGS
+%endif
+ENDPROC iemAImpl_bswap_u64
+
+
+;;
+; Macro for implementing a shift operation.
+;
+; This will generate code for the 8, 16, 32 and 64 bit accesses, except on
+; 32-bit system where the 64-bit accesses requires hand coding.
+;
+; All the functions takes a pointer to the destination memory operand in A0,
+; the shift count in A1 and a pointer to eflags in A2.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+; Makes ASSUMPTIONS about A0, A1 and A2 assignments.
+;
+; @note the _intel and _amd variants are implemented in C.
+;
+%macro IEMIMPL_SHIFT_OP 3
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+ %ifdef ASM_CALL64_GCC
+        mov     cl, A1_8
+        %1      byte [A0], cl
+ %else
+        xchg    A1, A0
+        %1      byte [A1], cl
+ %endif
+        IEM_SAVE_FLAGS       A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+ %ifdef ASM_CALL64_GCC
+        mov     cl, A1_8
+        %1      word [A0], cl
+ %else
+        xchg    A1, A0
+        %1      word [A1], cl
+ %endif
+        IEM_SAVE_FLAGS       A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+ %ifdef ASM_CALL64_GCC
+        mov     cl, A1_8
+        %1      dword [A0], cl
+ %else
+        xchg    A1, A0
+        %1      dword [A1], cl
+ %endif
+        IEM_SAVE_FLAGS       A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+  %ifdef ASM_CALL64_GCC
+        mov     cl, A1_8
+        %1      qword [A0], cl
+  %else
+        xchg    A1, A0
+        %1      qword [A1], cl
+  %endif
+        IEM_SAVE_FLAGS       A2, %2, %3
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif ; RT_ARCH_AMD64
+
+%endmacro
+
+IEMIMPL_SHIFT_OP rol, (X86_EFL_OF | X86_EFL_CF), 0
+IEMIMPL_SHIFT_OP ror, (X86_EFL_OF | X86_EFL_CF), 0
+IEMIMPL_SHIFT_OP rcl, (X86_EFL_OF | X86_EFL_CF), 0
+IEMIMPL_SHIFT_OP rcr, (X86_EFL_OF | X86_EFL_CF), 0
+IEMIMPL_SHIFT_OP shl, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+IEMIMPL_SHIFT_OP shr, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+IEMIMPL_SHIFT_OP sar, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+
+
+;;
+; Macro for implementing a double precision shift operation.
+;
+; This will generate code for the 16, 32 and 64 bit accesses, except on
+; 32-bit system where the 64-bit accesses requires hand coding.
+;
+; The functions takes the destination operand (r/m) in A0, the source (reg) in
+; A1, the shift count in A2 and a pointer to the eflags variable/register in A3.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+;
+; Makes ASSUMPTIONS about A0, A1, A2 and A3 assignments.
+;
+; @note the _intel and _amd variants are implemented in C.
+;
+%macro IEMIMPL_SHIFT_DBL_OP 3
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+ %ifdef ASM_CALL64_GCC
+        xchg    A3, A2
+        %1      [A0], A1_16, cl
+        xchg    A3, A2
+ %else
+        xchg    A0, A2
+        %1      [A2], A1_16, cl
+ %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+ %ifdef ASM_CALL64_GCC
+        xchg    A3, A2
+        %1      [A0], A1_32, cl
+        xchg    A3, A2
+ %else
+        xchg    A0, A2
+        %1      [A2], A1_32, cl
+ %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32
+
+ %ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 20
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+ %ifdef ASM_CALL64_GCC
+        xchg    A3, A2
+        %1      [A0], A1, cl
+        xchg    A3, A2
+ %else
+        xchg    A0, A2
+        %1      [A2], A1, cl
+ %endif
+        IEM_SAVE_FLAGS       A3, %2, %3
+        EPILOGUE_4_ARGS_EX 12
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif ; RT_ARCH_AMD64
+
+%endmacro
+
+IEMIMPL_SHIFT_DBL_OP shld, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+IEMIMPL_SHIFT_DBL_OP shrd, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+
+
+;;
+; Macro for implementing a multiplication operations.
+;
+; This will generate code for the 8, 16, 32 and 64 bit accesses, except on
+; 32-bit system where the 64-bit accesses requires hand coding.
+;
+; The 8-bit function only operates on AX, so it takes no DX pointer.  The other
+; functions takes a pointer to rAX in A0, rDX in A1, the operand in A2 and a
+; pointer to eflags in A3.
+;
+; The functions all return 0 so the caller can be used for div/idiv as well as
+; for the mul/imul implementation.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+; @param        4       Name suffix.
+; @param        5       EFLAGS behaviour: 0 for native, 1 for intel and 2 for AMD.
+;
+; Makes ASSUMPTIONS about A0, A1, A2, A3, T0 and T1 assignments.
+;
+%macro IEMIMPL_MUL_OP 5
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8 %+ %4, 12
+        PROLOGUE_3_ARGS
+        IEM_MAYBE_LOAD_FLAGS                     A2, %2, %3
+        mov     al, [A0]
+        %1      A1_8
+        mov     [A0], ax
+ %if %5 != 1
+        IEM_SAVE_FLAGS                           A2, %2, %3
+ %else
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A2, %2, X86_EFL_AF | X86_EFL_ZF, ax, 8, xAX
+ %endif
+        xor     eax, eax
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u8 %+ %4
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ %4, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3
+        mov     ax, [A0]
+ %ifdef ASM_CALL64_GCC
+        %1      A2_16
+        mov     [A0], ax
+        mov     [A1], dx
+ %else
+        mov     T1, A1
+        %1      A2_16
+        mov     [A0], ax
+        mov     [T1], dx
+ %endif
+ %if %5 != 1
+        IEM_SAVE_FLAGS                           A3, %2, %3
+ %else
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, ax, 16, xAX
+ %endif
+        xor     eax, eax
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u16 %+ %4
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ %4, 16
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3
+        mov     eax, [A0]
+ %ifdef ASM_CALL64_GCC
+        %1      A2_32
+        mov     [A0], eax
+        mov     [A1], edx
+ %else
+        mov     T1, A1
+        %1      A2_32
+        mov     [A0], eax
+        mov     [T1], edx
+ %endif
+ %if %5 != 1
+        IEM_SAVE_FLAGS                           A3, %2, %3
+ %else
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, eax, 32, xAX
+ %endif
+        xor     eax, eax
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u32 %+ %4
+
+ %ifdef RT_ARCH_AMD64 ; The 32-bit host version lives in IEMAllAImplC.cpp.
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ %4, 20
+        PROLOGUE_4_ARGS
+        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3
+        mov     rax, [A0]
+  %ifdef ASM_CALL64_GCC
+        %1      A2
+        mov     [A0], rax
+        mov     [A1], rdx
+  %else
+        mov     T1, A1
+        %1      A2
+        mov     [A0], rax
+        mov     [T1], rdx
+  %endif
+  %if %5 != 1
+        IEM_SAVE_FLAGS                           A3, %2, %3
+  %else
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, rax, 64, xAX
+  %endif
+        xor     eax, eax
+        EPILOGUE_4_ARGS_EX 12
+ENDPROC iemAImpl_ %+ %1 %+ _u64 %+ %4
+ %endif ; !RT_ARCH_AMD64
+
+%endmacro
+
+IEMIMPL_MUL_OP mul,  (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF),       , 0
+IEMIMPL_MUL_OP mul,  (X86_EFL_OF | X86_EFL_CF), 0,                                                   _intel, 1
+IEMIMPL_MUL_OP mul,  (X86_EFL_OF | X86_EFL_CF), 0,                                                   _amd,   2
+IEMIMPL_MUL_OP imul, (X86_EFL_OF | X86_EFL_CF), (X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF),       , 0
+IEMIMPL_MUL_OP imul, (X86_EFL_OF | X86_EFL_CF), 0,                                                   _intel, 1
+IEMIMPL_MUL_OP imul, (X86_EFL_OF | X86_EFL_CF), 0,                                                   _amd,   2
+
+
+BEGINCODE
+;;
+; Worker function for negating a 32-bit number in T1:T0
+; @uses None (T0,T1)
+BEGINPROC   iemAImpl_negate_T0_T1_u32
+        push    0
+        push    0
+        xchg    T0_32, [xSP]
+        xchg    T1_32, [xSP + xCB]
+        sub     T0_32, [xSP]
+        sbb     T1_32, [xSP + xCB]
+        add     xSP, xCB*2
+        ret
+ENDPROC     iemAImpl_negate_T0_T1_u32
+
+%ifdef RT_ARCH_AMD64
+;;
+; Worker function for negating a 64-bit number in T1:T0
+; @uses None (T0,T1)
+BEGINPROC   iemAImpl_negate_T0_T1_u64
+        push    0
+        push    0
+        xchg    T0, [xSP]
+        xchg    T1, [xSP + xCB]
+        sub     T0, [xSP]
+        sbb     T1, [xSP + xCB]
+        add     xSP, xCB*2
+        ret
+ENDPROC     iemAImpl_negate_T0_T1_u64
+%endif
+
+
+;;
+; Macro for implementing a division operations.
+;
+; This will generate code for the 8, 16, 32 and 64 bit accesses, except on
+; 32-bit system where the 64-bit accesses requires hand coding.
+;
+; The 8-bit function only operates on AX, so it takes no DX pointer.  The other
+; functions takes a pointer to rAX in A0, rDX in A1, the operand in A2 and a
+; pointer to eflags in A3.
+;
+; The functions all return 0 on success and -1 if a divide error should be
+; raised by the caller.
+;
+; @param        1       The instruction mnemonic.
+; @param        2       The modified flags.
+; @param        3       The undefined flags.
+; @param        4       1 if signed, 0 if unsigned.
+; @param        5       Function suffix.
+; @param        6       EFLAGS variation: 0 for native, 1 for intel (ignored),
+;                       2 for AMD (set AF, clear PF, ZF and SF).
+;
+; Makes ASSUMPTIONS about A0, A1, A2, A3, T0 and T1 assignments.
+;
+%macro IEMIMPL_DIV_OP 6
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8 %+ %5, 12
+        PROLOGUE_3_ARGS
+
+        ; div by chainsaw check.
+        test    A1_8, A1_8
+        jz      .div_zero
+
+        ; Overflow check - unsigned division is simple to verify, haven't
+        ; found a simple way to check signed division yet unfortunately.
+ %if %4 == 0
+        cmp     [A0 + 1], A1_8
+        jae     .div_overflow
+ %else
+        mov     T0_16, [A0]             ; T0 = dividend
+        mov     T1, A1                  ; T1 = saved divisor (because of missing T1_8 in 32-bit)
+        test    A1_8, A1_8
+        js      .divisor_negative
+        test    T0_16, T0_16
+        jns     .both_positive
+        neg     T0_16
+.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
+        push    T0                      ; Start off like unsigned below.
+        shr     T0_16, 7
+        cmp     T0_8, A1_8
+        pop     T0
+        jb      .div_no_overflow
+        ja      .div_overflow
+        and     T0_8, 0x7f              ; Special case for covering (divisor - 1).
+        cmp     T0_8, A1_8
+        jae     .div_overflow
+        jmp     .div_no_overflow
+
+.divisor_negative:
+        neg     A1_8
+        test    T0_16, T0_16
+        jns     .one_of_each
+        neg     T0_16
+.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
+        shr     T0_16, 7
+        cmp     T0_8, A1_8
+        jae     .div_overflow
+.div_no_overflow:
+        mov     A1, T1                  ; restore divisor
+ %endif
+
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+        mov     ax, [A0]
+        %1      A1_8
+        mov     [A0], ax
+ %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF.
+        IEM_ADJUST_FLAGS    A2, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
+ %else
+        IEM_SAVE_FLAGS      A2, %2, %3
+ %endif
+        xor     eax, eax
+
+.return:
+        EPILOGUE_3_ARGS
+
+.div_zero:
+.div_overflow:
+        mov     eax, -1
+        jmp     .return
+ENDPROC iemAImpl_ %+ %1 %+ _u8 %+ %5
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ %5, 16
+        PROLOGUE_4_ARGS
+
+        ; div by chainsaw check.
+        test    A2_16, A2_16
+        jz      .div_zero
+
+        ; Overflow check - unsigned division is simple to verify, haven't
+        ; found a simple way to check signed division yet unfortunately.
+ %if %4 == 0
+        cmp     [A1], A2_16
+        jae     .div_overflow
+ %else
+        mov     T0_16, [A1]
+        shl     T0_32, 16
+        mov     T0_16, [A0]              ; T0 = dividend
+        mov     T1, A2                   ; T1 = divisor
+        test    T1_16, T1_16
+        js      .divisor_negative
+        test    T0_32, T0_32
+        jns     .both_positive
+        neg     T0_32
+.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
+        push    T0                      ; Start off like unsigned below.
+        shr     T0_32, 15
+        cmp     T0_16, T1_16
+        pop     T0
+        jb      .div_no_overflow
+        ja      .div_overflow
+        and     T0_16, 0x7fff           ; Special case for covering (divisor - 1).
+        cmp     T0_16, T1_16
+        jae     .div_overflow
+        jmp     .div_no_overflow
+
+.divisor_negative:
+        neg     T1_16
+        test    T0_32, T0_32
+        jns     .one_of_each
+        neg     T0_32
+.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
+        shr     T0_32, 15
+        cmp     T0_16, T1_16
+        jae     .div_overflow
+.div_no_overflow:
+ %endif
+
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+ %ifdef ASM_CALL64_GCC
+        mov     T1, A2
+        mov     ax, [A0]
+        mov     dx, [A1]
+        %1      T1_16
+        mov     [A0], ax
+        mov     [A1], dx
+ %else
+        mov     T1, A1
+        mov     ax, [A0]
+        mov     dx, [T1]
+        %1      A2_16
+        mov     [A0], ax
+        mov     [T1], dx
+ %endif
+ %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF.
+        IEM_ADJUST_FLAGS    A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
+ %else
+        IEM_SAVE_FLAGS      A3, %2, %3
+ %endif
+        xor     eax, eax
+
+.return:
+        EPILOGUE_4_ARGS
+
+.div_zero:
+.div_overflow:
+        mov     eax, -1
+        jmp     .return
+ENDPROC iemAImpl_ %+ %1 %+ _u16 %+ %5
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ %5, 16
+        PROLOGUE_4_ARGS
+
+        ; div by chainsaw check.
+        test    A2_32, A2_32
+        jz      .div_zero
+
+        ; Overflow check - unsigned division is simple to verify, haven't
+        ; found a simple way to check signed division yet unfortunately.
+ %if %4 == 0
+        cmp     [A1], A2_32
+        jae     .div_overflow
+ %else
+        push    A2                      ; save A2 so we modify it (we out of regs on x86).
+        mov     T0_32, [A0]             ; T0 = dividend low
+        mov     T1_32, [A1]             ; T1 = dividend high
+        test    A2_32, A2_32
+        js      .divisor_negative
+        test    T1_32, T1_32
+        jns     .both_positive
+        call    NAME(iemAImpl_negate_T0_T1_u32)
+.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
+        push    T0                      ; Start off like unsigned below.
+        shl     T1_32, 1
+        shr     T0_32, 31
+        or      T1_32, T0_32
+        cmp     T1_32, A2_32
+        pop     T0
+        jb      .div_no_overflow
+        ja      .div_overflow
+        and     T0_32, 0x7fffffff       ; Special case for covering (divisor - 1).
+        cmp     T0_32, A2_32
+        jae     .div_overflow
+        jmp     .div_no_overflow
+
+.divisor_negative:
+        neg     A2_32
+        test    T1_32, T1_32
+        jns     .one_of_each
+        call    NAME(iemAImpl_negate_T0_T1_u32)
+.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
+        shl     T1_32, 1
+        shr     T0_32, 31
+        or      T1_32, T0_32
+        cmp     T1_32, A2_32
+        jae     .div_overflow
+.div_no_overflow:
+        pop     A2
+ %endif
+
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        mov     eax, [A0]
+ %ifdef ASM_CALL64_GCC
+        mov     T1, A2
+        mov     eax, [A0]
+        mov     edx, [A1]
+        %1      T1_32
+        mov     [A0], eax
+        mov     [A1], edx
+ %else
+        mov     T1, A1
+        mov     eax, [A0]
+        mov     edx, [T1]
+        %1      A2_32
+        mov     [A0], eax
+        mov     [T1], edx
+ %endif
+ %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF.
+        IEM_ADJUST_FLAGS    A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
+ %else
+        IEM_SAVE_FLAGS      A3, %2, %3
+ %endif
+        xor     eax, eax
+
+.return:
+        EPILOGUE_4_ARGS
+
+.div_overflow:
+ %if %4 != 0
+        pop     A2
+ %endif
+.div_zero:
+        mov     eax, -1
+        jmp     .return
+ENDPROC iemAImpl_ %+ %1 %+ _u32 %+ %5
+
+ %ifdef RT_ARCH_AMD64 ; The 32-bit host version lives in IEMAllAImplC.cpp.
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ %5, 20
+        PROLOGUE_4_ARGS
+
+        test    A2, A2
+        jz      .div_zero
+  %if %4 == 0
+        cmp     [A1], A2
+        jae     .div_overflow
+  %else
+        push    A2                      ; save A2 so we modify it (we out of regs on x86).
+        mov     T0, [A0]                ; T0 = dividend low
+        mov     T1, [A1]                ; T1 = dividend high
+        test    A2, A2
+        js      .divisor_negative
+        test    T1, T1
+        jns     .both_positive
+        call    NAME(iemAImpl_negate_T0_T1_u64)
+.one_of_each:                           ; OK range is 2^(result-with - 1) + (divisor - 1).
+        push    T0                      ; Start off like unsigned below.
+        shl     T1, 1
+        shr     T0, 63
+        or      T1, T0
+        cmp     T1, A2
+        pop     T0
+        jb      .div_no_overflow
+        ja      .div_overflow
+        mov     T1, 0x7fffffffffffffff
+        and     T0, T1                  ; Special case for covering (divisor - 1).
+        cmp     T0, A2
+        jae     .div_overflow
+        jmp     .div_no_overflow
+
+.divisor_negative:
+        neg     A2
+        test    T1, T1
+        jns     .one_of_each
+        call    NAME(iemAImpl_negate_T0_T1_u64)
+.both_positive:                         ; Same as unsigned shifted by sign indicator bit.
+        shl     T1, 1
+        shr     T0, 63
+        or      T1, T0
+        cmp     T1, A2
+        jae     .div_overflow
+.div_no_overflow:
+        pop     A2
+  %endif
+
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        mov     rax, [A0]
+  %ifdef ASM_CALL64_GCC
+        mov     T1, A2
+        mov     rax, [A0]
+        mov     rdx, [A1]
+        %1      T1
+        mov     [A0], rax
+        mov     [A1], rdx
+  %else
+        mov     T1, A1
+        mov     rax, [A0]
+        mov     rdx, [T1]
+        %1      A2
+        mov     [A0], rax
+        mov     [T1], rdx
+  %endif
+  %if %6 == 2 ; AMD64 3990X: Set AF and clear PF, ZF and SF.
+        IEM_ADJUST_FLAGS    A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
+  %else
+        IEM_SAVE_FLAGS      A3, %2, %3
+  %endif
+        xor     eax, eax
+
+.return:
+        EPILOGUE_4_ARGS_EX 12
+
+.div_overflow:
+  %if %4 != 0
+        pop     A2
+  %endif
+.div_zero:
+        mov     eax, -1
+        jmp     .return
+ENDPROC iemAImpl_ %+ %1 %+ _u64 %+ %5
+ %endif ; !RT_ARCH_AMD64
+
+%endmacro
+
+IEMIMPL_DIV_OP div,  0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,       , 0
+IEMIMPL_DIV_OP div,  0, 0,                                                                             0, _intel, 1
+IEMIMPL_DIV_OP div,  0, 0,                                                                             0, _amd,   2
+IEMIMPL_DIV_OP idiv, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 1,       , 0
+IEMIMPL_DIV_OP idiv, 0, 0,                                                                             1, _intel, 1
+IEMIMPL_DIV_OP idiv, 0, 0,                                                                             1, _amd,   2
+
+
+;;
+; Macro for implementing memory fence operation.
+;
+; No return value, no operands or anything.
+;
+; @param        1      The instruction.
+;
+%macro IEMIMPL_MEM_FENCE 1
+BEGINCODE
+BEGINPROC_FASTCALL iemAImpl_ %+ %1, 0
+        %1
+        ret
+ENDPROC iemAImpl_ %+ %1
+%endmacro
+
+IEMIMPL_MEM_FENCE lfence
+IEMIMPL_MEM_FENCE sfence
+IEMIMPL_MEM_FENCE mfence
+
+;;
+; Alternative for non-SSE2 host.
+;
+BEGINPROC_FASTCALL iemAImpl_alt_mem_fence, 0
+        push    xAX
+        xchg    xAX, [xSP]
+        add     xSP, xCB
+        ret
+ENDPROC iemAImpl_alt_mem_fence
+
+
+;;
+; Initialize the FPU for the actual instruction being emulated, this means
+; loading parts of the guest's control word and status word.
+;
+; @uses     24 bytes of stack. T0, T1
+; @param    1       Expression giving the address of the FXSTATE of the guest.
+;
+%macro FPU_LD_FXSTATE_FCW_AND_SAFE_FSW 1
+        fnstenv [xSP]
+
+        ; FCW - for exception, precision and rounding control.
+        movzx   T0, word [%1 + X86FXSTATE.FCW]
+        and     T0, X86_FCW_MASK_ALL | X86_FCW_PC_MASK | X86_FCW_RC_MASK
+        mov     [xSP + X86FSTENV32P.FCW], T0_16
+
+        ; FSW - for undefined C0, C1, C2, and C3.
+        movzx   T1, word [%1 + X86FXSTATE.FSW]
+        and     T1, X86_FSW_C_MASK
+        movzx   T0, word [xSP + X86FSTENV32P.FSW]
+        and     T0, X86_FSW_TOP_MASK
+        or      T0, T1
+        mov     [xSP + X86FSTENV32P.FSW], T0_16
+
+        fldenv  [xSP]
+%endmacro
+
+
+;;
+; Initialize the FPU for the actual instruction being emulated, this means
+; loading parts of the guest's control word, status word, and update the
+; tag word for the top register if it's empty.
+;
+; ASSUMES actual TOP=7
+;
+; @uses     24 bytes of stack.  T0, T1
+; @param    1       Expression giving the address of the FXSTATE of the guest.
+;
+%macro FPU_LD_FXSTATE_FCW_AND_SAFE_FSW_AND_FTW_0 1
+        fnstenv [xSP]
+
+        ; FCW - for exception, precision and rounding control.
+        movzx   T0_32, word [%1 + X86FXSTATE.FCW]
+        and     T0_32, X86_FCW_MASK_ALL | X86_FCW_PC_MASK | X86_FCW_RC_MASK
+        mov     [xSP + X86FSTENV32P.FCW], T0_16
+
+        ; FSW - for undefined C0, C1, C2, and C3.
+        movzx   T1_32, word [%1 + X86FXSTATE.FSW]
+        and     T1_32, X86_FSW_C_MASK
+        movzx   T0_32, word [xSP + X86FSTENV32P.FSW]
+        and     T0_32, X86_FSW_TOP_MASK
+        or      T0_32, T1_32
+        mov     [xSP + X86FSTENV32P.FSW], T0_16
+
+        ; FTW - Only for ST0 (in/out).
+        movzx   T1_32, word [%1 + X86FXSTATE.FSW]
+        shr     T1_32, X86_FSW_TOP_SHIFT
+        and     T1_32, X86_FSW_TOP_SMASK
+        bt      [%1 + X86FXSTATE.FTW], T1_16     ; Empty if FTW bit is clear.  Fixed register order.
+        jc      %%st0_not_empty
+        or      word [xSP + X86FSTENV32P.FTW], 0c000h ; TOP=7, so set TAG(7)=3
+%%st0_not_empty:
+
+        fldenv  [xSP]
+%endmacro
+
+
+;;
+; Need to move this as well somewhere better?
+;
+struc IEMFPURESULT
+    .r80Result  resw 5
+    .FSW        resw 1
+endstruc
+
+
+;;
+; Need to move this as well somewhere better?
+;
+struc IEMFPURESULTTWO
+    .r80Result1 resw 5
+    .FSW        resw 1
+    .r80Result2 resw 5
+endstruc
+
+
+;
+;---------------------- 16-bit signed integer operations ----------------------
+;
+
+
+;;
+; Converts a 16-bit floating point value to a 80-bit one (fpu register).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 16-bit floating point value to convert.
+;
+BEGINPROC_FASTCALL iemAImpl_fild_r80_from_i16, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fild    word [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fild_r80_from_i16
+
+
+;;
+; Store a 80-bit floating point value (register) as a 16-bit signed integer (memory).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 16-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i16, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fistp   word [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fist_r80_to_i16
+
+
+;;
+; Store a 80-bit floating point value (register) as a 16-bit signed integer
+; (memory) with truncation.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 16-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i16, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fisttp  word [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fistt_r80_to_i16
+
+
+;;
+; FPU instruction working on one 80-bit and one 16-bit signed integer value.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 16-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_I16 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i16, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      word [A3]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i16
+%endmacro
+
+IEMIMPL_FPU_R80_BY_I16 fiadd
+IEMIMPL_FPU_R80_BY_I16 fimul
+IEMIMPL_FPU_R80_BY_I16 fisub
+IEMIMPL_FPU_R80_BY_I16 fisubr
+IEMIMPL_FPU_R80_BY_I16 fidiv
+IEMIMPL_FPU_R80_BY_I16 fidivr
+
+
+;;
+; FPU instruction working on one 80-bit and one 16-bit signed integer value,
+; only returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to store the output FSW.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 64-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_I16_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i16, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      word [A3]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i16
+%endmacro
+
+IEMIMPL_FPU_R80_BY_I16_FSW ficom
+
+
+
+;
+;---------------------- 32-bit signed integer operations ----------------------
+;
+
+
+;;
+; Converts a 32-bit floating point value to a 80-bit one (fpu register).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 32-bit floating point value to convert.
+;
+BEGINPROC_FASTCALL iemAImpl_fild_r80_from_i32, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fild    dword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fild_r80_from_i32
+
+
+;;
+; Store a 80-bit floating point value (register) as a 32-bit signed integer (memory).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 32-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fistp   dword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fist_r80_to_i32
+
+
+;;
+; Store a 80-bit floating point value (register) as a 32-bit signed integer
+; (memory) with truncation.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 32-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fisttp  dword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fistt_r80_to_i32
+
+
+;;
+; FPU instruction working on one 80-bit and one 32-bit signed integer value.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 32-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_I32 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      dword [A3]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i32
+%endmacro
+
+IEMIMPL_FPU_R80_BY_I32 fiadd
+IEMIMPL_FPU_R80_BY_I32 fimul
+IEMIMPL_FPU_R80_BY_I32 fisub
+IEMIMPL_FPU_R80_BY_I32 fisubr
+IEMIMPL_FPU_R80_BY_I32 fidiv
+IEMIMPL_FPU_R80_BY_I32 fidivr
+
+
+;;
+; FPU instruction working on one 80-bit and one 32-bit signed integer value,
+; only returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to store the output FSW.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 64-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_I32_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_i32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      dword [A3]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_i32
+%endmacro
+
+IEMIMPL_FPU_R80_BY_I32_FSW ficom
+
+
+
+;
+;---------------------- 64-bit signed integer operations ----------------------
+;
+
+
+;;
+; Converts a 64-bit floating point value to a 80-bit one (fpu register).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 64-bit floating point value to convert.
+;
+BEGINPROC_FASTCALL iemAImpl_fild_r80_from_i64, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fild    qword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fild_r80_from_i64
+
+
+;;
+; Store a 80-bit floating point value (register) as a 64-bit signed integer (memory).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 64-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fist_r80_to_i64, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fistp   qword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fist_r80_to_i64
+
+
+;;
+; Store a 80-bit floating point value (register) as a 64-bit signed integer
+; (memory) with truncation.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 64-bit signed integer value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fistt_r80_to_i64, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fisttp  qword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fistt_r80_to_i64
+
+
+
+;
+;---------------------- 32-bit floating point operations ----------------------
+;
+
+;;
+; Converts a 32-bit floating point value to a 80-bit one (fpu register).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 32-bit floating point value to convert.
+;
+BEGINPROC_FASTCALL iemAImpl_fld_r80_from_r32, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fld     dword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fld_r80_from_r32
+
+
+;;
+; Store a 80-bit floating point value (register) as a 32-bit one (memory).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 32-bit value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fst     dword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fst_r80_to_r32
+
+
+;;
+; FPU instruction working on one 80-bit and one 32-bit floating point value.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 32-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R32 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      dword [A3]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r32
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R32 fadd
+IEMIMPL_FPU_R80_BY_R32 fmul
+IEMIMPL_FPU_R80_BY_R32 fsub
+IEMIMPL_FPU_R80_BY_R32 fsubr
+IEMIMPL_FPU_R80_BY_R32 fdiv
+IEMIMPL_FPU_R80_BY_R32 fdivr
+
+
+;;
+; FPU instruction working on one 80-bit and one 32-bit floating point value,
+; only returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to store the output FSW.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 64-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R32_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r32, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      dword [A3]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r32
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R32_FSW fcom
+
+
+
+;
+;---------------------- 64-bit floating point operations ----------------------
+;
+
+;;
+; Converts a 64-bit floating point value to a 80-bit one (fpu register).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 64-bit floating point value to convert.
+;
+BEGINPROC_FASTCALL iemAImpl_fld_r80_from_r64, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fld     qword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fld_r80_from_r64
+
+
+;;
+; Store a 80-bit floating point value (register) as a 64-bit one (memory).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 64-bit value.
+; @param    A3      Pointer to the 80-bit value.
+;
+BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r64, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fst     qword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fst_r80_to_r64
+
+
+;;
+; FPU instruction working on one 80-bit and one 64-bit floating point value.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 64-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R64 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r64, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      qword [A3]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r64
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R64 fadd
+IEMIMPL_FPU_R80_BY_R64 fmul
+IEMIMPL_FPU_R80_BY_R64 fsub
+IEMIMPL_FPU_R80_BY_R64 fsubr
+IEMIMPL_FPU_R80_BY_R64 fdiv
+IEMIMPL_FPU_R80_BY_R64 fdivr
+
+;;
+; FPU instruction working on one 80-bit and one 64-bit floating point value,
+; only returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to store the output FSW.
+; @param    A2      Pointer to the 80-bit value.
+; @param    A3      Pointer to the 64-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R64_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r64, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      qword [A3]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r64
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R64_FSW fcom
+
+
+
+;
+;---------------------- 80-bit floating point operations ----------------------
+;
+
+;;
+; Loads a 80-bit floating point register value from memory.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit floating point value to load.
+;
+BEGINPROC_FASTCALL iemAImpl_fld_r80_from_r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fld     tword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fld_r80_from_r80
+
+
+;;
+; Store a 80-bit floating point register to memory
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 80-bit value.
+; @param    A3      Pointer to the 80-bit register value.
+;
+BEGINPROC_FASTCALL iemAImpl_fst_r80_to_r80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fstp    tword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fst_r80_to_r80
+
+
+;;
+; Loads an 80-bit floating point register value in BCD format from memory.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit BCD value to load.
+;
+BEGINPROC_FASTCALL iemAImpl_fld_r80_from_d80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fbld    tword [A2]
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_fld_r80_from_d80
+
+
+;;
+; Store a 80-bit floating point register to memory as BCD
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Where to return the output FSW.
+; @param    A2      Where to store the 80-bit BCD value.
+; @param    A3      Pointer to the 80-bit register value.
+;
+BEGINPROC_FASTCALL iemAImpl_fst_r80_to_d80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        fbstp   tword [A2]
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_fst_r80_to_d80
+
+
+;;
+; FPU instruction working on two 80-bit floating point values.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the first 80-bit value (ST0)
+; @param    A3      Pointer to the second 80-bit value (STn).
+;
+%macro IEMIMPL_FPU_R80_BY_R80 2
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      %2
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R80 fadd,   {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fmul,   {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fsub,   {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fsubr,  {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fdiv,   {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fdivr,  {st0, st1}
+IEMIMPL_FPU_R80_BY_R80 fprem,  {}
+IEMIMPL_FPU_R80_BY_R80 fprem1, {}
+IEMIMPL_FPU_R80_BY_R80 fscale, {}
+
+
+;;
+; FPU instruction working on two 80-bit floating point values, ST1 and ST0,
+; storing the result in ST1 and popping the stack.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the first 80-bit value (ST1).
+; @param    A3      Pointer to the second 80-bit value (ST0).
+;
+%macro IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        fld     tword [A3]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fpatan
+IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fyl2x
+IEMIMPL_FPU_R80_BY_R80_ST1_ST0_POP fyl2xp1
+
+
+;;
+; FPU instruction working on two 80-bit floating point values, only
+; returning FSW.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a uint16_t for the resulting FSW.
+; @param    A2      Pointer to the first 80-bit value.
+; @param    A3      Pointer to the second 80-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R80_FSW 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      st0, st1
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R80_FSW fcom
+IEMIMPL_FPU_R80_BY_R80_FSW fucom
+
+
+;;
+; FPU instruction working on two 80-bit floating point values,
+; returning FSW and EFLAGS (eax).
+;
+; @param    1       The instruction
+;
+; @returns  EFLAGS in EAX.
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a uint16_t for the resulting FSW.
+; @param    A2      Pointer to the first 80-bit value.
+; @param    A3      Pointer to the second 80-bit value.
+;
+%macro IEMIMPL_FPU_R80_BY_R80_EFL 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_by_r80, 16
+        PROLOGUE_4_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A3]
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1      st1
+
+        fnstsw  word  [A1]
+        pushf
+        pop     xAX
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_by_r80
+%endmacro
+
+IEMIMPL_FPU_R80_BY_R80_EFL fcomi
+IEMIMPL_FPU_R80_BY_R80_EFL fucomi
+
+
+;;
+; FPU instruction working on one 80-bit floating point value.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+; @param    A2      Pointer to the 80-bit value.
+;
+%macro IEMIMPL_FPU_R80 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80
+%endmacro
+
+IEMIMPL_FPU_R80 fchs
+IEMIMPL_FPU_R80 fabs
+IEMIMPL_FPU_R80 f2xm1
+IEMIMPL_FPU_R80 fsqrt
+IEMIMPL_FPU_R80 frndint
+IEMIMPL_FPU_R80 fsin
+IEMIMPL_FPU_R80 fcos
+
+
+;;
+; FPU instruction working on one 80-bit floating point value, only
+; returning FSW.
+;
+; @param    1       The instruction
+; @param    2       Non-zero to also restore FTW.
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a uint16_t for the resulting FSW.
+; @param    A2      Pointer to the 80-bit value.
+;
+%macro IEMIMPL_FPU_R80_FSW 2
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+%if %2 != 0
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW_AND_FTW_0 A0
+%else
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+%endif
+        %1
+
+        fnstsw  word  [A1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80
+%endmacro
+
+IEMIMPL_FPU_R80_FSW ftst, 0
+IEMIMPL_FPU_R80_FSW fxam, 1 ; No #IS or any other FP exceptions.
+
+
+
+;;
+; FPU instruction loading a 80-bit floating point constant.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULT for the output.
+;
+%macro IEMIMPL_FPU_R80_CONST 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1, 8
+        PROLOGUE_2_ARGS
+        sub     xSP, 20h
+
+        fninit
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1
+
+        fnstsw  word  [A1 + IEMFPURESULT.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULT.r80Result]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+
+%endmacro
+
+IEMIMPL_FPU_R80_CONST fld1
+IEMIMPL_FPU_R80_CONST fldl2t
+IEMIMPL_FPU_R80_CONST fldl2e
+IEMIMPL_FPU_R80_CONST fldpi
+IEMIMPL_FPU_R80_CONST fldlg2
+IEMIMPL_FPU_R80_CONST fldln2
+IEMIMPL_FPU_R80_CONST fldz
+
+
+;;
+; FPU instruction working on one 80-bit floating point value, outputing two.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to a IEMFPURESULTTWO for the output.
+; @param    A2      Pointer to the 80-bit value.
+;
+%macro IEMIMPL_FPU_R80_R80 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _r80_r80, 12
+        PROLOGUE_3_ARGS
+        sub     xSP, 20h
+
+        fninit
+        fld     tword [A2]
+        FPU_LD_FXSTATE_FCW_AND_SAFE_FSW A0
+        %1
+
+        fnstsw  word  [A1 + IEMFPURESULTTWO.FSW]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULTTWO.r80Result2]
+        fnclex
+        fstp    tword [A1 + IEMFPURESULTTWO.r80Result1]
+
+        fninit
+        add     xSP, 20h
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _r80_r80
+%endmacro
+
+IEMIMPL_FPU_R80_R80 fptan
+IEMIMPL_FPU_R80_R80 fxtract
+IEMIMPL_FPU_R80_R80 fsincos
+
+
+
+
+;---------------------- SSE and MMX Operations ----------------------
+
+;; @todo what do we need to do for MMX?
+%macro IEMIMPL_MMX_PROLOGUE 0
+%endmacro
+%macro IEMIMPL_MMX_EPILOGUE 0
+%endmacro
+
+;; @todo what do we need to do for SSE?
+%macro IEMIMPL_SSE_PROLOGUE 0
+%endmacro
+%macro IEMIMPL_SSE_EPILOGUE 0
+%endmacro
+
+;; @todo what do we need to do for AVX?
+%macro IEMIMPL_AVX_PROLOGUE 0
+%endmacro
+%macro IEMIMPL_AVX_EPILOGUE 0
+%endmacro
+
+
+;;
+; Media instruction working on two full sized registers.
+;
+; @param    1       The instruction
+; @param    2       Whether there is an MMX variant (1) or not (0).
+;
+; @param    A0      FPU context (fxsave).
+; @param    A1      Pointer to the first media register size operand (input/output).
+; @param    A2      Pointer to the second media register size operand (input).
+;
+%macro IEMIMPL_MEDIA_F2 2
+%if %2 != 0
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm0, [A1]
+        movq    mm1, [A2]
+        %1      mm0, mm1
+        movq    [A1], mm0
+
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+%endif
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu   xmm0, [A1]
+        movdqu   xmm1, [A2]
+        %1       xmm0, xmm1
+        movdqu   [A1], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_F2 pshufb,  1
+IEMIMPL_MEDIA_F2 pand,    1
+IEMIMPL_MEDIA_F2 pandn,   1
+IEMIMPL_MEDIA_F2 por,     1
+IEMIMPL_MEDIA_F2 pxor,    1
+IEMIMPL_MEDIA_F2 pcmpeqb, 1
+IEMIMPL_MEDIA_F2 pcmpeqw, 1
+IEMIMPL_MEDIA_F2 pcmpeqd, 1
+IEMIMPL_MEDIA_F2 pcmpeqq, 0
+IEMIMPL_MEDIA_F2 pcmpgtb, 1
+IEMIMPL_MEDIA_F2 pcmpgtw, 1
+IEMIMPL_MEDIA_F2 pcmpgtd, 1
+IEMIMPL_MEDIA_F2 pcmpgtq, 0
+IEMIMPL_MEDIA_F2 paddb,   1
+IEMIMPL_MEDIA_F2 paddw,   1
+IEMIMPL_MEDIA_F2 paddd,   1
+IEMIMPL_MEDIA_F2 paddq,   1
+IEMIMPL_MEDIA_F2 paddsb,  1
+IEMIMPL_MEDIA_F2 paddsw,  1
+IEMIMPL_MEDIA_F2 paddusb, 1
+IEMIMPL_MEDIA_F2 paddusw, 1
+IEMIMPL_MEDIA_F2 psubb,   1
+IEMIMPL_MEDIA_F2 psubw,   1
+IEMIMPL_MEDIA_F2 psubd,   1
+IEMIMPL_MEDIA_F2 psubq,   1
+IEMIMPL_MEDIA_F2 psubsb,  1
+IEMIMPL_MEDIA_F2 psubsw,  1
+IEMIMPL_MEDIA_F2 psubusb, 1
+IEMIMPL_MEDIA_F2 psubusw, 1
+IEMIMPL_MEDIA_F2 pmullw,  1
+IEMIMPL_MEDIA_F2 pmulld,  0
+IEMIMPL_MEDIA_F2 pmulhw,  1
+IEMIMPL_MEDIA_F2 pmaddwd, 1
+IEMIMPL_MEDIA_F2 pminub,  1
+IEMIMPL_MEDIA_F2 pminuw,  0
+IEMIMPL_MEDIA_F2 pminud,  0
+IEMIMPL_MEDIA_F2 pminsb,  0
+IEMIMPL_MEDIA_F2 pminsw,  1
+IEMIMPL_MEDIA_F2 pminsd,  0
+IEMIMPL_MEDIA_F2 pmaxub,  1
+IEMIMPL_MEDIA_F2 pmaxuw,  0
+IEMIMPL_MEDIA_F2 pmaxud,  0
+IEMIMPL_MEDIA_F2 pmaxsb,  0
+IEMIMPL_MEDIA_F2 pmaxsw,  1
+IEMIMPL_MEDIA_F2 pmaxsd,  0
+IEMIMPL_MEDIA_F2 pabsb,   1
+IEMIMPL_MEDIA_F2 pabsw,   1
+IEMIMPL_MEDIA_F2 pabsd,   1
+IEMIMPL_MEDIA_F2 psignb,  1
+IEMIMPL_MEDIA_F2 psignw,  1
+IEMIMPL_MEDIA_F2 psignd,  1
+IEMIMPL_MEDIA_F2 phaddw,  1
+IEMIMPL_MEDIA_F2 phaddd,  1
+IEMIMPL_MEDIA_F2 phsubw,  1
+IEMIMPL_MEDIA_F2 phsubd,  1
+IEMIMPL_MEDIA_F2 phaddsw, 1
+IEMIMPL_MEDIA_F2 phsubsw, 1
+IEMIMPL_MEDIA_F2 pmaddubsw, 1
+IEMIMPL_MEDIA_F2 pmulhrsw,  1
+IEMIMPL_MEDIA_F2 pmuludq,   1
+
+
+;;
+; Media instruction working on two full sized registers, but no FXSAVE state argument.
+;
+; @param    1       The instruction
+; @param    2       Whether there is an MMX variant (1) or not (0).
+;
+; @param    A0      Pointer to the first media register size operand (input/output).
+; @param    A1      Pointer to the second media register size operand (input).
+;
+%macro IEMIMPL_MEDIA_OPT_F2 2
+%if %2 != 0
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 8
+        PROLOGUE_2_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm0, [A0]
+        movq    mm1, [A1]
+        %1      mm0, mm1
+        movq    [A0], mm0
+
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+%endif
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 8
+        PROLOGUE_2_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu   xmm0, [A0]
+        movdqu   xmm1, [A1]
+        %1       xmm0, xmm1
+        movdqu   [A0], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_OPT_F2 packsswb, 1
+IEMIMPL_MEDIA_OPT_F2 packssdw, 1
+IEMIMPL_MEDIA_OPT_F2 packuswb, 1
+IEMIMPL_MEDIA_OPT_F2 packusdw, 0
+IEMIMPL_MEDIA_OPT_F2 psllw,    1
+IEMIMPL_MEDIA_OPT_F2 pslld,    1
+IEMIMPL_MEDIA_OPT_F2 psllq,    1
+IEMIMPL_MEDIA_OPT_F2 psrlw,    1
+IEMIMPL_MEDIA_OPT_F2 psrld,    1
+IEMIMPL_MEDIA_OPT_F2 psrlq,    1
+IEMIMPL_MEDIA_OPT_F2 psraw,    1
+IEMIMPL_MEDIA_OPT_F2 psrad,    1
+IEMIMPL_MEDIA_OPT_F2 pmulhuw,  1
+IEMIMPL_MEDIA_OPT_F2 pavgb,    1
+IEMIMPL_MEDIA_OPT_F2 pavgw,    1
+IEMIMPL_MEDIA_OPT_F2 psadbw,   1
+IEMIMPL_MEDIA_OPT_F2 pmuldq,   0
+IEMIMPL_MEDIA_OPT_F2 unpcklps, 0
+IEMIMPL_MEDIA_OPT_F2 unpcklpd, 0
+IEMIMPL_MEDIA_OPT_F2 unpckhps, 0
+IEMIMPL_MEDIA_OPT_F2 unpckhpd, 0
+IEMIMPL_MEDIA_OPT_F2 phminposuw, 0
+IEMIMPL_MEDIA_OPT_F2 aesimc,   0
+IEMIMPL_MEDIA_OPT_F2 aesenc,   0
+IEMIMPL_MEDIA_OPT_F2 aesdec,   0
+IEMIMPL_MEDIA_OPT_F2 aesenclast, 0
+IEMIMPL_MEDIA_OPT_F2 aesdeclast, 0
+
+;;
+; Media instruction working on one full sized and one half sized register (lower half).
+;
+; @param    1       The instruction
+; @param    2       1 if MMX is included, 0 if not.
+;
+; @param    A0      Pointer to the first full sized media register operand (input/output).
+; @param    A1      Pointer to the second half sized media register operand (input).
+;
+%macro IEMIMPL_MEDIA_F1L1 2
+ %if %2 != 0
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 8
+        PROLOGUE_2_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm0, [A0]
+        movq    mm1, [A1]
+        %1      mm0, mm1
+        movq    [A0], mm0
+
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u64
+ %endif
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 8
+        PROLOGUE_2_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu   xmm0, [A0]
+        movdqu   xmm1, [A1]
+        %1       xmm0, xmm1
+        movdqu   [A0], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_F1L1 punpcklbw,  1
+IEMIMPL_MEDIA_F1L1 punpcklwd,  1
+IEMIMPL_MEDIA_F1L1 punpckldq,  1
+IEMIMPL_MEDIA_F1L1 punpcklqdq, 0
+
+
+;;
+; Media instruction working two half sized input registers (lower half) and a full sized
+; destination register (vpunpckh*).
+;
+; @param    1       The instruction
+;
+; @param    A0      Pointer to the destination register (full sized, output only).
+; @param    A1      Pointer to the first full sized media source register operand, where we
+;                   will only use the lower half as input - but we'll be loading it in full.
+; @param    A2      Pointer to the second full sized media source register operand, where we
+;                   will only use the lower half as input - but we'll be loading it in full.
+;
+%macro IEMIMPL_MEDIA_F1L1L1 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu xmm0, [A1]
+        vmovdqu xmm1, [A2]
+        %1      xmm0, xmm0, xmm1
+        vmovdqu [A0], xmm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu  ymm0, [A1]
+        vmovdqu  ymm1, [A2]
+        %1       ymm0, ymm0, ymm1
+        vmovdqu  [A0], ymm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u256
+%endmacro
+
+IEMIMPL_MEDIA_F1L1L1 vpunpcklbw
+IEMIMPL_MEDIA_F1L1L1 vpunpcklwd
+IEMIMPL_MEDIA_F1L1L1 vpunpckldq
+IEMIMPL_MEDIA_F1L1L1 vpunpcklqdq
+
+
+;;
+; Media instruction working on one full sized and one half sized register (high half).
+;
+; @param    1       The instruction
+; @param    2       1 if MMX is included, 0 if not.
+;
+; @param    A0      Pointer to the first full sized media register operand (input/output).
+; @param    A1      Pointer to the second full sized media register operand, where we
+;                   will only use the upper half as input - but we'll load it in full.
+;
+%macro IEMIMPL_MEDIA_F1H1 2
+IEMIMPL_MEDIA_F1L1 %1, %2
+%endmacro
+
+IEMIMPL_MEDIA_F1L1 punpckhbw,  1
+IEMIMPL_MEDIA_F1L1 punpckhwd,  1
+IEMIMPL_MEDIA_F1L1 punpckhdq,  1
+IEMIMPL_MEDIA_F1L1 punpckhqdq, 0
+
+
+;;
+; Media instruction working two half sized input registers (high half) and a full sized
+; destination register (vpunpckh*).
+;
+; @param    1       The instruction
+;
+; @param    A0      Pointer to the destination register (full sized, output only).
+; @param    A1      Pointer to the first full sized media source register operand, where we
+;                   will only use the upper half as input - but we'll be loading it in full.
+; @param    A2      Pointer to the second full sized media source register operand, where we
+;                   will only use the upper half as input - but we'll be loading it in full.
+;
+%macro IEMIMPL_MEDIA_F1H1H1 1
+IEMIMPL_MEDIA_F1L1L1 %1
+%endmacro
+
+IEMIMPL_MEDIA_F1H1H1 vpunpckhbw
+IEMIMPL_MEDIA_F1H1H1 vpunpckhwd
+IEMIMPL_MEDIA_F1H1H1 vpunpckhdq
+IEMIMPL_MEDIA_F1H1H1 vpunpckhqdq
+
+
+;
+; Shufflers with evil 8-bit immediates.
+;
+
+BEGINPROC_FASTCALL iemAImpl_pshufw_u64, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm1, [A1]
+        movq    mm0, mm0                ; paranoia!
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*8]         ; sizeof(pshufw+ret) == 9
+ %else
+        lea     T0, [A2 + A2*4]         ; sizeof(pshufw+ret) == 5
+ %endif
+        lea     T1, [T1 + T0]
+        IBT_NOTRACK
+        call    T1
+        movq    [A0], mm0
+
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_3_ARGS
+%assign bImm 0
+%rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        pshufw  mm0, mm1, bImm
+        ret
+ %assign bImm bImm + 1
+%endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500
+ENDPROC iemAImpl_pshufw_u64
+
+
+%macro IEMIMPL_MEDIA_SSE_PSHUFXX 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm1, [A1]
+        movdqu  xmm0, xmm1              ; paranoia!
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*4]         ; sizeof(pshufXX+ret) == 10: A2 * 10 = (A2 * 5) * 2
+ %else
+        lea     T0, [A2 + A2*2]         ; sizeof(pshufXX+ret) ==  6: A2 *  6 = (A2 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A0], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      xmm0, xmm1, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_SSE_PSHUFXX pshufhw
+IEMIMPL_MEDIA_SSE_PSHUFXX pshuflw
+IEMIMPL_MEDIA_SSE_PSHUFXX pshufd
+
+
+%macro IEMIMPL_MEDIA_AVX_VPSHUFXX 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        vmovdqu  ymm1, [A1]
+        vmovdqu  ymm0, ymm1             ; paranoia!
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*4]         ; sizeof(pshufXX+ret) == 10: A2 * 10 = (A2 * 5) * 2
+ %else
+        lea     T0, [A2 + A2*2]         ; sizeof(pshufXX+ret) ==  6: A2 *  6 = (A2 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        vmovdqu  [A0], ymm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      ymm0, ymm1, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_ %+ %1 %+ _u256
+%endmacro
+
+IEMIMPL_MEDIA_AVX_VPSHUFXX vpshufhw
+IEMIMPL_MEDIA_AVX_VPSHUFXX vpshuflw
+IEMIMPL_MEDIA_AVX_VPSHUFXX vpshufd
+
+
+;
+; Shifts with evil 8-bit immediates.
+;
+
+%macro IEMIMPL_MEDIA_MMX_PSHIFTXX 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _imm_u64, 16
+        PROLOGUE_2_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm0, [A0]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A1 + A1*8]         ; sizeof(psXX+ret) == 9
+ %else
+        lea     T0, [A1 + A1*4]         ; sizeof(psXX+ret) == 5
+ %endif
+        lea     T1, [T1 + T0]
+        IBT_NOTRACK
+        call    T1
+        movq    [A0], mm0
+
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_2_ARGS
+%assign bImm 0
+%rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      mm0, bImm
+        ret
+ %assign bImm bImm + 1
+%endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500
+ENDPROC iemAImpl_ %+ %1 %+ _imm_u64
+%endmacro
+
+IEMIMPL_MEDIA_MMX_PSHIFTXX psllw
+IEMIMPL_MEDIA_MMX_PSHIFTXX pslld
+IEMIMPL_MEDIA_MMX_PSHIFTXX psllq
+IEMIMPL_MEDIA_MMX_PSHIFTXX psrlw
+IEMIMPL_MEDIA_MMX_PSHIFTXX psrld
+IEMIMPL_MEDIA_MMX_PSHIFTXX psrlq
+IEMIMPL_MEDIA_MMX_PSHIFTXX psraw
+IEMIMPL_MEDIA_MMX_PSHIFTXX psrad
+
+
+%macro IEMIMPL_MEDIA_SSE_PSHIFTXX 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _imm_u128, 16
+        PROLOGUE_2_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A0]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A1 + A1*4]         ; sizeof(psXX+ret) == 10: A1 * 10 = (A1 * 5) * 2
+ %else
+        lea     T0, [A1 + A1*2]         ; sizeof(psXX+ret) ==  6: A1 *  6 = (A1 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A0], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_2_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      xmm0, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_ %+ %1 %+ _imm_u128
+%endmacro
+
+IEMIMPL_MEDIA_SSE_PSHIFTXX psllw
+IEMIMPL_MEDIA_SSE_PSHIFTXX pslld
+IEMIMPL_MEDIA_SSE_PSHIFTXX psllq
+IEMIMPL_MEDIA_SSE_PSHIFTXX psrlw
+IEMIMPL_MEDIA_SSE_PSHIFTXX psrld
+IEMIMPL_MEDIA_SSE_PSHIFTXX psrlq
+IEMIMPL_MEDIA_SSE_PSHIFTXX psraw
+IEMIMPL_MEDIA_SSE_PSHIFTXX psrad
+IEMIMPL_MEDIA_SSE_PSHIFTXX pslldq
+IEMIMPL_MEDIA_SSE_PSHIFTXX psrldq
+
+
+;
+; Move byte mask.
+;
+
+BEGINPROC_FASTCALL iemAImpl_pmovmskb_u64, 8
+        PROLOGUE_2_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm1, [A1]
+        pmovmskb T0, mm1
+        mov     [A0], T0
+%ifdef RT_ARCH_X86
+        mov     dword [A0 + 4], 0
+%endif
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_pmovmskb_u64
+
+BEGINPROC_FASTCALL iemAImpl_pmovmskb_u128, 8
+        PROLOGUE_2_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm1, [A1]
+        pmovmskb T0, xmm1
+        mov     [A0], T0
+%ifdef RT_ARCH_X86
+        mov     dword [A0 + 4], 0
+%endif
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_pmovmskb_u128
+
+BEGINPROC_FASTCALL iemAImpl_vpmovmskb_u256, 8
+        PROLOGUE_2_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu  ymm1, [A1]
+        vpmovmskb T0, ymm1
+        mov     [A0], T0
+%ifdef RT_ARCH_X86
+        mov     dword [A0 + 4], 0
+%endif
+        IEMIMPL_AVX_EPILOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_vpmovmskb_u256
+
+
+;;
+; Media instruction working on two full sized source registers and one destination (AVX).
+;
+; @param    1       The instruction
+;
+; @param    A0      Pointer to the extended CPU/FPU state (X86XSAVEAREA).
+; @param    A1      Pointer to the destination media register size operand (output).
+; @param    A2      Pointer to the first source media register size operand (input).
+; @param    A3      Pointer to the second source media register size operand (input).
+;
+%macro IEMIMPL_MEDIA_F3 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu  xmm0, [A2]
+        vmovdqu  xmm1, [A3]
+        %1       xmm0, xmm0, xmm1
+        vmovdqu  [A1], xmm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu  ymm0, [A2]
+        vmovdqu  ymm1, [A3]
+        %1       ymm0, ymm0, ymm1
+        vmovdqu  [A1], ymm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u256
+%endmacro
+
+IEMIMPL_MEDIA_F3 vpshufb
+IEMIMPL_MEDIA_F3 vpand
+IEMIMPL_MEDIA_F3 vpminub
+IEMIMPL_MEDIA_F3 vpminuw
+IEMIMPL_MEDIA_F3 vpminud
+IEMIMPL_MEDIA_F3 vpminsb
+IEMIMPL_MEDIA_F3 vpminsw
+IEMIMPL_MEDIA_F3 vpminsd
+IEMIMPL_MEDIA_F3 vpmaxub
+IEMIMPL_MEDIA_F3 vpmaxuw
+IEMIMPL_MEDIA_F3 vpmaxud
+IEMIMPL_MEDIA_F3 vpmaxsb
+IEMIMPL_MEDIA_F3 vpmaxsw
+IEMIMPL_MEDIA_F3 vpmaxsd
+IEMIMPL_MEDIA_F3 vpandn
+IEMIMPL_MEDIA_F3 vpor
+IEMIMPL_MEDIA_F3 vpxor
+IEMIMPL_MEDIA_F3 vpcmpeqb
+IEMIMPL_MEDIA_F3 vpcmpeqw
+IEMIMPL_MEDIA_F3 vpcmpeqd
+IEMIMPL_MEDIA_F3 vpcmpeqq
+IEMIMPL_MEDIA_F3 vpcmpgtb
+IEMIMPL_MEDIA_F3 vpcmpgtw
+IEMIMPL_MEDIA_F3 vpcmpgtd
+IEMIMPL_MEDIA_F3 vpcmpgtq
+IEMIMPL_MEDIA_F3 vpaddb
+IEMIMPL_MEDIA_F3 vpaddw
+IEMIMPL_MEDIA_F3 vpaddd
+IEMIMPL_MEDIA_F3 vpaddq
+IEMIMPL_MEDIA_F3 vpsubb
+IEMIMPL_MEDIA_F3 vpsubw
+IEMIMPL_MEDIA_F3 vpsubd
+IEMIMPL_MEDIA_F3 vpsubq
+
+
+;;
+; Media instruction working on two full sized source registers and one destination (AVX),
+; but no XSAVE state pointer argument.
+;
+; @param    1       The instruction
+;
+; @param    A0      Pointer to the destination media register size operand (output).
+; @param    A1      Pointer to the first source media register size operand (input).
+; @param    A2      Pointer to the second source media register size operand (input).
+;
+%macro IEMIMPL_MEDIA_OPT_F3 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu  xmm0, [A1]
+        vmovdqu  xmm1, [A2]
+        %1       xmm0, xmm0, xmm1
+        vmovdqu  [A0], xmm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu  ymm0, [A1]
+        vmovdqu  ymm1, [A2]
+        %1       ymm0, ymm0, ymm1
+        vmovdqu  [A0], ymm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u256
+%endmacro
+
+IEMIMPL_MEDIA_OPT_F3 vpacksswb
+IEMIMPL_MEDIA_OPT_F3 vpackssdw
+IEMIMPL_MEDIA_OPT_F3 vpackuswb
+IEMIMPL_MEDIA_OPT_F3 vpackusdw
+IEMIMPL_MEDIA_OPT_F3 vpmullw
+IEMIMPL_MEDIA_OPT_F3 vpmulld
+IEMIMPL_MEDIA_OPT_F3 vpmulhw
+IEMIMPL_MEDIA_OPT_F3 vpmulhuw
+IEMIMPL_MEDIA_OPT_F3 vpavgb
+IEMIMPL_MEDIA_OPT_F3 vpavgw
+IEMIMPL_MEDIA_OPT_F3 vpsignb
+IEMIMPL_MEDIA_OPT_F3 vpsignw
+IEMIMPL_MEDIA_OPT_F3 vpsignd
+IEMIMPL_MEDIA_OPT_F3 vphaddw
+IEMIMPL_MEDIA_OPT_F3 vphaddd
+IEMIMPL_MEDIA_OPT_F3 vphsubw
+IEMIMPL_MEDIA_OPT_F3 vphsubd
+IEMIMPL_MEDIA_OPT_F3 vphaddsw
+IEMIMPL_MEDIA_OPT_F3 vphsubsw
+IEMIMPL_MEDIA_OPT_F3 vpmaddubsw
+IEMIMPL_MEDIA_OPT_F3 vpmulhrsw
+IEMIMPL_MEDIA_OPT_F3 vpsadbw
+IEMIMPL_MEDIA_OPT_F3 vpmuldq
+IEMIMPL_MEDIA_OPT_F3 vpmuludq
+IEMIMPL_MEDIA_OPT_F3 vunpcklps
+IEMIMPL_MEDIA_OPT_F3 vunpcklpd
+IEMIMPL_MEDIA_OPT_F3 vunpckhps
+IEMIMPL_MEDIA_OPT_F3 vunpckhpd
+
+;;
+; Media instruction working on one full sized source registers and one destination (AVX),
+; but no XSAVE state pointer argument.
+;
+; @param    1       The instruction
+; @param    2       Flag whether the isntruction has a 256-bit (AVX2) variant (1) or not (0).
+;
+; @param    A0      Pointer to the destination media register size operand (output).
+; @param    A1      Pointer to the source media register size operand (input).
+;
+%macro IEMIMPL_MEDIA_OPT_F2_AVX 2
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
+        PROLOGUE_2_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu  xmm0, [A1]
+        %1       xmm0, xmm0
+        vmovdqu  [A0], xmm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+ %if %2 == 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 12
+        PROLOGUE_2_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu  ymm0, [A1]
+        %1       ymm0, ymm0
+        vmovdqu  [A0], ymm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u256
+ %endif
+%endmacro
+
+IEMIMPL_MEDIA_OPT_F2_AVX vpabsb, 1
+IEMIMPL_MEDIA_OPT_F2_AVX vpabsw, 1
+IEMIMPL_MEDIA_OPT_F2_AVX vpabsd, 1
+IEMIMPL_MEDIA_OPT_F2_AVX vphminposuw, 0
+
+
+;
+; The SSE 4.2 crc32
+;
+; @param    A1      Pointer to the 32-bit destination.
+; @param    A2      The source operand, sized according to the suffix.
+;
+BEGINPROC_FASTCALL iemAImpl_crc32_u8, 8
+        PROLOGUE_2_ARGS
+
+        mov     T0_32, [A0]
+        crc32   T0_32, A1_8
+        mov     [A0], T0_32
+
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_crc32_u8
+
+BEGINPROC_FASTCALL iemAImpl_crc32_u16, 8
+        PROLOGUE_2_ARGS
+
+        mov     T0_32, [A0]
+        crc32   T0_32, A1_16
+        mov     [A0], T0_32
+
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_crc32_u16
+
+BEGINPROC_FASTCALL iemAImpl_crc32_u32, 8
+        PROLOGUE_2_ARGS
+
+        mov     T0_32, [A0]
+        crc32   T0_32, A1_32
+        mov     [A0], T0_32
+
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_crc32_u32
+
+%ifdef RT_ARCH_AMD64
+BEGINPROC_FASTCALL iemAImpl_crc32_u64, 8
+        PROLOGUE_2_ARGS
+
+        mov     T0_32, [A0]
+        crc32   T0, A1
+        mov     [A0], T0_32
+
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_crc32_u64
+%endif
+
+
+;
+; PTEST (SSE 4.1)
+;
+; @param    A0      Pointer to the first source operand (aka readonly destination).
+; @param    A1      Pointer to the second source operand.
+; @param    A2      Pointer to the EFLAGS register.
+;
+BEGINPROC_FASTCALL  iemAImpl_ptest_u128, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A0]
+        movdqu  xmm1, [A1]
+        ptest   xmm0, xmm1
+        IEM_SAVE_FLAGS A2, X86_EFL_STATUS_BITS, 0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC             iemAImpl_ptest_u128
+
+BEGINPROC_FASTCALL  iemAImpl_vptest_u256, 12
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        vmovdqu ymm0, [A0]
+        vmovdqu ymm1, [A1]
+        vptest  ymm0, ymm1
+        IEM_SAVE_FLAGS A2, X86_EFL_STATUS_BITS, 0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC             iemAImpl_vptest_u256
+
+
+;;
+; Template for the [v]pmov{s,z}x* instructions
+;
+; @param    1       The instruction
+;
+; @param    A0      Pointer to the destination media register size operand (output).
+; @param    A1      The source operand value (input).
+;
+%macro IEMIMPL_V_PMOV_SZ_X 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
+        PROLOGUE_2_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movd     xmm0, A1
+        %1       xmm0, xmm0
+        vmovdqu  [A0], xmm0
+
+        IEMIMPL_SSE_PROLOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 12
+        PROLOGUE_2_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        movd     xmm0, A1
+        v %+ %1  xmm0, xmm0
+        vmovdqu  [A0], xmm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_v %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 12
+        PROLOGUE_2_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        movdqu   xmm0, [A1]
+        v %+ %1  ymm0, xmm0
+        vmovdqu  [A0], ymm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_v %+ %1 %+ _u256
+%endmacro
+
+IEMIMPL_V_PMOV_SZ_X pmovsxbw
+IEMIMPL_V_PMOV_SZ_X pmovsxbd
+IEMIMPL_V_PMOV_SZ_X pmovsxbq
+IEMIMPL_V_PMOV_SZ_X pmovsxwd
+IEMIMPL_V_PMOV_SZ_X pmovsxwq
+IEMIMPL_V_PMOV_SZ_X pmovsxdq
+
+IEMIMPL_V_PMOV_SZ_X pmovzxbw
+IEMIMPL_V_PMOV_SZ_X pmovzxbd
+IEMIMPL_V_PMOV_SZ_X pmovzxbq
+IEMIMPL_V_PMOV_SZ_X pmovzxwd
+IEMIMPL_V_PMOV_SZ_X pmovzxwq
+IEMIMPL_V_PMOV_SZ_X pmovzxdq
+
+
+;;
+; Need to move this as well somewhere better?
+;
+struc IEMSSERESULT
+    .uResult      resd 4
+    .MXCSR        resd 1
+endstruc
+
+
+;;
+; Need to move this as well somewhere better?
+;
+struc IEMAVX128RESULT
+    .uResult      resd 4
+    .MXCSR        resd 1
+endstruc
+
+
+;;
+; Need to move this as well somewhere better?
+;
+struc IEMAVX256RESULT
+    .uResult      resd 8
+    .MXCSR        resd 1
+endstruc
+
+
+;;
+; Initialize the SSE MXCSR register using the guest value partially to
+; account for rounding mode.
+;
+; @uses     4 bytes of stack to save the original value, T0.
+; @param    1       Expression giving the address of the FXSTATE of the guest.
+;
+%macro SSE_LD_FXSTATE_MXCSR 1
+        sub     xSP, 4
+
+        stmxcsr [xSP]
+        mov     T0_32, [%1 + X86FXSTATE.MXCSR]
+        and     T0_32, X86_MXCSR_FZ | X86_MXCSR_RC_MASK | X86_MXCSR_DAZ
+        or      T0_32, X86_MXCSR_XCPT_MASK
+        sub     xSP, 4
+        mov     [xSP], T0_32
+        ldmxcsr [xSP]
+        add     xSP, 4
+%endmacro
+
+
+;;
+; Restores the SSE MXCSR register with the original value.
+;
+; @uses     4 bytes of stack to save the content of MXCSR value, T0, T1.
+; @param    1       Expression giving the address where to return the MXCSR value.
+; @param    2       Expression giving the address of the FXSTATE of the guest.
+;
+; @note Restores the stack pointer.
+;
+%macro SSE_ST_FXSTATE_MXCSR 2
+        sub     xSP, 4
+        stmxcsr [xSP]
+        mov     T0_32, [xSP]
+        add     xSP, 4
+        ; Merge the status bits into the original MXCSR value.
+        mov     T1_32, [%2 + X86FXSTATE.MXCSR]
+        and     T0_32, X86_MXCSR_XCPT_FLAGS
+        or      T0_32, T1_32
+        mov     [%1 + IEMSSERESULT.MXCSR], T0_32
+
+        ldmxcsr [xSP]
+        add     xSP, 4
+%endmacro
+
+
+;;
+; Initialize the SSE MXCSR register using the guest value partially to
+; account for rounding mode.
+;
+; @uses     4 bytes of stack to save the original value.
+; @param    1       Expression giving the address of the FXSTATE of the guest.
+;
+%macro AVX_LD_XSAVEAREA_MXCSR 1
+        sub     xSP, 4
+
+        stmxcsr [xSP]
+        mov     T0_32, [%1 + X86FXSTATE.MXCSR]
+        and     T0_32, X86_MXCSR_FZ | X86_MXCSR_RC_MASK | X86_MXCSR_DAZ
+        sub     xSP, 4
+        mov     [xSP], T0_32
+        ldmxcsr [xSP]
+        add     xSP, 4
+%endmacro
+
+
+;;
+; Restores the AVX128 MXCSR register with the original value.
+;
+; @param    1       Expression giving the address where to return the MXCSR value.
+;
+; @note Restores the stack pointer.
+;
+%macro AVX128_ST_XSAVEAREA_MXCSR 1
+        stmxcsr [%1 + IEMAVX128RESULT.MXCSR]
+
+        ldmxcsr [xSP]
+        add     xSP, 4
+%endmacro
+
+
+;;
+; Restores the AVX256 MXCSR register with the original value.
+;
+; @param    1       Expression giving the address where to return the MXCSR value.
+;
+; @note Restores the stack pointer.
+;
+%macro AVX256_ST_XSAVEAREA_MXCSR 1
+        stmxcsr [%1 + IEMAVX256RESULT.MXCSR]
+
+        ldmxcsr [xSP]
+        add     xSP, 4
+%endmacro
+
+
+;;
+; Floating point instruction working on two full sized registers.
+;
+; @param    1       The instruction
+; @param    2       Flag whether the AVX variant of the instruction takes two or three operands, 0 to disable AVX variants
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the result including the MXCSR value.
+; @param    A2      Pointer to the first media register size operand (input/output).
+; @param    A3      Pointer to the second media register size operand (input).
+;
+%macro IEMIMPL_FP_F2 2
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 12
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        movdqu   xmm0, [A2]
+        movdqu   xmm1, [A3]
+        %1       xmm0, xmm1
+        movdqu   [A1 + IEMSSERESULT.uResult], xmm0
+
+        SSE_ST_FXSTATE_MXCSR A1, A0
+        IEMIMPL_SSE_PROLOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+ %if %2 == 3
+BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 12
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+        AVX_LD_XSAVEAREA_MXCSR A0
+
+        vmovdqu  xmm0, [A2]
+        vmovdqu  xmm1, [A3]
+        v %+ %1  xmm0, xmm0, xmm1
+        vmovdqu  [A1 + IEMAVX128RESULT.uResult], xmm0
+
+        AVX128_ST_XSAVEAREA_MXCSR A1
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_v %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 12
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+        AVX_LD_XSAVEAREA_MXCSR A0
+
+        vmovdqu  ymm0, [A2]
+        vmovdqu  ymm1, [A3]
+        v %+ %1  ymm0, ymm0, ymm1
+        vmovdqu  [A1 + IEMAVX256RESULT.uResult], ymm0
+
+        AVX256_ST_XSAVEAREA_MXCSR A1
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_v %+ %1 %+ _u256
+ %elif %2 == 2
+BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 12
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+        AVX_LD_XSAVEAREA_MXCSR A0
+
+        vmovdqu  xmm0, [A2]
+        vmovdqu  xmm1, [A3]
+        v %+ %1  xmm0, xmm1
+        vmovdqu  [A1 + IEMAVX128RESULT.uResult], xmm0
+
+        AVX128_ST_XSAVEAREA_MXCSR A1
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_v %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 12
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+        AVX_LD_XSAVEAREA_MXCSR A0
+
+        vmovdqu  ymm0, [A2]
+        vmovdqu  ymm1, [A3]
+        v %+ %1  ymm0, ymm1
+        vmovdqu  [A1 + IEMAVX256RESULT.uResult], ymm0
+
+        AVX256_ST_XSAVEAREA_MXCSR A1
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_v %+ %1 %+ _u256
+ %endif
+%endmacro
+
+IEMIMPL_FP_F2 addps, 3
+IEMIMPL_FP_F2 addpd, 3
+IEMIMPL_FP_F2 mulps, 3
+IEMIMPL_FP_F2 mulpd, 3
+IEMIMPL_FP_F2 subps, 3
+IEMIMPL_FP_F2 subpd, 3
+IEMIMPL_FP_F2 minps, 3
+IEMIMPL_FP_F2 minpd, 3
+IEMIMPL_FP_F2 divps, 3
+IEMIMPL_FP_F2 divpd, 3
+IEMIMPL_FP_F2 maxps, 3
+IEMIMPL_FP_F2 maxpd, 3
+IEMIMPL_FP_F2 haddps, 3
+IEMIMPL_FP_F2 haddpd, 3
+IEMIMPL_FP_F2 hsubps, 3
+IEMIMPL_FP_F2 hsubpd, 3
+IEMIMPL_FP_F2 addsubps, 3
+IEMIMPL_FP_F2 addsubpd, 3
+
+
+;;
+; These are actually unary operations but to keep it simple
+; we treat them as binary for now, so the output result is
+; always in sync with the register where the result might get written
+; to.
+IEMIMPL_FP_F2 sqrtps,    2
+IEMIMPL_FP_F2 rsqrtps,   2
+IEMIMPL_FP_F2 sqrtpd,    2
+IEMIMPL_FP_F2 cvtdq2ps,  2
+IEMIMPL_FP_F2 cvtps2dq,  2
+IEMIMPL_FP_F2 cvttps2dq, 2
+IEMIMPL_FP_F2 cvttpd2dq, 0 ; @todo AVX variants due to register size differences missing right now
+IEMIMPL_FP_F2 cvtdq2pd,  0 ; @todo AVX variants due to register size differences missing right now
+IEMIMPL_FP_F2 cvtpd2dq,  0 ; @todo AVX variants due to register size differences missing right now
+
+
+;;
+; Floating point instruction working on a full sized register and a single precision operand.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the result including the MXCSR value.
+; @param    A2      Pointer to the first media register size operand (input/output).
+; @param    A3      Pointer to the second single precision floating point value (input).
+;
+%macro IEMIMPL_FP_F2_R32 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128_r32, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        movdqu   xmm0, [A2]
+        movd     xmm1, [A3]
+        %1       xmm0, xmm1
+        movdqu   [A1 + IEMSSERESULT.uResult], xmm0
+
+        SSE_ST_FXSTATE_MXCSR A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128_r32
+
+BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128_r32, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+        AVX_LD_XSAVEAREA_MXCSR A0
+
+        vmovdqu  xmm0, [A2]
+        vmovd    xmm1, [A3]
+        v %+ %1  xmm0, xmm0, xmm1
+        vmovdqu  [A1 + IEMAVX128RESULT.uResult], xmm0
+
+        AVX128_ST_XSAVEAREA_MXCSR A1
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_v %+ %1 %+ _u128_r32
+%endmacro
+
+IEMIMPL_FP_F2_R32 addss
+IEMIMPL_FP_F2_R32 mulss
+IEMIMPL_FP_F2_R32 subss
+IEMIMPL_FP_F2_R32 minss
+IEMIMPL_FP_F2_R32 divss
+IEMIMPL_FP_F2_R32 maxss
+IEMIMPL_FP_F2_R32 cvtss2sd
+IEMIMPL_FP_F2_R32 sqrtss
+IEMIMPL_FP_F2_R32 rsqrtss
+
+
+;;
+; Floating point instruction working on a full sized register and a double precision operand.
+;
+; @param    1       The instruction
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the result including the MXCSR value.
+; @param    A2      Pointer to the first media register size operand (input/output).
+; @param    A3      Pointer to the second double precision floating point value (input).
+;
+%macro IEMIMPL_FP_F2_R64 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128_r64, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        movdqu   xmm0, [A2]
+        movq     xmm1, [A3]
+        %1       xmm0, xmm1
+        movdqu   [A1 + IEMSSERESULT.uResult], xmm0
+
+        SSE_ST_FXSTATE_MXCSR A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128_r64
+
+BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128_r64, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+        AVX_LD_XSAVEAREA_MXCSR A0
+
+        vmovdqu  xmm0, [A2]
+        vmovq    xmm1, [A3]
+        v %+ %1  xmm0, xmm0, xmm1
+        vmovdqu  [A1 + IEMAVX128RESULT.uResult], xmm0
+
+        AVX128_ST_XSAVEAREA_MXCSR A1
+        IEMIMPL_AVX_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_v %+ %1 %+ _u128_r64
+%endmacro
+
+IEMIMPL_FP_F2_R64 addsd
+IEMIMPL_FP_F2_R64 mulsd
+IEMIMPL_FP_F2_R64 subsd
+IEMIMPL_FP_F2_R64 minsd
+IEMIMPL_FP_F2_R64 divsd
+IEMIMPL_FP_F2_R64 maxsd
+IEMIMPL_FP_F2_R64 cvtsd2ss
+IEMIMPL_FP_F2_R64 sqrtsd
+
+
+;;
+; Macro for the cvtpd2ps/cvtps2pd instructions.
+;
+;           1       The instruction name.
+;           2       Whether the AVX256 result is 128-bit (0) or 256-bit (1).
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the result including the MXCSR value.
+; @param    A2      Pointer to the first media register size operand (input/output).
+; @param    A3      Pointer to the second media register size operand (input).
+;
+%macro IEMIMPL_CVT_F2 2
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        movdqu   xmm0, [A2]
+        movdqu   xmm1, [A3]
+        %1       xmm0, xmm1
+        movdqu   [A1 + IEMSSERESULT.uResult], xmm0
+
+        SSE_ST_FXSTATE_MXCSR A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+        AVX_LD_XSAVEAREA_MXCSR A0
+
+        vmovdqu   xmm0, [A2]
+        vmovdqu   xmm1, [A3]
+        v %+ %1   xmm0, xmm1
+        vmovdqu  [A1 + IEMAVX128RESULT.uResult], xmm0
+
+        AVX128_ST_XSAVEAREA_MXCSR A1
+        IEMIMPL_AVX_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_v %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_v %+ %1 %+ _u256, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+        AVX_LD_XSAVEAREA_MXCSR A0
+
+        vmovdqu    ymm0, [A2]
+        vmovdqu    ymm1, [A3]
+ %if %2 == 0
+        v %+ %1    xmm0, ymm1
+ %else
+        v %+ %1    ymm0, xmm1
+ %endif
+        vmovdqu    [A1 + IEMAVX256RESULT.uResult], ymm0
+
+        AVX256_ST_XSAVEAREA_MXCSR A1
+        IEMIMPL_AVX_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_v %+ %1 %+ _u256
+%endmacro
+
+IEMIMPL_CVT_F2 cvtpd2ps, 0
+IEMIMPL_CVT_F2 cvtps2pd, 1
+
+
+;;
+; shufps instructions with 8-bit immediates.
+;
+; @param    A0      Pointer to the destination media register size operand (input/output).
+; @param    A1      Pointer to the first source media register size operand (input).
+; @param    A2      The 8-bit immediate
+;
+BEGINPROC_FASTCALL iemAImpl_shufps_u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A0]
+        movdqu  xmm1, [A1]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*4]         ; sizeof(shufpX+ret+int3) == 10: A2 * 10 = (A2 * 5) * 2
+ %else
+        lea     T0, [A2 + A2*2]         ; sizeof(shufpX+ret+int3) ==  6: A2 *  6 = (A2 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A0], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        shufps  xmm0, xmm1, bImm
+        ret
+        int3
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_shufps_u128
+
+
+;;
+; shufpd instruction with 8-bit immediates.
+;
+; @param    A0      Pointer to the destination media register size operand (input/output).
+; @param    A1      Pointer to the first source media register size operand (input).
+; @param    A2      The 8-bit immediate
+;
+BEGINPROC_FASTCALL iemAImpl_shufpd_u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A0]
+        movdqu  xmm1, [A1]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*4]         ; sizeof(shufpX+ret) == 10: A2 * 10 = (A2 * 5) * 2
+ %else
+        lea     T0, [A2 + A2*2]         ; sizeof(shufpX+ret) ==  6: A2 *  6 = (A2 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A0], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        shufpd  xmm0, xmm1, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_shufpd_u128
+
+
+;;
+; vshufp{s,d} instructions with 8-bit immediates.
+;
+; @param    1       The instruction name.
+;
+; @param    A0      Pointer to the destination media register size operand (output).
+; @param    A1      Pointer to the first source media register size operand (input).
+; @param    A2      Pointer to the second source media register size operand (input).
+; @param    A3      The 8-bit immediate
+;
+%macro IEMIMPL_MEDIA_AVX_VSHUFPX 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        movdqu  xmm0, [A1]
+        movdqu  xmm1, [A2]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A3 + A3*4]         ; sizeof(vshufpX+ret) == 10: A3 * 10 = (A3 * 5) * 2
+ %else
+        lea     T0, [A3 + A3*2]         ; sizeof(vshufpX+ret) ==  6: A3 *  6 = (A3 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A0], xmm0
+
+        IEMIMPL_AVX_EPILOGUE
+        EPILOGUE_4_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      xmm0, xmm0, xmm1, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu ymm0, [A1]
+        vmovdqu ymm1, [A2]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A3 + A3*4]         ; sizeof(vshufpX+ret) == 10: A3 * 10 = (A3 * 5) * 2
+ %else
+        lea     T0, [A3 + A3*2]         ; sizeof(vshufpX+ret) ==  6: A3 *  6 = (A3 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        vmovdqu [A0], ymm0
+
+        IEMIMPL_AVX_EPILOGUE
+        EPILOGUE_4_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      ymm0, ymm0, ymm1, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_ %+ %1 %+ _u256
+%endmacro
+
+IEMIMPL_MEDIA_AVX_VSHUFPX vshufps
+IEMIMPL_MEDIA_AVX_VSHUFPX vshufpd
+
+
+;;
+; One of the [p]blendv{b,ps,pd} variants
+;
+; @param    1       The instruction
+;
+; @param    A0      Pointer to the first media register sized operand (input/output).
+; @param    A1      Pointer to the second media sized value (input).
+; @param    A2      Pointer to the media register sized mask value (input).
+;
+%macro IEMIMPL_P_BLEND 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu   xmm0, [A2] ; This is implicit
+        movdqu   xmm1, [A0]
+        movdqu   xmm2, [A1] ; @todo Do I need to save the original value here first?
+        %1       xmm1, xmm2
+        movdqu   [A0], xmm1
+
+        IEMIMPL_SSE_PROLOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_P_BLEND pblendvb
+IEMIMPL_P_BLEND blendvps
+IEMIMPL_P_BLEND blendvpd
+
+
+;;
+; One of the v[p]blendv{b,ps,pd} variants
+;
+; @param    1       The instruction
+;
+; @param    A0      Pointer to the first media register sized operand (output).
+; @param    A1      Pointer to the first media register sized operand (input).
+; @param    A2      Pointer to the second media register sized operand (input).
+; @param    A3      Pointer to the media register sized mask value (input).
+%macro IEMIMPL_AVX_P_BLEND 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu   xmm0, [A1]
+        vmovdqu   xmm1, [A2]
+        vmovdqu   xmm2, [A3]
+        %1        xmm0, xmm0, xmm1, xmm2
+        vmovdqu   [A0], xmm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu   ymm0, [A1]
+        vmovdqu   ymm1, [A2]
+        vmovdqu   ymm2, [A3]
+        %1        ymm0, ymm0, ymm1, ymm2
+        vmovdqu   [A0], ymm0
+
+        IEMIMPL_AVX_PROLOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u256
+%endmacro
+
+IEMIMPL_AVX_P_BLEND vpblendvb
+IEMIMPL_AVX_P_BLEND vblendvps
+IEMIMPL_AVX_P_BLEND vblendvpd
+
+
+;;
+; palignr mm1, mm2/m64 instruction.
+;
+; @param    A0      Pointer to the first media register sized operand (output).
+; @param    A1      The second register sized operand (input).
+; @param    A2      The 8-bit immediate.
+BEGINPROC_FASTCALL iemAImpl_palignr_u64, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_MMX_PROLOGUE
+
+        movq    mm0, [A0]
+        movq    mm1, A1
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*4]         ; sizeof(endbrxx+palignr+ret) == 10: A2 * 10 = (A2 * 5) * 2
+ %else
+        lea     T0, [A2 + A2*2]         ; sizeof(palignr+ret)         ==  6: A2 *  6 = (A2 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        movq    [A0], mm0
+
+        IEMIMPL_MMX_EPILOGUE
+        EPILOGUE_3_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        palignr mm0, mm1, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_palignr_u64
+
+
+;;
+; SSE instructions with 8-bit immediates of the form
+;    xxx xmm1, xmm2, imm8.
+; where the instruction encoding takes up 6 bytes.
+;
+; @param    1       The instruction name.
+;
+; @param    A0      Pointer to the first media register size operand (input/output).
+; @param    A1      Pointer to the second source media register size operand (input).
+; @param    A2      The 8-bit immediate
+;
+%macro IEMIMPL_MEDIA_SSE_INSN_IMM8_6 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A0]
+        movdqu  xmm1, [A1]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*2]         ; sizeof(endbrxx+insnX+ret+int3) == 12: A2 * 12 = (A2 * 3) * 4
+        lea     T1, [T1 + T0*4]
+ %else
+        lea     T1, [T1 + A2*8]         ; sizeof(insnX+ret+int3)         ==  8: A2 * 8
+ %endif
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A0], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      xmm0, xmm1, bImm
+        ret
+        int3
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_SSE_INSN_IMM8_6 blendps
+IEMIMPL_MEDIA_SSE_INSN_IMM8_6 blendpd
+IEMIMPL_MEDIA_SSE_INSN_IMM8_6 pblendw
+IEMIMPL_MEDIA_SSE_INSN_IMM8_6 palignr
+IEMIMPL_MEDIA_SSE_INSN_IMM8_6 pclmulqdq
+IEMIMPL_MEDIA_SSE_INSN_IMM8_6 aeskeygenassist
+
+
+;;
+; AVX instructions with 8-bit immediates of the form
+;    xxx {x,y}mm1, {x,y}mm2, {x,y}mm3, imm8.
+; where the instruction encoding takes up 6 bytes.
+;
+; @param    1       The instruction name.
+; @param    2       Whether the instruction has a 256-bit variant (1) or not (0).
+;
+; @param    A0      Pointer to the destination media register size operand (output).
+; @param    A1      Pointer to the first source media register size operand (input).
+; @param    A2      Pointer to the second source media register size operand (input).
+; @param    A3      The 8-bit immediate
+;
+%macro IEMIMPL_MEDIA_AVX_INSN_IMM8_6 2
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        movdqu  xmm0, [A1]
+        movdqu  xmm1, [A2]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A3 + A3*2]         ; sizeof(endbrxx+insnX+ret+int3) == 12: A3 * 12 = (A3 * 3) * 4
+        lea     T1, [T1 + T0*4]
+ %else
+        lea     T1, [T1 + A3*8]         ; sizeof(insnX+ret+int3)         ==  8: A3 * 8
+ %endif
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A0], xmm0
+
+        IEMIMPL_AVX_EPILOGUE
+        EPILOGUE_4_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      xmm0, xmm0, xmm1, bImm
+        ret
+        int3
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+ %if %2 == 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u256, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu ymm0, [A1]
+        vmovdqu ymm1, [A2]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A3 + A3*2]         ; sizeof(endbrxx+insnX+ret+int3) == 12: A3 * 12 = (A3 * 3) * 4
+        lea     T1, [T1 + T0*4]
+ %else
+        lea     T1, [T1 + A3*8]         ; sizeof(insnX+ret+int3)         ==  8: A3 *  8
+ %endif
+        IBT_NOTRACK
+        call    T1
+        vmovdqu [A0], ymm0
+
+        IEMIMPL_AVX_EPILOGUE
+        EPILOGUE_4_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      ymm0, ymm0, ymm1, bImm
+        ret
+        int3
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800
+ENDPROC iemAImpl_ %+ %1 %+ _u256
+ %endif
+%endmacro
+
+IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vblendps,   1
+IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vblendpd,   1
+IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vpblendw,   1
+IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vpalignr,   1
+IEMIMPL_MEDIA_AVX_INSN_IMM8_6 vpclmulqdq, 0
+
+
+;;
+; Need to move this as well somewhere better?
+;
+struc IEMPCMPISTRISRC
+    .uSrc1        resd 4
+    .uSrc2        resd 4
+endstruc
+
+;;
+; The pcmpistri instruction.
+;
+; @param    A0      Pointer to the ECX register to store the result to (output).
+; @param    A1      Pointer to the EFLAGS register.
+; @param    A2      Pointer to the structure containing the source operands (input).
+; @param    A3      The 8-bit immediate
+;
+BEGINPROC_FASTCALL iemAImpl_pcmpistri_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A2 + IEMPCMPISTRISRC.uSrc1]
+        movdqu  xmm1, [A2 + IEMPCMPISTRISRC.uSrc2]
+        mov     T2, A0                  ; A0 can be ecx/rcx in some calling conventions which gets overwritten later (T2 only available on AMD64)
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A3 + A3*2]         ; sizeof(endbrxx+insnX+ret) == 12: A3 * 12 = (A3 * 3) * 4
+        lea     T1, [T1 + T0*4]
+ %else
+        lea     T1, [T1 + A3*8]         ; sizeof(insnX+ret)         ==  8: A3 * 8
+ %endif
+        IBT_NOTRACK
+        call    T1
+
+        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        mov    [T2], ecx
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        pcmpistri xmm0, xmm1, bImm
+        ret
+        int3
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800
+ENDPROC iemAImpl_pcmpistri_u128
+
+
+;;
+; pinsrw instruction.
+;
+; @param    A0      Pointer to the first media register size operand (input/output).
+; @param    A1      The 16 bit input operand (input).
+; @param    A2      The 8-bit immediate
+;
+BEGINPROC_FASTCALL iemAImpl_pinsrw_u64, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movq    mm0,  [A0]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*8]         ; sizeof(endbrxx+pinsrw+ret) == 9: A2 * 9
+ %else
+        lea     T0, [A2 + A2*4]         ; sizeof(pinsrw+ret)         == 5: A2 * 5
+ %endif
+        lea     T1, [T1 + T0]
+        IBT_NOTRACK
+        call    T1
+        movq    [A0], mm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        pinsrw  mm0, A1_32, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500
+ENDPROC iemAImpl_pinsrw_u64
+
+BEGINPROC_FASTCALL iemAImpl_pinsrw_u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A0]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*4]         ; sizeof(endbrxx+pinsrw+ret) == 10: A2 * 10 = (A2 * 5) * 2
+ %else
+        lea     T0, [A2 + A2*2]         ; sizeof(pinsrw+ret)         ==  6: A2 *  6 = (A2 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A0], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        pinsrw  xmm0, A1_32, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_pinsrw_u128
+
+;;
+; vpinsrw instruction.
+;
+; @param    A0      Pointer to the first media register size operand (output).
+; @param    A1      Pointer to the source media register size operand (input).
+; @param    A2      The 16 bit input operand (input).
+; @param    A3      The 8-bit immediate
+;
+BEGINPROC_FASTCALL iemAImpl_vpinsrw_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A1]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A3 + A3*4]         ; sizeof(endbrxx+vpinsrw+ret) == 10: A3 * 10 = (A3 * 5) * 2
+ %else
+        lea     T0, [A3 + A3*2]         ; sizeof(vpinsrw+ret)         ==  6: A3 *  6 = (A3 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        mov     A1, A2                  ; A2 requires longer encoding on Windows
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A0], xmm0
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        vpinsrw xmm0, xmm0, A1_32, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_vpinsrw_u128
+
+
+;;
+; pextrw instruction.
+;
+; @param    A0      Pointer to the 16bit output operand (output).
+; @param    A1      Pointer to the media register size operand (input).
+; @param    A2      The 8-bit immediate
+;
+BEGINPROC_FASTCALL iemAImpl_pextrw_u64, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movq    mm0,  A1
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*8]         ; sizeof(endbrxx+pextrw+ret) == 9: A2 * 9
+ %else
+        lea     T0, [A2 + A2*4]         ; sizeof(pextrw+ret)         == 5: A2 * 5
+ %endif
+        lea     T1, [T1 + T0]
+        IBT_NOTRACK
+        call    T1
+        mov     word [A0], T0_16
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        pextrw  T0_32, mm0, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500
+ENDPROC iemAImpl_pextrw_u64
+
+BEGINPROC_FASTCALL iemAImpl_pextrw_u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A1]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*4]         ; sizeof(endbrxx+pextrw+ret) == 10: A2 * 10 = (A2 * 5) * 2
+ %else
+        lea     T0, [A2 + A2*2]         ; sizeof(pextrw+ret)         ==  6: A2 *  6 = (A2 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        mov     word [A0], T0_16
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        pextrw  T0_32, xmm0, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_pextrw_u128
+
+;;
+; vpextrw instruction.
+;
+; @param    A0      Pointer to the 16bit output operand (output).
+; @param    A1      Pointer to the source media register size operand (input).
+; @param    A2      The 8-bit immediate
+;
+BEGINPROC_FASTCALL iemAImpl_vpextrw_u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A1]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A2 + A2*4]         ; sizeof(endbrxx+vpextrw+ret) == 10: A2 * 10 = (A2 * 5) * 2
+ %else
+        lea     T0, [A2 + A2*2]         ; sizeof(vpextrw+ret)         ==  6: A2 *  6 = (A2 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        mov     word [A0], T0_16
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        vpextrw T0_32, xmm0, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_vpextrw_u128
+
+
+;;
+; movmskp{s,d} SSE instruction template
+;
+; @param    1       The SSE instruction name.
+; @param    2       The AVX instruction name.
+;
+; @param    A0      Pointer to the output register (output/byte sized).
+; @param    A1      Pointer to the source media register size operand (input).
+;
+%macro IEMIMPL_MEDIA_MOVMSK_P 2
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_2_ARGS
+        IEMIMPL_SSE_PROLOGUE
+
+        movdqu  xmm0, [A1]
+        %1      T0, xmm0
+        mov     byte [A0], T0_8
+
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %2 %+ _u128, 16
+        PROLOGUE_2_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        movdqu  xmm0, [A1]
+        %2      T0, xmm0
+        mov     byte [A0], T0_8
+
+        IEMIMPL_AVX_EPILOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %2 %+ _u128
+
+BEGINPROC_FASTCALL iemAImpl_ %+ %2 %+ _u256, 16
+        PROLOGUE_2_ARGS
+        IEMIMPL_AVX_PROLOGUE
+
+        vmovdqu ymm0, [A1]
+        %2      T0, ymm0
+        mov     byte [A0], T0_8
+
+        IEMIMPL_AVX_EPILOGUE
+        EPILOGUE_2_ARGS
+ENDPROC iemAImpl_ %+ %2 %+ _u256
+%endmacro
+
+IEMIMPL_MEDIA_MOVMSK_P movmskps, vmovmskps
+IEMIMPL_MEDIA_MOVMSK_P movmskpd, vmovmskpd
+
+
+;;
+; Restores the SSE MXCSR register with the original value.
+;
+; @uses     4 bytes of stack to save the content of MXCSR value, T0, T1.
+; @param    1       Expression giving the address where to return the MXCSR value - only the MXCSR is stored, no IEMSSERESULT is used.
+; @param    2       Expression giving the address of the FXSTATE of the guest.
+;
+; @note Restores the stack pointer.
+;
+%macro SSE_ST_FXSTATE_MXCSR_ONLY 2
+        sub     xSP, 4
+        stmxcsr [xSP]
+        mov     T0_32, [xSP]
+        add     xSP, 4
+        ; Merge the status bits into the original MXCSR value.
+        mov     T1_32, [%2 + X86FXSTATE.MXCSR]
+        and     T0_32, X86_MXCSR_XCPT_FLAGS
+        or      T0_32, T1_32
+        mov     [%1], T0_32
+
+        ldmxcsr [xSP]
+        add     xSP, 4
+%endmacro
+
+
+;;
+; cvttsd2si instruction - 32-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvttsd2si_i32_r64, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvttsd2si T0_32, [A3]
+        mov       dword [A2], T0_32
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvttsd2si_i32_r64
+
+;;
+; cvttsd2si instruction - 64-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvttsd2si_i64_r64, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvttsd2si T0, [A3]
+        mov       qword [A2], T0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvttsd2si_i64_r64
+
+
+;;
+; cvtsd2si instruction - 32-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvtsd2si_i32_r64, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvtsd2si  T0_32, [A3]
+        mov       dword [A2], T0_32
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvtsd2si_i32_r64
+
+;;
+; cvtsd2si instruction - 64-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvtsd2si_i64_r64, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvtsd2si  T0, [A3]
+        mov       qword [A2], T0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvtsd2si_i64_r64
+
+
+;;
+; cvttss2si instruction - 32-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvttss2si_i32_r32, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvttss2si T0_32, [A3]
+        mov       dword [A2], T0_32
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvttss2si_i32_r32
+
+;;
+; cvttss2si instruction - 64-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvttss2si_i64_r32, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvttss2si T0, [A3]
+        mov       qword [A2], T0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvttss2si_i64_r32
+
+
+;;
+; cvtss2si instruction - 32-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvtss2si_i32_r32, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvtss2si  T0_32, [A3]
+        mov       dword [A2], T0_32
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvtss2si_i32_r32
+
+;;
+; cvtss2si instruction - 64-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvtss2si_i64_r32, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvtss2si  T0, [A3]
+        mov       qword [A2], T0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvtss2si_i64_r32
+
+
+;;
+; cvtsi2ss instruction - 32-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvtsi2ss_r32_i32, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvtsi2ss  xmm0, dword [A3]
+        movd      dword [A2], xmm0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvtsi2ss_r32_i32
+
+;;
+; cvtsi2ss instruction - 64-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvtsi2ss_r32_i64, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvtsi2ss  xmm0, qword [A3]
+        movd      dword [A2], xmm0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvtsi2ss_r32_i64
+
+
+;;
+; cvtsi2sd instruction - 32-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvtsi2sd_r64_i32, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvtsi2sd  xmm0, dword [A3]
+        movq      [A2], xmm0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvtsi2sd_r64_i32
+
+;;
+; cvtsi2sd instruction - 64-bit variant.
+;
+; @param    A0      FPU context (FXSTATE or XSAVEAREA).
+; @param    A1      Where to return the MXCSR value.
+; @param    A2      Pointer to the result operand (output).
+; @param    A3      Pointer to the second operand (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cvtsi2sd_r64_i64, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR A0
+
+        cvtsi2sd  xmm0, qword [A3]
+        movq      [A2], xmm0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY A1, A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC iemAImpl_cvtsi2sd_r64_i64
+
+
+;;
+; Initialize the SSE MXCSR register using the guest value partially to
+; account for rounding mode.
+;
+; @uses     4 bytes of stack to save the original value, T0.
+; @param    1       Expression giving the address of the MXCSR register of the guest.
+;
+%macro SSE_LD_FXSTATE_MXCSR_ONLY 1
+        sub     xSP, 4
+
+        stmxcsr [xSP]
+        mov     T0_32, [%1]
+        and     T0_32, X86_MXCSR_FZ | X86_MXCSR_RC_MASK | X86_MXCSR_DAZ
+        or      T0_32, X86_MXCSR_XCPT_MASK
+        sub     xSP, 4
+        mov     [xSP], T0_32
+        ldmxcsr [xSP]
+        add     xSP, 4
+%endmacro
+
+
+;;
+; Restores the SSE MXCSR register with the original value.
+;
+; @uses     4 bytes of stack to save the content of MXCSR value, T0, T1.
+; @param    1       Expression giving the address where to return the MXCSR value - only the MXCSR is stored, no IEMSSERESULT is used.
+;
+; @note Restores the stack pointer.
+;
+%macro SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE 1
+        sub     xSP, 4
+        stmxcsr [xSP]
+        mov     T0_32, [xSP]
+        add     xSP, 4
+        ; Merge the status bits into the original MXCSR value.
+        mov     T1_32, [%1]
+        and     T0_32, X86_MXCSR_XCPT_FLAGS
+        or      T0_32, T1_32
+        mov     [%1], T0_32
+
+        ldmxcsr [xSP]
+        add     xSP, 4
+%endmacro
+
+
+;
+; UCOMISS (SSE)
+;
+; @param    A0      Pointer to the MXCSR value (input/output).
+; @param    A1      Pointer to the EFLAGS value (input/output).
+; @param    A2      Pointer to the first source operand (aka readonly destination).
+; @param    A3      Pointer to the second source operand.
+;
+BEGINPROC_FASTCALL  iemAImpl_ucomiss_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2]
+        movdqu  xmm1, [A3]
+        ucomiss xmm0, xmm1
+        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC             iemAImpl_ucomiss_u128
+
+BEGINPROC_FASTCALL  iemAImpl_vucomiss_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2]
+        movdqu  xmm1, [A3]
+        vucomiss xmm0, xmm1
+        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC             iemAImpl_vucomiss_u128
+
+
+;
+; UCOMISD (SSE)
+;
+; @param    A0      Pointer to the MXCSR value (input/output).
+; @param    A1      Pointer to the EFLAGS value (input/output).
+; @param    A2      Pointer to the first source operand (aka readonly destination).
+; @param    A3      Pointer to the second source operand.
+;
+BEGINPROC_FASTCALL  iemAImpl_ucomisd_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2]
+        movdqu  xmm1, [A3]
+        ucomisd xmm0, xmm1
+        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC             iemAImpl_ucomisd_u128
+
+BEGINPROC_FASTCALL  iemAImpl_vucomisd_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2]
+        movdqu  xmm1, [A3]
+        vucomisd xmm0, xmm1
+        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC             iemAImpl_vucomisd_u128
+
+;
+; COMISS (SSE)
+;
+; @param    A0      Pointer to the MXCSR value (input/output).
+; @param    A1      Pointer to the EFLAGS value (input/output).
+; @param    A2      Pointer to the first source operand (aka readonly destination).
+; @param    A3      Pointer to the second source operand.
+;
+BEGINPROC_FASTCALL  iemAImpl_comiss_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2]
+        movdqu  xmm1, [A3]
+        comiss xmm0, xmm1
+        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC             iemAImpl_comiss_u128
+
+BEGINPROC_FASTCALL  iemAImpl_vcomiss_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2]
+        movdqu  xmm1, [A3]
+        vcomiss xmm0, xmm1
+        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC             iemAImpl_vcomiss_u128
+
+
+;
+; COMISD (SSE)
+;
+; @param    A0      Pointer to the MXCSR value (input/output).
+; @param    A1      Pointer to the EFLAGS value (input/output).
+; @param    A2      Pointer to the first source operand (aka readonly destination).
+; @param    A3      Pointer to the second source operand.
+;
+BEGINPROC_FASTCALL  iemAImpl_comisd_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2]
+        movdqu  xmm1, [A3]
+        comisd xmm0, xmm1
+        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC             iemAImpl_comisd_u128
+
+BEGINPROC_FASTCALL  iemAImpl_vcomisd_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2]
+        movdqu  xmm1, [A3]
+        vcomisd xmm0, xmm1
+        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ENDPROC             iemAImpl_vcomisd_u128
+
+
+;;
+; Need to move this as well somewhere better?
+;
+struc IEMMEDIAF2XMMSRC
+    .uSrc1        resd 4
+    .uSrc2        resd 4
+endstruc
+
+
+;
+; CMPPS (SSE)
+;
+; @param    A0      Pointer to the MXCSR value (input/output).
+; @param    A1      Pointer to the first media register size operand (output).
+; @param    A2      Pointer to the two media register sized inputs - IEMMEDIAF2XMMSRC (input).
+; @param    A3      The 8-bit immediate (input).
+;
+BEGINPROC_FASTCALL iemAImpl_cmpps_u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2 + IEMMEDIAF2XMMSRC.uSrc1]
+        movdqu  xmm1, [A2 + IEMMEDIAF2XMMSRC.uSrc2]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A3 + A3*8]         ; sizeof(endbrxx+cmpps+ret) == 9: A3 * 9
+ %else
+        lea     T0, [A3 + A3*4]         ; sizeof(cmpps+ret)         == 5: A3 * 5
+ %endif
+        lea     T1, [T1 + T0]
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A1], xmm0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        cmpps   xmm0, xmm1, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x500
+ENDPROC iemAImpl_cmpps_u128
+
+;;
+; SSE instructions with 8-bit immediates of the form
+;    xxx xmm1, xmm2, imm8.
+; where the instruction encoding takes up 5 bytes and we need to load and save the MXCSR
+; register.
+;
+; @param    1       The instruction name.
+;
+; @param    A0      Pointer to the MXCSR value (input/output).
+; @param    A1      Pointer to the first media register size operand (output).
+; @param    A2      Pointer to the two media register sized inputs - IEMMEDIAF2XMMSRC (input).
+; @param    A3      The 8-bit immediate (input).
+;
+%macro IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2 + IEMMEDIAF2XMMSRC.uSrc1]
+        movdqu  xmm1, [A2 + IEMMEDIAF2XMMSRC.uSrc2]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A3 + A3*4]         ; sizeof(endbrxx+cmpXX+ret) == 10: A3 * 10 = (A3 * 5) * 2
+ %else
+        lea     T0, [A3 + A3*2]         ; sizeof(cmpXX+ret)         ==  6: A3 *  6 = (A3 * 3) * 2
+ %endif
+        lea     T1, [T1 + T0*2]
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A1], xmm0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      xmm0, xmm1, bImm
+        ret
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x600
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 cmppd
+IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 cmpss
+IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_5 cmpsd
+
+;;
+; SSE instructions with 8-bit immediates of the form
+;    xxx xmm1, xmm2, imm8.
+; where the instruction encoding takes up 6 bytes and we need to load and save the MXCSR
+; register.
+;
+; @param    1       The instruction name.
+;
+; @param    A0      Pointer to the MXCSR value (input/output).
+; @param    A1      Pointer to the first media register size operand (output).
+; @param    A2      Pointer to the two media register sized inputs - IEMMEDIAF2XMMSRC (input).
+; @param    A3      The 8-bit immediate (input).
+;
+%macro IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_4_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2 + IEMMEDIAF2XMMSRC.uSrc1]
+        movdqu  xmm1, [A2 + IEMMEDIAF2XMMSRC.uSrc2]
+        lea     T1, [.imm0 xWrtRIP]
+ %ifdef RT_WITH_IBT_BRANCH_PROTECTION_WITHOUT_NOTRACK
+        lea     T0, [A3 + A3*2]         ; sizeof(endbrxx+insn+ret+int3) == 12: A3 * 12 = (A3 * 3) * 4
+        lea     T1, [T1 + T0*4]
+ %else
+        lea     T1, [T1 + A3*8]         ; sizeof(insn+ret+int3)         ==  8: A3 * 8
+ %endif
+        IBT_NOTRACK
+        call    T1
+        movdqu  [A1], xmm0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_4_ARGS
+ %assign bImm 0
+ %rep 256
+.imm %+ bImm:
+        IBT_ENDBRxx_WITHOUT_NOTRACK
+        %1      xmm0, xmm1, bImm
+        ret
+        int3
+  %assign bImm bImm + 1
+ %endrep
+.immEnd: IEMCHECK_256_JUMP_ARRAY_SIZE (.immEnd - .imm0), 0x800
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundps
+IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundpd
+IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundss
+IEMIMPL_MEDIA_SSE_INSN_IMM8_MXCSR_6 roundsd
+
+
+;;
+; SSE instructions of the form
+;    xxx mm, xmm.
+; and we need to load and save the MXCSR register.
+;
+; @param    1       The instruction name.
+;
+; @param    A0      Pointer to the MXCSR value (input/output).
+; @param    A1      Pointer to the first MMX register sized operand (output).
+; @param    A2      Pointer to the media register sized operand (input).
+;
+%macro IEMIMPL_MEDIA_SSE_MXCSR_I64_U128 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A2]
+        %1      mm0, xmm0
+        movq    [A1], mm0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_SSE_MXCSR_I64_U128 cvtpd2pi
+IEMIMPL_MEDIA_SSE_MXCSR_I64_U128 cvttpd2pi
+
+;;
+; SSE instructions of the form
+;    xxx xmm, xmm/m64.
+; and we need to load and save the MXCSR register.
+;
+; @param    1       The instruction name.
+;
+; @param    A0      Pointer to the MXCSR value (input/output).
+; @param    A1      Pointer to the first media register sized operand (input/output).
+; @param    A2      The 64bit source value from a MMX media register (input)
+;
+%macro IEMIMPL_MEDIA_SSE_MXCSR_U128_U64 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movdqu  xmm0, [A1]
+        movq    mm0, A2
+        %1      xmm0, mm0
+        movdqu  [A1], xmm0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_SSE_MXCSR_U128_U64 cvtpi2ps
+IEMIMPL_MEDIA_SSE_MXCSR_U128_U64 cvtpi2pd
+
+;;
+; SSE instructions of the form
+;    xxx mm, xmm/m64.
+; and we need to load and save the MXCSR register.
+;
+; @param    1       The instruction name.
+;
+; @param    A0      Pointer to the MXCSR value (input/output).
+; @param    A1      Pointer to the first MMX media register sized operand (output).
+; @param    A2      The 64bit source value (input).
+;
+%macro IEMIMPL_MEDIA_SSE_MXCSR_U64_U64 1
+BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u128, 16
+        PROLOGUE_3_ARGS
+        IEMIMPL_SSE_PROLOGUE
+        SSE_LD_FXSTATE_MXCSR_ONLY A0
+
+        movq    xmm0, A2
+        %1      mm0, xmm0
+        movq    [A1], mm0
+
+        SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
+        IEMIMPL_SSE_EPILOGUE
+        EPILOGUE_3_ARGS
+ENDPROC iemAImpl_ %+ %1 %+ _u128
+%endmacro
+
+IEMIMPL_MEDIA_SSE_MXCSR_U64_U64 cvtps2pi
+IEMIMPL_MEDIA_SSE_MXCSR_U64_U64 cvttps2pi
+
+;
+; All forms of RDRAND and RDSEED
+;
+; @param    A0      Pointer to the destination operand.
+; @param    A1      Pointer to the EFLAGS value (input/output).
+;
+%macro IEMIMPL_RDRAND_RDSEED 3
+BEGINPROC_FASTCALL  iemAImpl_ %+ %1 %+ _u %+ %3, 8
+        PROLOGUE_2_ARGS
+
+        %1      %2
+        mov     [A0], %2
+        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+
+        EPILOGUE_2_ARGS
+ENDPROC             iemAImpl_ %+ %1 %+ _u %+ %3
+%endmacro
+
+IEMIMPL_RDRAND_RDSEED rdrand, ax,  16
+IEMIMPL_RDRAND_RDSEED rdrand, eax, 32
+IEMIMPL_RDRAND_RDSEED rdrand, rax, 64
+IEMIMPL_RDRAND_RDSEED rdseed, ax,  16
+IEMIMPL_RDRAND_RDSEED rdseed, eax, 32
+IEMIMPL_RDRAND_RDSEED rdseed, rax, 64
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp b/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp
new file mode 100644
index 00000000..1196f775
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp
@@ -0,0 +1,17407 @@
+/* $Id: IEMAllAImplC.cpp $ */
+/** @file
+ * IEM - Instruction Implementation in Assembly, portable C variant.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#include "IEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <iprt/errcore.h>
+#include <iprt/x86.h>
+#include <iprt/uint128.h>
+#include <iprt/uint256.h>
+#include <iprt/crc.h>
+
+RT_C_DECLS_BEGIN
+#include <softfloat.h>
+RT_C_DECLS_END
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def IEM_WITHOUT_ASSEMBLY
+ * Enables all the code in this file.
+ */
+#if !defined(IEM_WITHOUT_ASSEMBLY)
+# if defined(RT_ARCH_ARM32) || defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)
+#  define IEM_WITHOUT_ASSEMBLY
+# endif
+#endif
+/* IEM_WITH_ASSEMBLY trumps IEM_WITHOUT_ASSEMBLY for tstIEMAImplAsm purposes. */
+#ifdef IEM_WITH_ASSEMBLY
+# undef IEM_WITHOUT_ASSEMBLY
+#endif
+
+/**
+ * Calculates the signed flag value given a result and it's bit width.
+ *
+ * The signed flag (SF) is a duplication of the most significant bit in the
+ * result.
+ *
+ * @returns X86_EFL_SF or 0.
+ * @param   a_uResult       Unsigned result value.
+ * @param   a_cBitsWidth    The width of the result (8, 16, 32, 64).
+ */
+#define X86_EFL_CALC_SF(a_uResult, a_cBitsWidth) \
+    ( (uint32_t)((a_uResult) >> ((a_cBitsWidth) - X86_EFL_SF_BIT - 1)) & X86_EFL_SF )
+
+/**
+ * Calculates the zero flag value given a result.
+ *
+ * The zero flag (ZF) indicates whether the result is zero or not.
+ *
+ * @returns X86_EFL_ZF or 0.
+ * @param   a_uResult       Unsigned result value.
+ */
+#define X86_EFL_CALC_ZF(a_uResult) \
+    ( (uint32_t)((a_uResult) == 0) << X86_EFL_ZF_BIT )
+
+/**
+ * Extracts the OF flag from a OF calculation result.
+ *
+ * These are typically used by concating with a bitcount.  The problem is that
+ * 8-bit values needs shifting in the other direction than the others.
+ */
+#define X86_EFL_GET_OF_8(a_uValue)  (((uint32_t)(a_uValue) << (X86_EFL_OF_BIT - 8 + 1))  & X86_EFL_OF)
+#define X86_EFL_GET_OF_16(a_uValue) ((uint32_t)((a_uValue) >> (16 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF)
+#define X86_EFL_GET_OF_32(a_uValue) ((uint32_t)((a_uValue) >> (32 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF)
+#define X86_EFL_GET_OF_64(a_uValue) ((uint32_t)((a_uValue) >> (64 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF)
+
+/**
+ * Updates the status bits (CF, PF, AF, ZF, SF, and OF) after arithmetic op.
+ *
+ * @returns Status bits.
+ * @param   a_pfEFlags      Pointer to the 32-bit EFLAGS value to update.
+ * @param   a_uResult       Unsigned result value.
+ * @param   a_uSrc          The source value (for AF calc).
+ * @param   a_uDst          The original destination value (for AF calc).
+ * @param   a_cBitsWidth    The width of the result (8, 16, 32, 64).
+ * @param   a_CfExpr        Bool expression for the carry flag (CF).
+ * @param   a_uSrcOf        The a_uSrc value to use for overflow calculation.
+ */
+#define IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(a_pfEFlags, a_uResult, a_uDst, a_uSrc, a_cBitsWidth, a_CfExpr, a_uSrcOf) \
+    do { \
+        uint32_t fEflTmp = *(a_pfEFlags); \
+        fEflTmp &= ~X86_EFL_STATUS_BITS; \
+        fEflTmp |= (a_CfExpr) << X86_EFL_CF_BIT; \
+        fEflTmp |= g_afParity[(a_uResult) & 0xff]; \
+        fEflTmp |= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uSrc) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \
+        fEflTmp |= X86_EFL_CALC_ZF(a_uResult); \
+        fEflTmp |= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
+        \
+        /* Overflow during ADDition happens when both inputs have the same signed \
+           bit value and the result has a different sign bit value. \
+           \
+           Since subtraction can be rewritten as addition: 2 - 1 == 2 + -1, it \
+           follows that for SUBtraction the signed bit value must differ between \
+           the two inputs and the result's signed bit diff from the first input. \
+           Note! Must xor with sign bit to convert, not do (0 - a_uSrc). \
+           \
+           See also: http://teaching.idallen.com/dat2343/10f/notes/040_overflow.txt */ \
+        fEflTmp |= X86_EFL_GET_OF_ ## a_cBitsWidth(  (  ((uint ## a_cBitsWidth ## _t)~((a_uDst) ^ (a_uSrcOf))) \
+                                                      & RT_BIT_64(a_cBitsWidth - 1)) \
+                                                   & ((a_uResult) ^ (a_uDst)) ); \
+        *(a_pfEFlags) = fEflTmp; \
+    } while (0)
+
+/**
+ * Updates the status bits (CF, PF, AF, ZF, SF, and OF) after a logical op.
+ *
+ * CF and OF are defined to be 0 by logical operations.  AF on the other hand is
+ * undefined.  We do not set AF, as that seems to make the most sense (which
+ * probably makes it the most wrong in real life).
+ *
+ * @returns Status bits.
+ * @param   a_pfEFlags      Pointer to the 32-bit EFLAGS value to update.
+ * @param   a_uResult       Unsigned result value.
+ * @param   a_cBitsWidth    The width of the result (8, 16, 32, 64).
+ * @param   a_fExtra        Additional bits to set.
+ */
+#define IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(a_pfEFlags, a_uResult, a_cBitsWidth, a_fExtra) \
+    do { \
+        uint32_t fEflTmp = *(a_pfEFlags); \
+        fEflTmp &= ~X86_EFL_STATUS_BITS; \
+        fEflTmp |= g_afParity[(a_uResult) & 0xff]; \
+        fEflTmp |= X86_EFL_CALC_ZF(a_uResult); \
+        fEflTmp |= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
+        fEflTmp |= (a_fExtra); \
+        *(a_pfEFlags) = fEflTmp; \
+    } while (0)
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/**
+ * Parity calculation table.
+ *
+ * This is also used by iemAllAImpl.asm.
+ *
+ * The generator code:
+ * @code
+ * #include <stdio.h>
+ *
+ * int main()
+ * {
+ *     unsigned b;
+ *     for (b = 0; b < 256; b++)
+ *     {
+ *         int cOnes = ( b       & 1)
+ *                   + ((b >> 1) & 1)
+ *                   + ((b >> 2) & 1)
+ *                   + ((b >> 3) & 1)
+ *                   + ((b >> 4) & 1)
+ *                   + ((b >> 5) & 1)
+ *                   + ((b >> 6) & 1)
+ *                   + ((b >> 7) & 1);
+ *         printf("    /" "* %#04x = %u%u%u%u%u%u%u%ub *" "/ %s,\n",
+ *                b,
+ *                (b >> 7) & 1,
+ *                (b >> 6) & 1,
+ *                (b >> 5) & 1,
+ *                (b >> 4) & 1,
+ *                (b >> 3) & 1,
+ *                (b >> 2) & 1,
+ *                (b >> 1) & 1,
+ *                 b       & 1,
+ *                cOnes & 1 ? "0" : "X86_EFL_PF");
+ *     }
+ *     return 0;
+ * }
+ * @endcode
+ */
+uint8_t const g_afParity[256] =
+{
+    /* 0000 = 00000000b */ X86_EFL_PF,
+    /* 0x01 = 00000001b */ 0,
+    /* 0x02 = 00000010b */ 0,
+    /* 0x03 = 00000011b */ X86_EFL_PF,
+    /* 0x04 = 00000100b */ 0,
+    /* 0x05 = 00000101b */ X86_EFL_PF,
+    /* 0x06 = 00000110b */ X86_EFL_PF,
+    /* 0x07 = 00000111b */ 0,
+    /* 0x08 = 00001000b */ 0,
+    /* 0x09 = 00001001b */ X86_EFL_PF,
+    /* 0x0a = 00001010b */ X86_EFL_PF,
+    /* 0x0b = 00001011b */ 0,
+    /* 0x0c = 00001100b */ X86_EFL_PF,
+    /* 0x0d = 00001101b */ 0,
+    /* 0x0e = 00001110b */ 0,
+    /* 0x0f = 00001111b */ X86_EFL_PF,
+    /* 0x10 = 00010000b */ 0,
+    /* 0x11 = 00010001b */ X86_EFL_PF,
+    /* 0x12 = 00010010b */ X86_EFL_PF,
+    /* 0x13 = 00010011b */ 0,
+    /* 0x14 = 00010100b */ X86_EFL_PF,
+    /* 0x15 = 00010101b */ 0,
+    /* 0x16 = 00010110b */ 0,
+    /* 0x17 = 00010111b */ X86_EFL_PF,
+    /* 0x18 = 00011000b */ X86_EFL_PF,
+    /* 0x19 = 00011001b */ 0,
+    /* 0x1a = 00011010b */ 0,
+    /* 0x1b = 00011011b */ X86_EFL_PF,
+    /* 0x1c = 00011100b */ 0,
+    /* 0x1d = 00011101b */ X86_EFL_PF,
+    /* 0x1e = 00011110b */ X86_EFL_PF,
+    /* 0x1f = 00011111b */ 0,
+    /* 0x20 = 00100000b */ 0,
+    /* 0x21 = 00100001b */ X86_EFL_PF,
+    /* 0x22 = 00100010b */ X86_EFL_PF,
+    /* 0x23 = 00100011b */ 0,
+    /* 0x24 = 00100100b */ X86_EFL_PF,
+    /* 0x25 = 00100101b */ 0,
+    /* 0x26 = 00100110b */ 0,
+    /* 0x27 = 00100111b */ X86_EFL_PF,
+    /* 0x28 = 00101000b */ X86_EFL_PF,
+    /* 0x29 = 00101001b */ 0,
+    /* 0x2a = 00101010b */ 0,
+    /* 0x2b = 00101011b */ X86_EFL_PF,
+    /* 0x2c = 00101100b */ 0,
+    /* 0x2d = 00101101b */ X86_EFL_PF,
+    /* 0x2e = 00101110b */ X86_EFL_PF,
+    /* 0x2f = 00101111b */ 0,
+    /* 0x30 = 00110000b */ X86_EFL_PF,
+    /* 0x31 = 00110001b */ 0,
+    /* 0x32 = 00110010b */ 0,
+    /* 0x33 = 00110011b */ X86_EFL_PF,
+    /* 0x34 = 00110100b */ 0,
+    /* 0x35 = 00110101b */ X86_EFL_PF,
+    /* 0x36 = 00110110b */ X86_EFL_PF,
+    /* 0x37 = 00110111b */ 0,
+    /* 0x38 = 00111000b */ 0,
+    /* 0x39 = 00111001b */ X86_EFL_PF,
+    /* 0x3a = 00111010b */ X86_EFL_PF,
+    /* 0x3b = 00111011b */ 0,
+    /* 0x3c = 00111100b */ X86_EFL_PF,
+    /* 0x3d = 00111101b */ 0,
+    /* 0x3e = 00111110b */ 0,
+    /* 0x3f = 00111111b */ X86_EFL_PF,
+    /* 0x40 = 01000000b */ 0,
+    /* 0x41 = 01000001b */ X86_EFL_PF,
+    /* 0x42 = 01000010b */ X86_EFL_PF,
+    /* 0x43 = 01000011b */ 0,
+    /* 0x44 = 01000100b */ X86_EFL_PF,
+    /* 0x45 = 01000101b */ 0,
+    /* 0x46 = 01000110b */ 0,
+    /* 0x47 = 01000111b */ X86_EFL_PF,
+    /* 0x48 = 01001000b */ X86_EFL_PF,
+    /* 0x49 = 01001001b */ 0,
+    /* 0x4a = 01001010b */ 0,
+    /* 0x4b = 01001011b */ X86_EFL_PF,
+    /* 0x4c = 01001100b */ 0,
+    /* 0x4d = 01001101b */ X86_EFL_PF,
+    /* 0x4e = 01001110b */ X86_EFL_PF,
+    /* 0x4f = 01001111b */ 0,
+    /* 0x50 = 01010000b */ X86_EFL_PF,
+    /* 0x51 = 01010001b */ 0,
+    /* 0x52 = 01010010b */ 0,
+    /* 0x53 = 01010011b */ X86_EFL_PF,
+    /* 0x54 = 01010100b */ 0,
+    /* 0x55 = 01010101b */ X86_EFL_PF,
+    /* 0x56 = 01010110b */ X86_EFL_PF,
+    /* 0x57 = 01010111b */ 0,
+    /* 0x58 = 01011000b */ 0,
+    /* 0x59 = 01011001b */ X86_EFL_PF,
+    /* 0x5a = 01011010b */ X86_EFL_PF,
+    /* 0x5b = 01011011b */ 0,
+    /* 0x5c = 01011100b */ X86_EFL_PF,
+    /* 0x5d = 01011101b */ 0,
+    /* 0x5e = 01011110b */ 0,
+    /* 0x5f = 01011111b */ X86_EFL_PF,
+    /* 0x60 = 01100000b */ X86_EFL_PF,
+    /* 0x61 = 01100001b */ 0,
+    /* 0x62 = 01100010b */ 0,
+    /* 0x63 = 01100011b */ X86_EFL_PF,
+    /* 0x64 = 01100100b */ 0,
+    /* 0x65 = 01100101b */ X86_EFL_PF,
+    /* 0x66 = 01100110b */ X86_EFL_PF,
+    /* 0x67 = 01100111b */ 0,
+    /* 0x68 = 01101000b */ 0,
+    /* 0x69 = 01101001b */ X86_EFL_PF,
+    /* 0x6a = 01101010b */ X86_EFL_PF,
+    /* 0x6b = 01101011b */ 0,
+    /* 0x6c = 01101100b */ X86_EFL_PF,
+    /* 0x6d = 01101101b */ 0,
+    /* 0x6e = 01101110b */ 0,
+    /* 0x6f = 01101111b */ X86_EFL_PF,
+    /* 0x70 = 01110000b */ 0,
+    /* 0x71 = 01110001b */ X86_EFL_PF,
+    /* 0x72 = 01110010b */ X86_EFL_PF,
+    /* 0x73 = 01110011b */ 0,
+    /* 0x74 = 01110100b */ X86_EFL_PF,
+    /* 0x75 = 01110101b */ 0,
+    /* 0x76 = 01110110b */ 0,
+    /* 0x77 = 01110111b */ X86_EFL_PF,
+    /* 0x78 = 01111000b */ X86_EFL_PF,
+    /* 0x79 = 01111001b */ 0,
+    /* 0x7a = 01111010b */ 0,
+    /* 0x7b = 01111011b */ X86_EFL_PF,
+    /* 0x7c = 01111100b */ 0,
+    /* 0x7d = 01111101b */ X86_EFL_PF,
+    /* 0x7e = 01111110b */ X86_EFL_PF,
+    /* 0x7f = 01111111b */ 0,
+    /* 0x80 = 10000000b */ 0,
+    /* 0x81 = 10000001b */ X86_EFL_PF,
+    /* 0x82 = 10000010b */ X86_EFL_PF,
+    /* 0x83 = 10000011b */ 0,
+    /* 0x84 = 10000100b */ X86_EFL_PF,
+    /* 0x85 = 10000101b */ 0,
+    /* 0x86 = 10000110b */ 0,
+    /* 0x87 = 10000111b */ X86_EFL_PF,
+    /* 0x88 = 10001000b */ X86_EFL_PF,
+    /* 0x89 = 10001001b */ 0,
+    /* 0x8a = 10001010b */ 0,
+    /* 0x8b = 10001011b */ X86_EFL_PF,
+    /* 0x8c = 10001100b */ 0,
+    /* 0x8d = 10001101b */ X86_EFL_PF,
+    /* 0x8e = 10001110b */ X86_EFL_PF,
+    /* 0x8f = 10001111b */ 0,
+    /* 0x90 = 10010000b */ X86_EFL_PF,
+    /* 0x91 = 10010001b */ 0,
+    /* 0x92 = 10010010b */ 0,
+    /* 0x93 = 10010011b */ X86_EFL_PF,
+    /* 0x94 = 10010100b */ 0,
+    /* 0x95 = 10010101b */ X86_EFL_PF,
+    /* 0x96 = 10010110b */ X86_EFL_PF,
+    /* 0x97 = 10010111b */ 0,
+    /* 0x98 = 10011000b */ 0,
+    /* 0x99 = 10011001b */ X86_EFL_PF,
+    /* 0x9a = 10011010b */ X86_EFL_PF,
+    /* 0x9b = 10011011b */ 0,
+    /* 0x9c = 10011100b */ X86_EFL_PF,
+    /* 0x9d = 10011101b */ 0,
+    /* 0x9e = 10011110b */ 0,
+    /* 0x9f = 10011111b */ X86_EFL_PF,
+    /* 0xa0 = 10100000b */ X86_EFL_PF,
+    /* 0xa1 = 10100001b */ 0,
+    /* 0xa2 = 10100010b */ 0,
+    /* 0xa3 = 10100011b */ X86_EFL_PF,
+    /* 0xa4 = 10100100b */ 0,
+    /* 0xa5 = 10100101b */ X86_EFL_PF,
+    /* 0xa6 = 10100110b */ X86_EFL_PF,
+    /* 0xa7 = 10100111b */ 0,
+    /* 0xa8 = 10101000b */ 0,
+    /* 0xa9 = 10101001b */ X86_EFL_PF,
+    /* 0xaa = 10101010b */ X86_EFL_PF,
+    /* 0xab = 10101011b */ 0,
+    /* 0xac = 10101100b */ X86_EFL_PF,
+    /* 0xad = 10101101b */ 0,
+    /* 0xae = 10101110b */ 0,
+    /* 0xaf = 10101111b */ X86_EFL_PF,
+    /* 0xb0 = 10110000b */ 0,
+    /* 0xb1 = 10110001b */ X86_EFL_PF,
+    /* 0xb2 = 10110010b */ X86_EFL_PF,
+    /* 0xb3 = 10110011b */ 0,
+    /* 0xb4 = 10110100b */ X86_EFL_PF,
+    /* 0xb5 = 10110101b */ 0,
+    /* 0xb6 = 10110110b */ 0,
+    /* 0xb7 = 10110111b */ X86_EFL_PF,
+    /* 0xb8 = 10111000b */ X86_EFL_PF,
+    /* 0xb9 = 10111001b */ 0,
+    /* 0xba = 10111010b */ 0,
+    /* 0xbb = 10111011b */ X86_EFL_PF,
+    /* 0xbc = 10111100b */ 0,
+    /* 0xbd = 10111101b */ X86_EFL_PF,
+    /* 0xbe = 10111110b */ X86_EFL_PF,
+    /* 0xbf = 10111111b */ 0,
+    /* 0xc0 = 11000000b */ X86_EFL_PF,
+    /* 0xc1 = 11000001b */ 0,
+    /* 0xc2 = 11000010b */ 0,
+    /* 0xc3 = 11000011b */ X86_EFL_PF,
+    /* 0xc4 = 11000100b */ 0,
+    /* 0xc5 = 11000101b */ X86_EFL_PF,
+    /* 0xc6 = 11000110b */ X86_EFL_PF,
+    /* 0xc7 = 11000111b */ 0,
+    /* 0xc8 = 11001000b */ 0,
+    /* 0xc9 = 11001001b */ X86_EFL_PF,
+    /* 0xca = 11001010b */ X86_EFL_PF,
+    /* 0xcb = 11001011b */ 0,
+    /* 0xcc = 11001100b */ X86_EFL_PF,
+    /* 0xcd = 11001101b */ 0,
+    /* 0xce = 11001110b */ 0,
+    /* 0xcf = 11001111b */ X86_EFL_PF,
+    /* 0xd0 = 11010000b */ 0,
+    /* 0xd1 = 11010001b */ X86_EFL_PF,
+    /* 0xd2 = 11010010b */ X86_EFL_PF,
+    /* 0xd3 = 11010011b */ 0,
+    /* 0xd4 = 11010100b */ X86_EFL_PF,
+    /* 0xd5 = 11010101b */ 0,
+    /* 0xd6 = 11010110b */ 0,
+    /* 0xd7 = 11010111b */ X86_EFL_PF,
+    /* 0xd8 = 11011000b */ X86_EFL_PF,
+    /* 0xd9 = 11011001b */ 0,
+    /* 0xda = 11011010b */ 0,
+    /* 0xdb = 11011011b */ X86_EFL_PF,
+    /* 0xdc = 11011100b */ 0,
+    /* 0xdd = 11011101b */ X86_EFL_PF,
+    /* 0xde = 11011110b */ X86_EFL_PF,
+    /* 0xdf = 11011111b */ 0,
+    /* 0xe0 = 11100000b */ 0,
+    /* 0xe1 = 11100001b */ X86_EFL_PF,
+    /* 0xe2 = 11100010b */ X86_EFL_PF,
+    /* 0xe3 = 11100011b */ 0,
+    /* 0xe4 = 11100100b */ X86_EFL_PF,
+    /* 0xe5 = 11100101b */ 0,
+    /* 0xe6 = 11100110b */ 0,
+    /* 0xe7 = 11100111b */ X86_EFL_PF,
+    /* 0xe8 = 11101000b */ X86_EFL_PF,
+    /* 0xe9 = 11101001b */ 0,
+    /* 0xea = 11101010b */ 0,
+    /* 0xeb = 11101011b */ X86_EFL_PF,
+    /* 0xec = 11101100b */ 0,
+    /* 0xed = 11101101b */ X86_EFL_PF,
+    /* 0xee = 11101110b */ X86_EFL_PF,
+    /* 0xef = 11101111b */ 0,
+    /* 0xf0 = 11110000b */ X86_EFL_PF,
+    /* 0xf1 = 11110001b */ 0,
+    /* 0xf2 = 11110010b */ 0,
+    /* 0xf3 = 11110011b */ X86_EFL_PF,
+    /* 0xf4 = 11110100b */ 0,
+    /* 0xf5 = 11110101b */ X86_EFL_PF,
+    /* 0xf6 = 11110110b */ X86_EFL_PF,
+    /* 0xf7 = 11110111b */ 0,
+    /* 0xf8 = 11111000b */ 0,
+    /* 0xf9 = 11111001b */ X86_EFL_PF,
+    /* 0xfa = 11111010b */ X86_EFL_PF,
+    /* 0xfb = 11111011b */ 0,
+    /* 0xfc = 11111100b */ X86_EFL_PF,
+    /* 0xfd = 11111101b */ 0,
+    /* 0xfe = 11111110b */ 0,
+    /* 0xff = 11111111b */ X86_EFL_PF,
+};
+
+/* for clang: */
+extern const RTFLOAT32U  g_ar32Zero[];
+extern const RTFLOAT64U  g_ar64Zero[];
+extern const RTFLOAT80U  g_ar80Zero[];
+extern const RTFLOAT80U  g_ar80One[];
+extern const RTFLOAT80U  g_r80Indefinite;
+extern const RTFLOAT32U  g_ar32Infinity[];
+extern const RTFLOAT64U  g_ar64Infinity[];
+extern const RTFLOAT80U  g_ar80Infinity[];
+extern const RTFLOAT128U g_r128Ln2;
+extern const RTUINT128U  g_u128Ln2Mantissa;
+extern const RTUINT128U  g_u128Ln2MantissaIntel;
+extern const RTFLOAT128U g_ar128F2xm1HornerConsts[];
+extern const RTFLOAT32U  g_ar32QNaN[];
+extern const RTFLOAT64U  g_ar64QNaN[];
+
+/** Zero values (indexed by fSign). */
+RTFLOAT32U const g_ar32Zero[] = { RTFLOAT32U_INIT_ZERO(0), RTFLOAT32U_INIT_ZERO(1) };
+RTFLOAT64U const g_ar64Zero[] = { RTFLOAT64U_INIT_ZERO(0), RTFLOAT64U_INIT_ZERO(1) };
+RTFLOAT80U const g_ar80Zero[] = { RTFLOAT80U_INIT_ZERO(0), RTFLOAT80U_INIT_ZERO(1) };
+
+/** One values (indexed by fSign). */
+RTFLOAT80U const g_ar80One[] =
+{ RTFLOAT80U_INIT(0, RT_BIT_64(63), RTFLOAT80U_EXP_BIAS), RTFLOAT80U_INIT(1, RT_BIT_64(63), RTFLOAT80U_EXP_BIAS) };
+
+/** Indefinite (negative). */
+RTFLOAT80U const g_r80Indefinite = RTFLOAT80U_INIT_INDEFINITE(1);
+
+/** Infinities (indexed by fSign). */
+RTFLOAT32U const g_ar32Infinity[] = { RTFLOAT32U_INIT_INF(0), RTFLOAT32U_INIT_INF(1) };
+RTFLOAT64U const g_ar64Infinity[] = { RTFLOAT64U_INIT_INF(0), RTFLOAT64U_INIT_INF(1) };
+RTFLOAT80U const g_ar80Infinity[] = { RTFLOAT80U_INIT_INF(0), RTFLOAT80U_INIT_INF(1) };
+
+/** Default QNaNs (indexed by fSign). */
+RTFLOAT32U const g_ar32QNaN[] = { RTFLOAT32U_INIT_QNAN(0), RTFLOAT32U_INIT_QNAN(1) };
+RTFLOAT64U const g_ar64QNaN[] = { RTFLOAT64U_INIT_QNAN(0), RTFLOAT64U_INIT_QNAN(1) };
+
+
+#if 0
+/** 128-bit floating point constant: 2.0 */
+const RTFLOAT128U g_r128Two = RTFLOAT128U_INIT_C(0, 0, 0, RTFLOAT128U_EXP_BIAS + 1);
+#endif
+
+
+/* The next section is generated by tools/IEMGenFpuConstants: */
+
+/** The ln2 constant as 128-bit floating point value.
+ * base-10: 6.93147180559945309417232121458176575e-1
+ * base-16: b.17217f7d1cf79abc9e3b39803f30@-1
+ * base-2 : 1.0110001011100100001011111110111110100011100111101111001101010111100100111100011101100111001100000000011111100110e-1 */
+//const RTFLOAT128U g_r128Ln2 = RTFLOAT128U_INIT_C(0, 0x62e42fefa39e, 0xf35793c7673007e6, 0x3ffe);
+const RTFLOAT128U g_r128Ln2 = RTFLOAT128U_INIT_C(0, 0x62e42fefa39e, 0xf357900000000000, 0x3ffe);
+/** High precision ln2 value.
+ * base-10: 6.931471805599453094172321214581765680747e-1
+ * base-16: b.17217f7d1cf79abc9e3b39803f2f6af0@-1
+ * base-2 : 1.0110001011100100001011111110111110100011100111101111001101010111100100111100011101100111001100000000011111100101111011010101111e-1 */
+const RTUINT128U g_u128Ln2Mantissa = RTUINT128_INIT_C(0xb17217f7d1cf79ab, 0xc9e3b39803f2f6af);
+/** High precision ln2 value, compatible with f2xm1 results on intel 10980XE.
+ * base-10: 6.931471805599453094151379470289064954613e-1
+ * base-16: b.17217f7d1cf79abc0000000000000000@-1
+ * base-2 : 1.0110001011100100001011111110111110100011100111101111001101010111100000000000000000000000000000000000000000000000000000000000000e-1 */
+const RTUINT128U g_u128Ln2MantissaIntel = RTUINT128_INIT_C(0xb17217f7d1cf79ab, 0xc000000000000000);
+
+/** Horner constants for f2xm1 */
+const RTFLOAT128U g_ar128F2xm1HornerConsts[] =
+{
+    /* a0
+     * base-10: 1.00000000000000000000000000000000000e0
+     * base-16: 1.0000000000000000000000000000@0
+     * base-2 : 1.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e0 */
+    RTFLOAT128U_INIT_C(0, 0x000000000000, 0x0000000000000000, 0x3fff),
+    /* a1
+     * base-10: 5.00000000000000000000000000000000000e-1
+     * base-16: 8.0000000000000000000000000000@-1
+     * base-2 : 1.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e-1 */
+    RTFLOAT128U_INIT_C(0, 0x000000000000, 0x0000000000000000, 0x3ffe),
+    /* a2
+     * base-10: 1.66666666666666666666666666666666658e-1
+     * base-16: 2.aaaaaaaaaaaaaaaaaaaaaaaaaaaa@-1
+     * base-2 : 1.0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101e-3 */
+    RTFLOAT128U_INIT_C(0, 0x555555555555, 0x5555555555555555, 0x3ffc),
+    /* a3
+     * base-10: 4.16666666666666666666666666666666646e-2
+     * base-16: a.aaaaaaaaaaaaaaaaaaaaaaaaaaa8@-2
+     * base-2 : 1.0101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101e-5 */
+    RTFLOAT128U_INIT_C(0, 0x555555555555, 0x5555555555555555, 0x3ffa),
+    /* a4
+     * base-10: 8.33333333333333333333333333333333323e-3
+     * base-16: 2.2222222222222222222222222222@-2
+     * base-2 : 1.0001000100010001000100010001000100010001000100010001000100010001000100010001000100010001000100010001000100010001e-7 */
+    RTFLOAT128U_INIT_C(0, 0x111111111111, 0x1111111111111111, 0x3ff8),
+    /* a5
+     * base-10: 1.38888888888888888888888888888888874e-3
+     * base-16: 5.b05b05b05b05b05b05b05b05b058@-3
+     * base-2 : 1.0110110000010110110000010110110000010110110000010110110000010110110000010110110000010110110000010110110000010110e-10 */
+    RTFLOAT128U_INIT_C(0, 0x6c16c16c16c1, 0x6c16c16c16c16c16, 0x3ff5),
+    /* a6
+     * base-10: 1.98412698412698412698412698412698412e-4
+     * base-16: d.00d00d00d00d00d00d00d00d00d0@-4
+     * base-2 : 1.1010000000011010000000011010000000011010000000011010000000011010000000011010000000011010000000011010000000011010e-13 */
+    RTFLOAT128U_INIT_C(0, 0xa01a01a01a01, 0xa01a01a01a01a01a, 0x3ff2),
+    /* a7
+     * base-10: 2.48015873015873015873015873015873015e-5
+     * base-16: 1.a01a01a01a01a01a01a01a01a01a@-4
+     * base-2 : 1.1010000000011010000000011010000000011010000000011010000000011010000000011010000000011010000000011010000000011010e-16 */
+    RTFLOAT128U_INIT_C(0, 0xa01a01a01a01, 0xa01a01a01a01a01a, 0x3fef),
+    /* a8
+     * base-10: 2.75573192239858906525573192239858902e-6
+     * base-16: 2.e3bc74aad8e671f5583911ca002e@-5
+     * base-2 : 1.0111000111011110001110100101010101101100011100110011100011111010101011000001110010001000111001010000000000010111e-19 */
+    RTFLOAT128U_INIT_C(0, 0x71de3a556c73, 0x38faac1c88e50017, 0x3fec),
+    /* a9
+     * base-10: 2.75573192239858906525573192239858865e-7
+     * base-16: 4.9f93edde27d71cbbc05b4fa999e0@-6
+     * base-2 : 1.0010011111100100111110110111011110001001111101011100011100101110111100000001011011010011111010100110011001111000e-22 */
+    RTFLOAT128U_INIT_C(0, 0x27e4fb7789f5, 0xc72ef016d3ea6678, 0x3fe9),
+    /* a10
+     * base-10: 2.50521083854417187750521083854417184e-8
+     * base-16: 6.b99159fd5138e3f9d1f92e0df71c@-7
+     * base-2 : 1.1010111001100100010101100111111101010100010011100011100011111110011101000111111001001011100000110111110111000111e-26 */
+    RTFLOAT128U_INIT_C(0, 0xae64567f544e, 0x38fe747e4b837dc7, 0x3fe5),
+    /* a11
+     * base-10: 2.08767569878680989792100903212014296e-9
+     * base-16: 8.f76c77fc6c4bdaa26d4c3d67f420@-8
+     * base-2 : 1.0001111011101101100011101111111110001101100010010111101101010100010011011010100110000111101011001111111010000100e-29 */
+    RTFLOAT128U_INIT_C(0, 0x1eed8eff8d89, 0x7b544da987acfe84, 0x3fe2),
+    /* a12
+     * base-10: 1.60590438368216145993923771701549472e-10
+     * base-16: b.092309d43684be51c198e91d7b40@-9
+     * base-2 : 1.0110000100100100011000010011101010000110110100001001011111001010001110000011001100011101001000111010111101101000e-33 */
+    RTFLOAT128U_INIT_C(0, 0x6124613a86d0, 0x97ca38331d23af68, 0x3fde),
+    /* a13
+     * base-10: 1.14707455977297247138516979786821043e-11
+     * base-16: c.9cba54603e4e905d6f8a2efd1f20@-10
+     * base-2 : 1.1001001110010111010010101000110000000111110010011101001000001011101011011111000101000101110111111010001111100100e-37 */
+    RTFLOAT128U_INIT_C(0, 0x93974a8c07c9, 0xd20badf145dfa3e4, 0x3fda),
+    /* a14
+     * base-10: 7.64716373181981647590113198578806964e-13
+     * base-16: d.73f9f399dc0f88ec32b587746578@-11
+     * base-2 : 1.1010111001111111001111100111001100111011100000011111000100011101100001100101011010110000111011101000110010101111e-41 */
+    RTFLOAT128U_INIT_C(0, 0xae7f3e733b81, 0xf11d8656b0ee8caf, 0x3fd6),
+    /* a15
+     * base-10: 4.77947733238738529743820749111754352e-14
+     * base-16: d.73f9f399dc0f88ec32b587746578@-12
+     * base-2 : 1.1010111001111111001111100111001100111011100000011111000100011101100001100101011010110000111011101000110010101111e-45 */
+    RTFLOAT128U_INIT_C(0, 0xae7f3e733b81, 0xf11d8656b0ee8caf, 0x3fd2),
+    /* a16
+     * base-10: 2.81145725434552076319894558301031970e-15
+     * base-16: c.a963b81856a53593028cbbb8d7f8@-13
+     * base-2 : 1.1001010100101100011101110000001100001010110101001010011010110010011000000101000110010111011101110001101011111111e-49 */
+    RTFLOAT128U_INIT_C(0, 0x952c77030ad4, 0xa6b2605197771aff, 0x3fce),
+    /* a17
+     * base-10: 1.56192069685862264622163643500573321e-16
+     * base-16: b.413c31dcbecbbdd8024435161550@-14
+     * base-2 : 1.0110100000100111100001100011101110010111110110010111011110111011000000000100100010000110101000101100001010101010e-53 */
+    RTFLOAT128U_INIT_C(0, 0x6827863b97d9, 0x77bb004886a2c2aa, 0x3fca),
+    /* a18
+     * base-10: 8.22063524662432971695598123687227980e-18
+     * base-16: 9.7a4da340a0ab92650f61dbdcb3a0@-15
+     * base-2 : 1.0010111101001001101101000110100000010100000101010111001001001100101000011110110000111011011110111001011001110100e-57 */
+    RTFLOAT128U_INIT_C(0, 0x2f49b4681415, 0x724ca1ec3b7b9674, 0x3fc6),
+    /* a19
+     * base-10: 4.11031762331216485847799061843614006e-19
+     * base-16: 7.950ae900808941ea72b4afe3c2e8@-16
+     * base-2 : 1.1110010101000010101110100100000000100000001000100101000001111010100111001010110100101011111110001111000010111010e-62 */
+    RTFLOAT128U_INIT_C(0, 0xe542ba402022, 0x507a9cad2bf8f0ba, 0x3fc1),
+    /* a20
+     * base-10: 1.95729410633912612308475743735054143e-20
+     * base-16: 5.c6e3bdb73d5c62fbc51bf3b9b8fc@-17
+     * base-2 : 1.0111000110111000111011110110110111001111010101110001100010111110111100010100011011111100111011100110111000111111e-66 */
+    RTFLOAT128U_INIT_C(0, 0x71b8ef6dcf57, 0x18bef146fcee6e3f, 0x3fbd),
+    /* a21
+     * base-10: 8.89679139245057328674889744250246106e-22
+     * base-16: 4.338e5b6dfe14a5143242dfcce3a0@-18
+     * base-2 : 1.0000110011100011100101101101101101111111100001010010100101000101000011001001000010110111111100110011100011101000e-70 */
+    RTFLOAT128U_INIT_C(0, 0x0ce396db7f85, 0x29450c90b7f338e8, 0x3fb9),
+};
+
+
+/*
+ * There are a few 64-bit on 32-bit things we'd rather do in C.  Actually, doing
+ * it all in C is probably safer atm., optimize what's necessary later, maybe.
+ */
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+
+
+/*********************************************************************************************************************************
+*   Binary Operations                                                                                                            *
+*********************************************************************************************************************************/
+
+/*
+ * ADD
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_add_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = uDst + uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uResult < uDst, uSrc);
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_add_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    uint32_t uDst    = *puDst;
+    uint32_t uResult = uDst + uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uResult < uDst, uSrc);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_add_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    uint16_t uDst    = *puDst;
+    uint16_t uResult = uDst + uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uResult < uDst, uSrc);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_add_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags))
+{
+    uint8_t uDst    = *puDst;
+    uint8_t uResult = uDst + uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uResult < uDst, uSrc);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * ADC
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    if (!(*pfEFlags & X86_EFL_CF))
+        iemAImpl_add_u64(puDst, uSrc, pfEFlags);
+    else
+    {
+        uint64_t uDst    = *puDst;
+        uint64_t uResult = uDst + uSrc + 1;
+        *puDst = uResult;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uResult <= uDst, uSrc);
+    }
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    if (!(*pfEFlags & X86_EFL_CF))
+        iemAImpl_add_u32(puDst, uSrc, pfEFlags);
+    else
+    {
+        uint32_t uDst    = *puDst;
+        uint32_t uResult = uDst + uSrc + 1;
+        *puDst = uResult;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uResult <= uDst, uSrc);
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    if (!(*pfEFlags & X86_EFL_CF))
+        iemAImpl_add_u16(puDst, uSrc, pfEFlags);
+    else
+    {
+        uint16_t uDst    = *puDst;
+        uint16_t uResult = uDst + uSrc + 1;
+        *puDst = uResult;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uResult <= uDst, uSrc);
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags))
+{
+    if (!(*pfEFlags & X86_EFL_CF))
+        iemAImpl_add_u8(puDst, uSrc, pfEFlags);
+    else
+    {
+        uint8_t uDst    = *puDst;
+        uint8_t uResult = uDst + uSrc + 1;
+        *puDst = uResult;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uResult <= uDst, uSrc);
+    }
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * SUB
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = uDst - uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uDst < uSrc, uSrc ^ RT_BIT_64(63));
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    uint32_t uDst    = *puDst;
+    uint32_t uResult = uDst - uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uDst < uSrc, uSrc ^ RT_BIT_32(31));
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    uint16_t uDst    = *puDst;
+    uint16_t uResult = uDst - uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uDst < uSrc, uSrc ^ (uint16_t)0x8000);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags))
+{
+    uint8_t uDst    = *puDst;
+    uint8_t uResult = uDst - uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uDst < uSrc, uSrc ^ (uint8_t)0x80);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * SBB
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    if (!(*pfEFlags & X86_EFL_CF))
+        iemAImpl_sub_u64(puDst, uSrc, pfEFlags);
+    else
+    {
+        uint64_t uDst    = *puDst;
+        uint64_t uResult = uDst - uSrc - 1;
+        *puDst = uResult;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uDst <= uSrc, uSrc ^ RT_BIT_64(63));
+    }
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    if (!(*pfEFlags & X86_EFL_CF))
+        iemAImpl_sub_u32(puDst, uSrc, pfEFlags);
+    else
+    {
+        uint32_t uDst    = *puDst;
+        uint32_t uResult = uDst - uSrc - 1;
+        *puDst = uResult;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uDst <= uSrc, uSrc ^ RT_BIT_32(31));
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    if (!(*pfEFlags & X86_EFL_CF))
+        iemAImpl_sub_u16(puDst, uSrc, pfEFlags);
+    else
+    {
+        uint16_t uDst    = *puDst;
+        uint16_t uResult = uDst - uSrc - 1;
+        *puDst = uResult;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uDst <= uSrc, uSrc ^ (uint16_t)0x8000);
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags))
+{
+    if (!(*pfEFlags & X86_EFL_CF))
+        iemAImpl_sub_u8(puDst, uSrc, pfEFlags);
+    else
+    {
+        uint8_t uDst    = *puDst;
+        uint8_t uResult = uDst - uSrc - 1;
+        *puDst = uResult;
+        IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uDst <= uSrc, uSrc ^ (uint8_t)0x80);
+    }
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * OR
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_or_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uResult = *puDst | uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_or_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    uint32_t uResult = *puDst | uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_or_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    uint16_t uResult = *puDst | uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_or_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags))
+{
+    uint8_t uResult = *puDst | uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * XOR
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uResult = *puDst ^ uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    uint32_t uResult = *puDst ^ uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    uint16_t uResult = *puDst ^ uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags))
+{
+    uint8_t uResult = *puDst ^ uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * AND
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_and_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t const uResult = *puDst & uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_and_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    uint32_t const uResult = *puDst & uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_and_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    uint16_t const uResult = *puDst & uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_and_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags))
+{
+    uint8_t const uResult = *puDst & uSrc;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+#endif /* !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * ANDN (BMI1 instruction)
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_andn_u64_fallback,(uint64_t *puDst, uint64_t uSrc1, uint64_t uSrc2, uint32_t *pfEFlags))
+{
+    uint64_t const uResult = ~uSrc1 & uSrc2;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_andn_u32_fallback,(uint32_t *puDst, uint32_t uSrc1, uint32_t uSrc2, uint32_t *pfEFlags))
+{
+    uint32_t const uResult = ~uSrc1 & uSrc2;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
+}
+
+
+#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+IEM_DECL_IMPL_DEF(void, iemAImpl_andn_u64,(uint64_t *puDst, uint64_t uSrc1, uint64_t uSrc2, uint32_t *pfEFlags))
+{
+    iemAImpl_andn_u64_fallback(puDst, uSrc1, uSrc2, pfEFlags);
+}
+#endif
+
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+IEM_DECL_IMPL_DEF(void, iemAImpl_andn_u32,(uint32_t *puDst, uint32_t uSrc1, uint32_t uSrc2, uint32_t *pfEFlags))
+{
+    iemAImpl_andn_u32_fallback(puDst, uSrc1, uSrc2, pfEFlags);
+}
+#endif
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+
+/*
+ * CMP
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uDstTmp = *puDst;
+    iemAImpl_sub_u64(&uDstTmp, uSrc, pfEFlags);
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    uint32_t uDstTmp = *puDst;
+    iemAImpl_sub_u32(&uDstTmp, uSrc, pfEFlags);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    uint16_t uDstTmp = *puDst;
+    iemAImpl_sub_u16(&uDstTmp, uSrc, pfEFlags);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags))
+{
+    uint8_t uDstTmp = *puDst;
+    iemAImpl_sub_u8(&uDstTmp, uSrc, pfEFlags);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * TEST
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_test_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    uint64_t uResult = *puDst & uSrc;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_test_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    uint32_t uResult = *puDst & uSrc;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_test_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    uint16_t uResult = *puDst & uSrc;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_test_u8,(uint8_t *puDst, uint8_t uSrc, uint32_t *pfEFlags))
+{
+    uint8_t uResult = *puDst & uSrc;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * LOCK prefixed variants of the above
+ */
+
+/** 64-bit locked binary operand operation. */
+# define DO_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth) \
+    do { \
+        uint ## a_cBitsWidth ## _t uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \
+        uint ## a_cBitsWidth ## _t uTmp; \
+        uint32_t fEflTmp; \
+        do \
+        { \
+            uTmp = uOld; \
+            fEflTmp = *pfEFlags; \
+            iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth(&uTmp, uSrc, &fEflTmp); \
+        } while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uTmp, uOld, &uOld)); \
+        *pfEFlags = fEflTmp; \
+    } while (0)
+
+
+#define EMIT_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth) \
+    IEM_DECL_IMPL_DEF(void, iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth ##  _locked,(uint ## a_cBitsWidth ## _t *puDst, \
+                                                                                       uint ## a_cBitsWidth ## _t uSrc, \
+                                                                                       uint32_t *pfEFlags)) \
+    { \
+        DO_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth); \
+    }
+
+EMIT_LOCKED_BIN_OP(add, 64)
+EMIT_LOCKED_BIN_OP(adc, 64)
+EMIT_LOCKED_BIN_OP(sub, 64)
+EMIT_LOCKED_BIN_OP(sbb, 64)
+EMIT_LOCKED_BIN_OP(or,  64)
+EMIT_LOCKED_BIN_OP(xor, 64)
+EMIT_LOCKED_BIN_OP(and, 64)
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_LOCKED_BIN_OP(add, 32)
+EMIT_LOCKED_BIN_OP(adc, 32)
+EMIT_LOCKED_BIN_OP(sub, 32)
+EMIT_LOCKED_BIN_OP(sbb, 32)
+EMIT_LOCKED_BIN_OP(or,  32)
+EMIT_LOCKED_BIN_OP(xor, 32)
+EMIT_LOCKED_BIN_OP(and, 32)
+
+EMIT_LOCKED_BIN_OP(add, 16)
+EMIT_LOCKED_BIN_OP(adc, 16)
+EMIT_LOCKED_BIN_OP(sub, 16)
+EMIT_LOCKED_BIN_OP(sbb, 16)
+EMIT_LOCKED_BIN_OP(or,  16)
+EMIT_LOCKED_BIN_OP(xor, 16)
+EMIT_LOCKED_BIN_OP(and, 16)
+
+EMIT_LOCKED_BIN_OP(add, 8)
+EMIT_LOCKED_BIN_OP(adc, 8)
+EMIT_LOCKED_BIN_OP(sub, 8)
+EMIT_LOCKED_BIN_OP(sbb, 8)
+EMIT_LOCKED_BIN_OP(or,  8)
+EMIT_LOCKED_BIN_OP(xor, 8)
+EMIT_LOCKED_BIN_OP(and, 8)
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * Bit operations (same signature as above).
+ */
+
+/*
+ * BT
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  However, it seems they're
+             not modified by either AMD (3990x) or Intel (i9-9980HK). */
+    Assert(uSrc < 64);
+    uint64_t uDst = *puDst;
+    if (uDst & RT_BIT_64(uSrc))
+        *pfEFlags |= X86_EFL_CF;
+    else
+        *pfEFlags &= ~X86_EFL_CF;
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  However, it seems they're
+             not modified by either AMD (3990x) or Intel (i9-9980HK). */
+    Assert(uSrc < 32);
+    uint32_t uDst = *puDst;
+    if (uDst & RT_BIT_32(uSrc))
+        *pfEFlags |= X86_EFL_CF;
+    else
+        *pfEFlags &= ~X86_EFL_CF;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  However, it seems they're
+             not modified by either AMD (3990x) or Intel (i9-9980HK). */
+    Assert(uSrc < 16);
+    uint16_t uDst = *puDst;
+    if (uDst & RT_BIT_32(uSrc))
+        *pfEFlags |= X86_EFL_CF;
+    else
+        *pfEFlags &= ~X86_EFL_CF;
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * BTC
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  However, it seems they're
+             not modified by either AMD (3990x) or Intel (i9-9980HK). */
+    Assert(uSrc < 64);
+    uint64_t fMask = RT_BIT_64(uSrc);
+    uint64_t uDst = *puDst;
+    if (uDst & fMask)
+    {
+        uDst &= ~fMask;
+        *puDst = uDst;
+        *pfEFlags |= X86_EFL_CF;
+    }
+    else
+    {
+        uDst |= fMask;
+        *puDst = uDst;
+        *pfEFlags &= ~X86_EFL_CF;
+    }
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  However, it seems they're
+             not modified by either AMD (3990x) or Intel (i9-9980HK). */
+    Assert(uSrc < 32);
+    uint32_t fMask = RT_BIT_32(uSrc);
+    uint32_t uDst = *puDst;
+    if (uDst & fMask)
+    {
+        uDst &= ~fMask;
+        *puDst = uDst;
+        *pfEFlags |= X86_EFL_CF;
+    }
+    else
+    {
+        uDst |= fMask;
+        *puDst = uDst;
+        *pfEFlags &= ~X86_EFL_CF;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  However, it seems they're
+             not modified by either AMD (3990x) or Intel (i9-9980HK). */
+    Assert(uSrc < 16);
+    uint16_t fMask = RT_BIT_32(uSrc);
+    uint16_t uDst = *puDst;
+    if (uDst & fMask)
+    {
+        uDst &= ~fMask;
+        *puDst = uDst;
+        *pfEFlags |= X86_EFL_CF;
+    }
+    else
+    {
+        uDst |= fMask;
+        *puDst = uDst;
+        *pfEFlags &= ~X86_EFL_CF;
+    }
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * BTR
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  We set them as after an
+       logical operation (AND/OR/whatever). */
+    Assert(uSrc < 64);
+    uint64_t fMask = RT_BIT_64(uSrc);
+    uint64_t uDst = *puDst;
+    if (uDst & fMask)
+    {
+        uDst &= ~fMask;
+        *puDst = uDst;
+        *pfEFlags |= X86_EFL_CF;
+    }
+    else
+        *pfEFlags &= ~X86_EFL_CF;
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  We set them as after an
+       logical operation (AND/OR/whatever). */
+    Assert(uSrc < 32);
+    uint32_t fMask = RT_BIT_32(uSrc);
+    uint32_t uDst = *puDst;
+    if (uDst & fMask)
+    {
+        uDst &= ~fMask;
+        *puDst = uDst;
+        *pfEFlags |= X86_EFL_CF;
+    }
+    else
+        *pfEFlags &= ~X86_EFL_CF;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  We set them as after an
+       logical operation (AND/OR/whatever). */
+    Assert(uSrc < 16);
+    uint16_t fMask = RT_BIT_32(uSrc);
+    uint16_t uDst = *puDst;
+    if (uDst & fMask)
+    {
+        uDst &= ~fMask;
+        *puDst = uDst;
+        *pfEFlags |= X86_EFL_CF;
+    }
+    else
+        *pfEFlags &= ~X86_EFL_CF;
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * BTS
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  We set them as after an
+       logical operation (AND/OR/whatever). */
+    Assert(uSrc < 64);
+    uint64_t fMask = RT_BIT_64(uSrc);
+    uint64_t uDst = *puDst;
+    if (uDst & fMask)
+        *pfEFlags |= X86_EFL_CF;
+    else
+    {
+        uDst |= fMask;
+        *puDst = uDst;
+        *pfEFlags &= ~X86_EFL_CF;
+    }
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  We set them as after an
+       logical operation (AND/OR/whatever). */
+    Assert(uSrc < 32);
+    uint32_t fMask = RT_BIT_32(uSrc);
+    uint32_t uDst = *puDst;
+    if (uDst & fMask)
+        *pfEFlags |= X86_EFL_CF;
+    else
+    {
+        uDst |= fMask;
+        *puDst = uDst;
+        *pfEFlags &= ~X86_EFL_CF;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    /* Note! "undefined" flags: OF, SF, ZF, AF, PF.  We set them as after an
+       logical operation (AND/OR/whatever). */
+    Assert(uSrc < 16);
+    uint16_t fMask = RT_BIT_32(uSrc);
+    uint32_t uDst = *puDst;
+    if (uDst & fMask)
+        *pfEFlags |= X86_EFL_CF;
+    else
+    {
+        uDst |= fMask;
+        *puDst = uDst;
+        *pfEFlags &= ~X86_EFL_CF;
+    }
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+EMIT_LOCKED_BIN_OP(btc, 64)
+EMIT_LOCKED_BIN_OP(btr, 64)
+EMIT_LOCKED_BIN_OP(bts, 64)
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_LOCKED_BIN_OP(btc, 32)
+EMIT_LOCKED_BIN_OP(btr, 32)
+EMIT_LOCKED_BIN_OP(bts, 32)
+
+EMIT_LOCKED_BIN_OP(btc, 16)
+EMIT_LOCKED_BIN_OP(btr, 16)
+EMIT_LOCKED_BIN_OP(bts, 16)
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * Helpers for BSR and BSF.
+ *
+ * Note! "undefined" flags: OF, SF, AF, PF, CF.
+ *       Intel behavior modelled on 10980xe, AMD on 3990X.  Other marchs may
+ *       produce different result (see https://www.sandpile.org/x86/flags.htm),
+ *       but we restrict ourselves to emulating these recent marchs.
+ */
+#define SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlag, a_iBit) do { \
+        unsigned iBit = (a_iBit); \
+        uint32_t fEfl = *pfEFlags & ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); \
+        if (iBit) \
+        { \
+            *puDst    = --iBit; \
+            fEfl     |= g_afParity[iBit]; \
+        } \
+        else \
+            fEfl     |= X86_EFL_ZF | X86_EFL_PF; \
+        *pfEFlags = fEfl; \
+    } while (0)
+#define SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlag, a_iBit) do { \
+        unsigned const iBit = (a_iBit); \
+        if (iBit) \
+        { \
+            *puDst     = iBit - 1; \
+            *pfEFlags &= ~X86_EFL_ZF; \
+        } \
+        else \
+            *pfEFlags |= X86_EFL_ZF; \
+    } while (0)
+
+
+/*
+ * BSF - first (least significant) bit set
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU64(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU64(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64_amd,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU64(uSrc));
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU32(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU32(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32_amd,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU32(uSrc));
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU16(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU16(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16_amd,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU16(uSrc));
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * BSR - last (most significant) bit set
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU64(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU64(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64_amd,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU64(uSrc));
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU32(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU32(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32_amd,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU32(uSrc));
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16_amd,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * Helpers for LZCNT and TZCNT.
+ */
+#define SET_BIT_CNT_SEARCH_RESULT_INTEL(a_puDst, a_uSrc, a_pfEFlags, a_uResult) do { \
+        unsigned const uResult = (a_uResult); \
+        *(a_puDst) = uResult; \
+        uint32_t fEfl = *(a_pfEFlags) & ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); \
+        if (uResult) \
+            fEfl |= g_afParity[uResult]; \
+        else \
+            fEfl |= X86_EFL_ZF | X86_EFL_PF; \
+        if (!a_uSrc) \
+            fEfl |= X86_EFL_CF; \
+        *(a_pfEFlags) = fEfl; \
+    } while (0)
+#define SET_BIT_CNT_SEARCH_RESULT_AMD(a_puDst, a_uSrc, a_pfEFlags, a_uResult) do { \
+        unsigned const uResult = (a_uResult); \
+        *(a_puDst) = uResult; \
+        uint32_t fEfl = *(a_pfEFlags) & ~(X86_EFL_ZF | X86_EFL_CF); \
+        if (!uResult) \
+            fEfl |= X86_EFL_ZF; \
+        if (!a_uSrc) \
+            fEfl |= X86_EFL_CF; \
+        *(a_pfEFlags) = fEfl; \
+    } while (0)
+
+
+/*
+ * LZCNT - count leading zero bits.
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    iemAImpl_lzcnt_u64_intel(puDst, uSrc, pfEFlags);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU64(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u64_amd,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU64(uSrc));
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    iemAImpl_lzcnt_u32_intel(puDst, uSrc, pfEFlags);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU32(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u32_amd,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU32(uSrc));
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    iemAImpl_lzcnt_u16_intel(puDst, uSrc, pfEFlags);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU16(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u16_amd,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU16(uSrc));
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * TZCNT - count leading zero bits.
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    iemAImpl_tzcnt_u64_intel(puDst, uSrc, pfEFlags);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU64(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u64_amd,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU64(uSrc));
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    iemAImpl_tzcnt_u32_intel(puDst, uSrc, pfEFlags);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU32(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u32_amd,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU32(uSrc));
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    iemAImpl_tzcnt_u16_intel(puDst, uSrc, pfEFlags);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_INTEL(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU16(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u16_amd,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
+{
+    SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountTrailingZerosU16(uSrc));
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+#endif /* !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * BEXTR (BMI1 instruction)
+ */
+#define EMIT_BEXTR(a_cBits, a_Type, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_bextr_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc1, \
+                                                                       a_Type uSrc2, uint32_t *pfEFlags)) \
+{ \
+    /* uSrc1 is considered virtually zero extended to 512 bits width. */ \
+    uint32_t      fEfl      = *pfEFlags & ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); \
+    a_Type        uResult; \
+    uint8_t const iFirstBit = (uint8_t)uSrc2; \
+    if (iFirstBit < a_cBits) \
+    { \
+        uResult = uSrc1 >> iFirstBit; \
+        uint8_t const cBits = (uint8_t)(uSrc2 >> 8); \
+        if (cBits < a_cBits) \
+            uResult &= RT_CONCAT(RT_BIT_,a_cBits)(cBits) - 1; \
+        *puDst    = uResult; \
+        if (!uResult) \
+            fEfl |= X86_EFL_ZF; \
+    } \
+    else \
+    { \
+        *puDst  = uResult = 0; \
+        fEfl   |= X86_EFL_ZF; \
+    } \
+    /** @todo complete flag calculations. */ \
+    *pfEFlags = fEfl; \
+}
+
+EMIT_BEXTR(64, uint64_t, _fallback)
+EMIT_BEXTR(32, uint32_t, _fallback)
+#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_BEXTR(64, uint64_t, RT_NOTHING)
+#endif
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_BEXTR(32, uint32_t, RT_NOTHING)
+#endif
+
+/*
+ * BLSR (BMI1 instruction)
+ */
+#define EMIT_BLSR(a_cBits, a_Type, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_blsr_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc, uint32_t *pfEFlags)) \
+{ \
+    uint32_t fEfl1 = *pfEFlags; \
+    uint32_t fEfl2 = fEfl1; \
+    *puDst = uSrc; \
+    iemAImpl_sub_u ## a_cBits(&uSrc, 1, &fEfl1); \
+    iemAImpl_and_u ## a_cBits(puDst, uSrc, &fEfl2); \
+    \
+    /* AMD: The carry flag is from the SUB operation. */ \
+    /* 10890xe: PF always cleared? */ \
+    fEfl2 &= ~(X86_EFL_CF | X86_EFL_PF); \
+    fEfl2 |= fEfl1 & X86_EFL_CF; \
+    *pfEFlags = fEfl2; \
+}
+
+EMIT_BLSR(64, uint64_t, _fallback)
+EMIT_BLSR(32, uint32_t, _fallback)
+#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_BLSR(64, uint64_t, RT_NOTHING)
+#endif
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_BLSR(32, uint32_t, RT_NOTHING)
+#endif
+
+/*
+ * BLSMSK (BMI1 instruction)
+ */
+#define EMIT_BLSMSK(a_cBits, a_Type, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_blsmsk_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc, uint32_t *pfEFlags)) \
+{ \
+    uint32_t fEfl1 = *pfEFlags; \
+    uint32_t fEfl2 = fEfl1; \
+    *puDst = uSrc; \
+    iemAImpl_sub_u ## a_cBits(&uSrc, 1, &fEfl1); \
+    iemAImpl_xor_u ## a_cBits(puDst, uSrc, &fEfl2); \
+    \
+    /* AMD: The carry flag is from the SUB operation. */ \
+    /* 10890xe: PF always cleared? */ \
+    fEfl2 &= ~(X86_EFL_CF | X86_EFL_PF); \
+    fEfl2 |= fEfl1 & X86_EFL_CF; \
+    *pfEFlags = fEfl2; \
+}
+
+EMIT_BLSMSK(64, uint64_t, _fallback)
+EMIT_BLSMSK(32, uint32_t, _fallback)
+#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_BLSMSK(64, uint64_t, RT_NOTHING)
+#endif
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_BLSMSK(32, uint32_t, RT_NOTHING)
+#endif
+
+/*
+ * BLSI (BMI1 instruction)
+ */
+#define EMIT_BLSI(a_cBits, a_Type, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_blsi_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc, uint32_t *pfEFlags)) \
+{ \
+    uint32_t fEfl1 = *pfEFlags; \
+    uint32_t fEfl2 = fEfl1; \
+    *puDst = uSrc; \
+    iemAImpl_neg_u ## a_cBits(&uSrc, &fEfl1); \
+    iemAImpl_and_u ## a_cBits(puDst, uSrc, &fEfl2); \
+    \
+    /* AMD: The carry flag is from the SUB operation. */ \
+    /* 10890xe: PF always cleared? */ \
+    fEfl2 &= ~(X86_EFL_CF | X86_EFL_PF); \
+    fEfl2 |= fEfl1 & X86_EFL_CF; \
+    *pfEFlags = fEfl2; \
+}
+
+EMIT_BLSI(64, uint64_t, _fallback)
+EMIT_BLSI(32, uint32_t, _fallback)
+#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_BLSI(64, uint64_t, RT_NOTHING)
+#endif
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_BLSI(32, uint32_t, RT_NOTHING)
+#endif
+
+/*
+ * BZHI (BMI2 instruction)
+ */
+#define EMIT_BZHI(a_cBits, a_Type, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_bzhi_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc1, \
+                                                                      a_Type uSrc2, uint32_t *pfEFlags)) \
+{ \
+    uint32_t      fEfl      = *pfEFlags & ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); \
+    a_Type        uResult; \
+    uint8_t const iFirstBit = (uint8_t)uSrc2; \
+    if (iFirstBit < a_cBits) \
+        uResult = uSrc1 & (((a_Type)1 << iFirstBit) - 1); \
+    else \
+    { \
+        uResult = uSrc1; \
+        fEfl   |= X86_EFL_CF; \
+    } \
+    *puDst = uResult; \
+    fEfl |= X86_EFL_CALC_ZF(uResult); \
+    fEfl |= X86_EFL_CALC_SF(uResult, a_cBits); \
+    *pfEFlags = fEfl; \
+}
+
+EMIT_BZHI(64, uint64_t, _fallback)
+EMIT_BZHI(32, uint32_t, _fallback)
+#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_BZHI(64, uint64_t, RT_NOTHING)
+#endif
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_BZHI(32, uint32_t, RT_NOTHING)
+#endif
+
+/*
+ * POPCNT
+ */
+RT_ALIGNAS_VAR(64) static uint8_t const g_abBitCounts6[64] =
+{
+    0, 1, 1, 2,  1, 2, 2, 3,  1, 2, 2, 3,  2, 3, 3, 4,
+    1, 2, 2, 3,  2, 3, 3, 4,  2, 3, 3, 4,  3, 4, 4, 5,
+    1, 2, 2, 3,  2, 3, 3, 4,  2, 3, 3, 4,  3, 4, 4, 5,
+    2, 3, 3, 4,  3, 4, 4, 5,  3, 4, 4, 5,  4, 5, 5, 6,
+};
+
+/** @todo Use native popcount where possible and employ some more efficient
+ *        algorithm here (or in asm.h fallback)! */
+
+DECLINLINE(uint8_t) iemPopCountU16(uint16_t u16)
+{
+    return g_abBitCounts6[ u16        & 0x3f]
+        +  g_abBitCounts6[(u16 >> 6)  & 0x3f]
+        +  g_abBitCounts6[(u16 >> 12) & 0x3f];
+}
+
+DECLINLINE(uint8_t) iemPopCountU32(uint32_t u32)
+{
+    return g_abBitCounts6[ u32        & 0x3f]
+        +  g_abBitCounts6[(u32 >> 6)  & 0x3f]
+        +  g_abBitCounts6[(u32 >> 12) & 0x3f]
+        +  g_abBitCounts6[(u32 >> 18) & 0x3f]
+        +  g_abBitCounts6[(u32 >> 24) & 0x3f]
+        +  g_abBitCounts6[(u32 >> 30) & 0x3f];
+}
+
+DECLINLINE(uint8_t) iemPopCountU64(uint64_t u64)
+{
+    return g_abBitCounts6[ u64        & 0x3f]
+        +  g_abBitCounts6[(u64 >> 6)  & 0x3f]
+        +  g_abBitCounts6[(u64 >> 12) & 0x3f]
+        +  g_abBitCounts6[(u64 >> 18) & 0x3f]
+        +  g_abBitCounts6[(u64 >> 24) & 0x3f]
+        +  g_abBitCounts6[(u64 >> 30) & 0x3f]
+        +  g_abBitCounts6[(u64 >> 36) & 0x3f]
+        +  g_abBitCounts6[(u64 >> 42) & 0x3f]
+        +  g_abBitCounts6[(u64 >> 48) & 0x3f]
+        +  g_abBitCounts6[(u64 >> 54) & 0x3f]
+        +  g_abBitCounts6[(u64 >> 60) & 0x3f];
+}
+
+#define EMIT_POPCNT(a_cBits, a_Type, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_popcnt_u,a_cBits,a_Suffix),(a_Type *puDst, a_Type uSrc, uint32_t *pfEFlags)) \
+{ \
+    uint32_t    fEfl = *pfEFlags & ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF); \
+    a_Type      uResult; \
+    if (uSrc) \
+        uResult = iemPopCountU ## a_cBits(uSrc); \
+    else \
+    { \
+        fEfl |= X86_EFL_ZF; \
+        uResult = 0; \
+    } \
+    *puDst    = uResult; \
+    *pfEFlags = fEfl; \
+}
+
+EMIT_POPCNT(64, uint64_t, _fallback)
+EMIT_POPCNT(32, uint32_t, _fallback)
+EMIT_POPCNT(16, uint16_t, _fallback)
+#if defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_POPCNT(64, uint64_t, RT_NOTHING)
+#endif
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_POPCNT(32, uint32_t, RT_NOTHING)
+EMIT_POPCNT(16, uint16_t, RT_NOTHING)
+#endif
+
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+
+/*
+ * XCHG
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_locked,(uint64_t *puMem, uint64_t *puReg))
+{
+#if ARCH_BITS >= 64
+    *puReg = ASMAtomicXchgU64(puMem, *puReg);
+#else
+    uint64_t uOldMem = *puMem;
+    while (!ASMAtomicCmpXchgExU64(puMem, *puReg, uOldMem, &uOldMem))
+        ASMNopPause();
+    *puReg = uOldMem;
+#endif
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_locked,(uint32_t *puMem, uint32_t *puReg))
+{
+    *puReg = ASMAtomicXchgU32(puMem, *puReg);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_locked,(uint16_t *puMem, uint16_t *puReg))
+{
+    *puReg = ASMAtomicXchgU16(puMem, *puReg);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_locked,(uint8_t *puMem, uint8_t *puReg))
+{
+    *puReg = ASMAtomicXchgU8(puMem, *puReg);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/* Unlocked variants for fDisregardLock mode: */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_unlocked,(uint64_t *puMem, uint64_t *puReg))
+{
+    uint64_t const uOld = *puMem;
+    *puMem = *puReg;
+    *puReg = uOld;
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_unlocked,(uint32_t *puMem, uint32_t *puReg))
+{
+    uint32_t const uOld = *puMem;
+    *puMem = *puReg;
+    *puReg = uOld;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_unlocked,(uint16_t *puMem, uint16_t *puReg))
+{
+    uint16_t const uOld = *puMem;
+    *puMem = *puReg;
+    *puReg = uOld;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_unlocked,(uint8_t *puMem, uint8_t *puReg))
+{
+    uint8_t const uOld = *puMem;
+    *puMem = *puReg;
+    *puReg = uOld;
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * XADD and LOCK XADD.
+ */
+#define EMIT_XADD(a_cBitsWidth, a_Type) \
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u ## a_cBitsWidth,(a_Type *puDst, a_Type *puReg, uint32_t *pfEFlags)) \
+{ \
+    a_Type uDst    = *puDst; \
+    a_Type uResult = uDst; \
+    iemAImpl_add_u ## a_cBitsWidth(&uResult, *puReg, pfEFlags); \
+    *puDst = uResult; \
+    *puReg = uDst; \
+} \
+\
+IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u ## a_cBitsWidth ## _locked,(a_Type *puDst, a_Type *puReg, uint32_t *pfEFlags)) \
+{ \
+    a_Type uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \
+    a_Type uResult; \
+    uint32_t fEflTmp; \
+    do \
+    { \
+        uResult = uOld; \
+        fEflTmp = *pfEFlags; \
+        iemAImpl_add_u ## a_cBitsWidth(&uResult, *puReg, &fEflTmp); \
+    } while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uResult, uOld, &uOld)); \
+    *puReg    = uOld; \
+    *pfEFlags = fEflTmp; \
+}
+EMIT_XADD(64, uint64_t)
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_XADD(32, uint32_t)
+EMIT_XADD(16, uint16_t)
+EMIT_XADD(8, uint8_t)
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+#endif
+
+/*
+ * CMPXCHG, CMPXCHG8B, CMPXCHG16B
+ *
+ * Note! We don't have non-locking/atomic cmpxchg primitives, so all cmpxchg
+ *       instructions are emulated as locked.
+ */
+#if defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8_locked, (uint8_t  *pu8Dst,  uint8_t  *puAl,  uint8_t  uSrcReg, uint32_t *pEFlags))
+{
+    uint8_t uOld = *puAl;
+    if (ASMAtomicCmpXchgExU8(pu8Dst, uSrcReg, uOld, puAl))
+        Assert(*puAl == uOld);
+    iemAImpl_cmp_u8(&uOld, *puAl, pEFlags);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16_locked,(uint16_t *pu16Dst, uint16_t *puAx,  uint16_t uSrcReg, uint32_t *pEFlags))
+{
+    uint16_t uOld = *puAx;
+    if (ASMAtomicCmpXchgExU16(pu16Dst, uSrcReg, uOld, puAx))
+        Assert(*puAx == uOld);
+    iemAImpl_cmp_u16(&uOld, *puAx, pEFlags);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32_locked,(uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags))
+{
+    uint32_t uOld = *puEax;
+    if (ASMAtomicCmpXchgExU32(pu32Dst, uSrcReg, uOld, puEax))
+        Assert(*puEax == uOld);
+    iemAImpl_cmp_u32(&uOld, *puEax, pEFlags);
+}
+
+
+# if ARCH_BITS == 32
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags))
+# else
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags))
+# endif
+{
+# if ARCH_BITS == 32
+    uint64_t const uSrcReg = *puSrcReg;
+# endif
+    uint64_t uOld = *puRax;
+    if (ASMAtomicCmpXchgExU64(pu64Dst, uSrcReg, uOld, puRax))
+        Assert(*puRax == uOld);
+    iemAImpl_cmp_u64(&uOld, *puRax, pEFlags);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b_locked,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
+                                                   uint32_t *pEFlags))
+{
+    uint64_t const uNew = pu64EbxEcx->u;
+    uint64_t const uOld = pu64EaxEdx->u;
+    if (ASMAtomicCmpXchgExU64(pu64Dst, uNew, uOld, &pu64EaxEdx->u))
+    {
+        Assert(pu64EaxEdx->u == uOld);
+        *pEFlags |= X86_EFL_ZF;
+    }
+    else
+        *pEFlags &= ~X86_EFL_ZF;
+}
+
+
+# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_ARM64)
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_locked,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
+                                                    uint32_t *pEFlags))
+{
+#  ifdef VBOX_STRICT
+    RTUINT128U const uOld = *pu128RaxRdx;
+#  endif
+#  if defined(RT_ARCH_AMD64)
+    if (ASMAtomicCmpXchgU128v2(&pu128Dst->u, pu128RbxRcx->s.Hi, pu128RbxRcx->s.Lo, pu128RaxRdx->s.Hi, pu128RaxRdx->s.Lo,
+                               &pu128RaxRdx->u))
+#  else
+    if (ASMAtomicCmpXchgU128(&pu128Dst->u, pu128RbxRcx->u, pu128RaxRdx->u, &pu128RaxRdx->u))
+#  endif
+    {
+        Assert(pu128RaxRdx->s.Lo == uOld.s.Lo && pu128RaxRdx->s.Hi == uOld.s.Hi);
+        *pEFlags |= X86_EFL_ZF;
+    }
+    else
+        *pEFlags &= ~X86_EFL_ZF;
+}
+# endif
+
+#endif /* defined(IEM_WITHOUT_ASSEMBLY) */
+
+# if !defined(RT_ARCH_ARM64) /** @todo may need this for unaligned accesses... */
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_fallback,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx,
+                                                      PRTUINT128U pu128RbxRcx, uint32_t *pEFlags))
+{
+    RTUINT128U u128Tmp = *pu128Dst;
+    if (   u128Tmp.s.Lo == pu128RaxRdx->s.Lo
+        && u128Tmp.s.Hi == pu128RaxRdx->s.Hi)
+    {
+        *pu128Dst = *pu128RbxRcx;
+        *pEFlags |= X86_EFL_ZF;
+    }
+    else
+    {
+        *pu128RaxRdx = u128Tmp;
+        *pEFlags &= ~X86_EFL_ZF;
+    }
+}
+#endif /* !RT_ARCH_ARM64 */
+
+#if defined(IEM_WITHOUT_ASSEMBLY)
+
+/* Unlocked versions mapped to the locked ones: */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8,        (uint8_t  *pu8Dst,  uint8_t  *puAl,  uint8_t  uSrcReg, uint32_t *pEFlags))
+{
+    iemAImpl_cmpxchg_u8_locked(pu8Dst, puAl, uSrcReg, pEFlags);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16,       (uint16_t *pu16Dst, uint16_t *puAx,  uint16_t uSrcReg, uint32_t *pEFlags))
+{
+    iemAImpl_cmpxchg_u16_locked(pu16Dst, puAx, uSrcReg, pEFlags);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32,       (uint32_t *pu32Dst, uint32_t *puEax, uint32_t uSrcReg, uint32_t *pEFlags))
+{
+    iemAImpl_cmpxchg_u32_locked(pu32Dst, puEax, uSrcReg, pEFlags);
+}
+
+
+# if ARCH_BITS == 32
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64,       (uint64_t *pu64Dst, uint64_t *puRax, uint64_t *puSrcReg, uint32_t *pEFlags))
+{
+    iemAImpl_cmpxchg_u64_locked(pu64Dst, puRax, puSrcReg, pEFlags);
+}
+# else
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64,       (uint64_t *pu64Dst, uint64_t *puRax, uint64_t uSrcReg, uint32_t *pEFlags))
+{
+    iemAImpl_cmpxchg_u64_locked(pu64Dst, puRax, uSrcReg, pEFlags);
+}
+# endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx, uint32_t *pEFlags))
+{
+    iemAImpl_cmpxchg8b_locked(pu64Dst, pu64EaxEdx, pu64EbxEcx, pEFlags);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
+                                             uint32_t *pEFlags))
+{
+    iemAImpl_cmpxchg16b_locked(pu128Dst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+}
+
+#endif /* defined(IEM_WITHOUT_ASSEMBLY) */
+
+#if (!defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)) \
+ && !defined(DOXYGEN_RUNNING) /* Doxygen has some groking issues here and ends up mixing up input. Not worth tracking down now. */
+
+/*
+ * MUL, IMUL, DIV and IDIV helpers.
+ *
+ * - The U64 versions must use 128-bit intermediates, so we need to abstract the
+ *   division step so we can select between using C operators and
+ *   RTUInt128DivRem/RTUInt128MulU64ByU64.
+ *
+ * - The U8 versions work returns output in AL + AH instead of xDX + xAX, with the
+ *   IDIV/DIV taking all the input in AX too.  This means we have to abstract some
+ *   input loads and the result storing.
+ */
+
+DECLINLINE(void) RTUInt128DivRemByU64(PRTUINT128U pQuotient, PRTUINT128U pRemainder, PCRTUINT128U pDividend, uint64_t u64Divisor)
+{
+# ifdef __GNUC__ /* GCC maybe really annoying in function. */
+    pQuotient->s.Lo = 0;
+    pQuotient->s.Hi = 0;
+# endif
+    RTUINT128U Divisor;
+    Divisor.s.Lo = u64Divisor;
+    Divisor.s.Hi = 0;
+    RTUInt128DivRem(pQuotient, pRemainder, pDividend, &Divisor);
+}
+
+# define DIV_LOAD(a_Dividend)  \
+    a_Dividend.s.Lo = *puA, a_Dividend.s.Hi = *puD
+# define DIV_LOAD_U8(a_Dividend) \
+    a_Dividend.u = *puAX
+
+# define DIV_STORE(a_Quotient, a_uReminder)    *puA  = (a_Quotient),    *puD = (a_uReminder)
+# define DIV_STORE_U8(a_Quotient, a_uReminder) *puAX = (uint8_t)(a_Quotient) | ((uint16_t)(a_uReminder) << 8)
+
+# define MUL_LOAD_F1()                         *puA
+# define MUL_LOAD_F1_U8()                      ((uint8_t)*puAX)
+
+# define MUL_STORE(a_Result)                   *puA  = (a_Result).s.Lo, *puD = (a_Result).s.Hi
+# define MUL_STORE_U8(a_Result)                *puAX = a_Result.u
+
+# define MULDIV_NEG(a_Value, a_cBitsWidth2x) \
+    (a_Value).u = UINT ## a_cBitsWidth2x ## _C(0) - (a_Value).u
+# define MULDIV_NEG_U128(a_Value, a_cBitsWidth2x) \
+    RTUInt128AssignNeg(&(a_Value))
+
+# define MULDIV_MUL(a_Result, a_Factor1, a_Factor2, a_cBitsWidth2x) \
+    (a_Result).u = (uint ## a_cBitsWidth2x ## _t)(a_Factor1) * (a_Factor2)
+# define MULDIV_MUL_U128(a_Result, a_Factor1, a_Factor2, a_cBitsWidth2x) \
+    RTUInt128MulU64ByU64(&(a_Result), a_Factor1, a_Factor2);
+
+# define MULDIV_MODDIV(a_Quotient, a_Remainder, a_Dividend, a_uDivisor) \
+    a_Quotient.u = (a_Dividend).u / (a_uDivisor), \
+    a_Remainder.u = (a_Dividend).u % (a_uDivisor)
+# define MULDIV_MODDIV_U128(a_Quotient, a_Remainder, a_Dividend, a_uDivisor) \
+    RTUInt128DivRemByU64(&a_Quotient, &a_Remainder, &a_Dividend, a_uDivisor)
+
+
+/*
+ * MUL
+ */
+# define EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_mul_u,a_cBitsWidth,a_Suffix), a_Args) \
+{ \
+    RTUINT ## a_cBitsWidth2x ## U Result; \
+    a_fnMul(Result, a_fnLoadF1(), uFactor, a_cBitsWidth2x); \
+    a_fnStore(Result); \
+    \
+    /* Calc EFLAGS: */ \
+    uint32_t fEfl = *pfEFlags; \
+    if (a_fIntelFlags) \
+    { /* Intel: 6700K and 10980XE behavior */ \
+        fEfl &= ~(X86_EFL_SF | X86_EFL_CF | X86_EFL_OF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_PF); \
+        if (Result.s.Lo & RT_BIT_64(a_cBitsWidth - 1)) \
+            fEfl |= X86_EFL_SF; \
+        fEfl |= g_afParity[Result.s.Lo & 0xff]; \
+        if (Result.s.Hi != 0) \
+            fEfl |= X86_EFL_CF | X86_EFL_OF; \
+    } \
+    else \
+    { /* AMD: 3990X */ \
+        if (Result.s.Hi != 0) \
+            fEfl |= X86_EFL_CF | X86_EFL_OF; \
+        else \
+            fEfl &= ~(X86_EFL_CF | X86_EFL_OF); \
+    } \
+    *pfEFlags = fEfl; \
+    return 0; \
+} \
+
+# define EMIT_MUL(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul) \
+    EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, RT_NOTHING, 1) \
+    EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, _intel,     1) \
+    EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, _amd,       0) \
+
+# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */
+EMIT_MUL(64, 128, (uint64_t *puA, uint64_t *puD, uint64_t uFactor, uint32_t *pfEFlags), (puA, puD, uFactor, pfEFlags),
+         MUL_LOAD_F1, MUL_STORE, MULDIV_MUL_U128)
+#  if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_MUL(32, 64, (uint32_t *puA, uint32_t *puD, uint32_t uFactor, uint32_t *pfEFlags),  (puA, puD, uFactor, pfEFlags),
+         MUL_LOAD_F1, MUL_STORE, MULDIV_MUL)
+EMIT_MUL(16, 32, (uint16_t *puA, uint16_t *puD, uint16_t uFactor, uint32_t *pfEFlags),  (puA, puD, uFactor, pfEFlags),
+         MUL_LOAD_F1, MUL_STORE, MULDIV_MUL)
+EMIT_MUL(8, 16, (uint16_t *puAX, uint8_t uFactor, uint32_t *pfEFlags),                  (puAX,     uFactor, pfEFlags),
+         MUL_LOAD_F1_U8, MUL_STORE_U8, MULDIV_MUL)
+#  endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+# endif /* !DOXYGEN_RUNNING */
+
+/*
+ * MULX
+ */
+# define EMIT_MULX(a_cBitsWidth, a_cBitsWidth2x, a_uType, a_fnMul, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_mulx_u,a_cBitsWidth,a_Suffix), \
+    (a_uType *puDst1, a_uType *puDst2, a_uType uSrc1, a_uType uSrc2)) \
+{ \
+    RTUINT ## a_cBitsWidth2x ## U Result; \
+    a_fnMul(Result, uSrc1, uSrc2, a_cBitsWidth2x); \
+    *puDst2 = Result.s.Lo; /* Lower part first, as we should return the high part when puDst2 == puDst1. */ \
+    *puDst1 = Result.s.Hi; \
+} \
+
+# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */
+EMIT_MULX(64, 128, uint64_t, MULDIV_MUL_U128, RT_NOTHING)
+EMIT_MULX(64, 128, uint64_t, MULDIV_MUL_U128, _fallback)
+#  if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_MULX(32, 64,  uint32_t, MULDIV_MUL, RT_NOTHING)
+EMIT_MULX(32, 64,  uint32_t, MULDIV_MUL, _fallback)
+#  endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+# endif /* !DOXYGEN_RUNNING */
+
+
+/*
+ * IMUL
+ *
+ * The SF, ZF, AF and PF flags are "undefined". AMD (3990x) leaves these
+ * flags as is.  Whereas Intel skylake (6700K and 10980X (Cascade Lake)) always
+ * clear AF and ZF and calculates SF and PF as per the lower half of the result.
+ */
+# define EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, \
+                         a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_imul_u,a_cBitsWidth,a_Suffix),a_Args) \
+{ \
+    RTUINT ## a_cBitsWidth2x ## U Result; \
+    uint32_t fEfl = *pfEFlags & ~(X86_EFL_CF | X86_EFL_OF); \
+    \
+    uint ## a_cBitsWidth ## _t const uFactor1 = a_fnLoadF1(); \
+    if (!(uFactor1 & RT_BIT_64(a_cBitsWidth - 1))) \
+    { \
+        if (!(uFactor2 & RT_BIT_64(a_cBitsWidth - 1))) \
+        { \
+            a_fnMul(Result, uFactor1, uFactor2, a_cBitsWidth2x); \
+            if (Result.s.Hi != 0 || Result.s.Lo >= RT_BIT_64(a_cBitsWidth - 1)) \
+                fEfl |= X86_EFL_CF | X86_EFL_OF; \
+        } \
+        else \
+        { \
+            uint ## a_cBitsWidth ## _t const uPositiveFactor2 = UINT ## a_cBitsWidth ## _C(0) - uFactor2; \
+            a_fnMul(Result, uFactor1, uPositiveFactor2, a_cBitsWidth2x); \
+            if (Result.s.Hi != 0 || Result.s.Lo > RT_BIT_64(a_cBitsWidth - 1)) \
+                fEfl |= X86_EFL_CF | X86_EFL_OF; \
+            a_fnNeg(Result, a_cBitsWidth2x); \
+        } \
+    } \
+    else \
+    { \
+        if (!(uFactor2 & RT_BIT_64(a_cBitsWidth - 1))) \
+        { \
+            uint ## a_cBitsWidth ## _t const uPositiveFactor1 = UINT ## a_cBitsWidth ## _C(0) - uFactor1; \
+            a_fnMul(Result, uPositiveFactor1, uFactor2, a_cBitsWidth2x); \
+            if (Result.s.Hi != 0 || Result.s.Lo > RT_BIT_64(a_cBitsWidth - 1)) \
+                fEfl |= X86_EFL_CF | X86_EFL_OF; \
+            a_fnNeg(Result, a_cBitsWidth2x); \
+        } \
+        else \
+        { \
+            uint ## a_cBitsWidth ## _t const uPositiveFactor1 = UINT ## a_cBitsWidth ## _C(0) - uFactor1; \
+            uint ## a_cBitsWidth ## _t const uPositiveFactor2 = UINT ## a_cBitsWidth ## _C(0) - uFactor2; \
+            a_fnMul(Result, uPositiveFactor1, uPositiveFactor2, a_cBitsWidth2x); \
+            if (Result.s.Hi != 0 || Result.s.Lo >= RT_BIT_64(a_cBitsWidth - 1)) \
+                fEfl |= X86_EFL_CF | X86_EFL_OF; \
+        } \
+    } \
+    a_fnStore(Result); \
+    \
+    if (a_fIntelFlags) \
+    { \
+        fEfl &= ~(X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF | X86_EFL_PF); \
+        if (Result.s.Lo & RT_BIT_64(a_cBitsWidth - 1)) \
+            fEfl |= X86_EFL_SF;  \
+        fEfl |= g_afParity[Result.s.Lo & 0xff]; \
+    } \
+    *pfEFlags = fEfl; \
+    return 0; \
+}
+# define EMIT_IMUL(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul) \
+    EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, RT_NOTHING, 1) \
+    EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, _intel,     1) \
+    EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, _amd,       0)
+
+# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */
+EMIT_IMUL(64, 128, (uint64_t *puA, uint64_t *puD, uint64_t uFactor2, uint32_t *pfEFlags), (puA, puD, uFactor2, pfEFlags),
+          MUL_LOAD_F1, MUL_STORE, MULDIV_NEG_U128, MULDIV_MUL_U128)
+#  if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_IMUL(32, 64, (uint32_t *puA, uint32_t *puD, uint32_t uFactor2, uint32_t *pfEFlags),  (puA, puD, uFactor2, pfEFlags),
+          MUL_LOAD_F1, MUL_STORE, MULDIV_NEG, MULDIV_MUL)
+EMIT_IMUL(16, 32, (uint16_t *puA, uint16_t *puD, uint16_t uFactor2, uint32_t *pfEFlags),  (puA, puD, uFactor2, pfEFlags),
+          MUL_LOAD_F1, MUL_STORE, MULDIV_NEG, MULDIV_MUL)
+EMIT_IMUL(8, 16, (uint16_t *puAX, uint8_t uFactor2, uint32_t *pfEFlags),                  (puAX,     uFactor2, pfEFlags),
+          MUL_LOAD_F1_U8, MUL_STORE_U8, MULDIV_NEG, MULDIV_MUL)
+#  endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+# endif /* !DOXYGEN_RUNNING */
+
+
+/*
+ * IMUL with two operands are mapped onto the three operand variant, ignoring
+ * the high part of the product.
+ */
+# define EMIT_IMUL_TWO(a_cBits, a_uType) \
+IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits,(a_uType *puDst, a_uType uSrc, uint32_t *pfEFlags)) \
+{ \
+    a_uType uIgn; \
+    iemAImpl_imul_u ## a_cBits(puDst, &uIgn, uSrc, pfEFlags); \
+} \
+\
+IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits ## _intel,(a_uType *puDst, a_uType uSrc, uint32_t *pfEFlags)) \
+{ \
+    a_uType uIgn; \
+    iemAImpl_imul_u ## a_cBits ## _intel(puDst, &uIgn, uSrc, pfEFlags); \
+} \
+\
+IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits ## _amd,(a_uType *puDst, a_uType uSrc, uint32_t *pfEFlags)) \
+{ \
+    a_uType uIgn; \
+    iemAImpl_imul_u ## a_cBits ## _amd(puDst, &uIgn, uSrc, pfEFlags); \
+}
+
+EMIT_IMUL_TWO(64, uint64_t)
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_IMUL_TWO(32, uint32_t)
+EMIT_IMUL_TWO(16, uint16_t)
+# endif
+
+
+/*
+ * DIV
+ */
+# define EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, \
+                        a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_div_u,a_cBitsWidth,a_Suffix),a_Args) \
+{ \
+    RTUINT ## a_cBitsWidth2x ## U Dividend; \
+    a_fnLoad(Dividend); \
+    if (   uDivisor != 0 \
+        && Dividend.s.Hi < uDivisor) \
+    { \
+        RTUINT ## a_cBitsWidth2x ## U Remainder, Quotient; \
+        a_fnDivRem(Quotient, Remainder, Dividend, uDivisor); \
+        a_fnStore(Quotient.s.Lo, Remainder.s.Lo); \
+        \
+        /* Calc EFLAGS: Intel 6700K and 10980XE leaves them alone.  AMD 3990X sets AF and clears PF, ZF and SF. */ \
+        if (!a_fIntelFlags) \
+            *pfEFlags = (*pfEFlags & ~(X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF)) | X86_EFL_AF; \
+        return 0; \
+    } \
+    /* #DE */ \
+    return -1; \
+}
+# define EMIT_DIV(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem) \
+    EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, RT_NOTHING, 1) \
+    EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, _intel,     1) \
+    EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, _amd,       0)
+
+# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */
+EMIT_DIV(64,128,(uint64_t *puA, uint64_t *puD, uint64_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
+         DIV_LOAD, DIV_STORE, MULDIV_MODDIV_U128)
+#  if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_DIV(32,64, (uint32_t *puA, uint32_t *puD, uint32_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
+         DIV_LOAD, DIV_STORE, MULDIV_MODDIV)
+EMIT_DIV(16,32, (uint16_t *puA, uint16_t *puD, uint16_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
+         DIV_LOAD, DIV_STORE, MULDIV_MODDIV)
+EMIT_DIV(8,16,  (uint16_t *puAX, uint8_t uDivisor, uint32_t *pfEFlags),                (puAX,     uDivisor, pfEFlags),
+         DIV_LOAD_U8, DIV_STORE_U8, MULDIV_MODDIV)
+#  endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+# endif /* !DOXYGEN_RUNNING */
+
+
+/*
+ * IDIV
+ *
+ * EFLAGS are ignored and left as-is by Intel 6700K and 10980XE.  AMD 3990X will
+ * set AF and clear PF, ZF and SF just like it does for DIV.
+ *
+ */
+# define EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, \
+                         a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_idiv_u,a_cBitsWidth,a_Suffix),a_Args) \
+{ \
+    /* Note! Skylake leaves all flags alone. */ \
+    \
+    /** @todo overflow checks */ \
+    if (uDivisor != 0) \
+    { \
+        /* \
+         * Convert to unsigned division. \
+         */ \
+        RTUINT ## a_cBitsWidth2x ## U Dividend; \
+        a_fnLoad(Dividend); \
+        bool const fSignedDividend = RT_BOOL(Dividend.s.Hi & RT_BIT_64(a_cBitsWidth - 1)); \
+        if (fSignedDividend) \
+            a_fnNeg(Dividend, a_cBitsWidth2x); \
+        \
+        uint ## a_cBitsWidth ## _t uDivisorPositive; \
+        if (!(uDivisor & RT_BIT_64(a_cBitsWidth - 1))) \
+            uDivisorPositive = uDivisor; \
+        else \
+            uDivisorPositive = UINT ## a_cBitsWidth ## _C(0) - uDivisor; \
+        \
+        RTUINT ## a_cBitsWidth2x ## U Remainder, Quotient; \
+        a_fnDivRem(Quotient, Remainder, Dividend, uDivisorPositive); \
+        \
+        /* \
+         * Setup the result, checking for overflows. \
+         */ \
+        if (!(uDivisor & RT_BIT_64(a_cBitsWidth - 1))) \
+        { \
+            if (!fSignedDividend) \
+            { \
+                /* Positive divisor, positive dividend => result positive. */ \
+                if (Quotient.s.Hi == 0 && Quotient.s.Lo <= (uint ## a_cBitsWidth ## _t)INT ## a_cBitsWidth ## _MAX) \
+                { \
+                    a_fnStore(Quotient.s.Lo, Remainder.s.Lo); \
+                    if (!a_fIntelFlags) \
+                        *pfEFlags = (*pfEFlags & ~(X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF)) | X86_EFL_AF; \
+                    return 0; \
+                } \
+            } \
+            else \
+            { \
+                /* Positive divisor, negative dividend => result negative. */ \
+                if (Quotient.s.Hi == 0 && Quotient.s.Lo <= RT_BIT_64(a_cBitsWidth - 1)) \
+                { \
+                    a_fnStore(UINT ## a_cBitsWidth ## _C(0) - Quotient.s.Lo, UINT ## a_cBitsWidth ## _C(0) - Remainder.s.Lo); \
+                    if (!a_fIntelFlags) \
+                        *pfEFlags = (*pfEFlags & ~(X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF)) | X86_EFL_AF; \
+                    return 0; \
+                } \
+            } \
+        } \
+        else \
+        { \
+            if (!fSignedDividend) \
+            { \
+                /* Negative divisor, positive dividend => negative quotient, positive remainder. */ \
+                if (Quotient.s.Hi == 0 && Quotient.s.Lo <= RT_BIT_64(a_cBitsWidth - 1)) \
+                { \
+                    a_fnStore(UINT ## a_cBitsWidth ## _C(0) - Quotient.s.Lo, Remainder.s.Lo); \
+                    if (!a_fIntelFlags) \
+                        *pfEFlags = (*pfEFlags & ~(X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF)) | X86_EFL_AF; \
+                    return 0; \
+                } \
+            } \
+            else \
+            { \
+                /* Negative divisor, negative dividend => positive quotient, negative remainder. */ \
+                if (Quotient.s.Hi == 0 && Quotient.s.Lo <= (uint ## a_cBitsWidth ## _t)INT ## a_cBitsWidth ## _MAX) \
+                { \
+                    a_fnStore(Quotient.s.Lo, UINT ## a_cBitsWidth ## _C(0) - Remainder.s.Lo); \
+                    if (!a_fIntelFlags) \
+                        *pfEFlags = (*pfEFlags & ~(X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF)) | X86_EFL_AF; \
+                    return 0; \
+                } \
+            } \
+        } \
+    } \
+    /* #DE */ \
+    return -1; \
+}
+# define EMIT_IDIV(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem) \
+     EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, RT_NOTHING, 1) \
+     EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, _intel,     1) \
+     EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, _amd,       0)
+
+# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */
+EMIT_IDIV(64,128,(uint64_t *puA, uint64_t *puD, uint64_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
+          DIV_LOAD, DIV_STORE, MULDIV_NEG_U128, MULDIV_MODDIV_U128)
+#  if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_IDIV(32,64,(uint32_t *puA, uint32_t *puD, uint32_t uDivisor, uint32_t *pfEFlags),  (puA, puD, uDivisor, pfEFlags),
+          DIV_LOAD, DIV_STORE, MULDIV_NEG, MULDIV_MODDIV)
+EMIT_IDIV(16,32,(uint16_t *puA, uint16_t *puD, uint16_t uDivisor, uint32_t *pfEFlags),  (puA, puD, uDivisor, pfEFlags),
+          DIV_LOAD, DIV_STORE, MULDIV_NEG, MULDIV_MODDIV)
+EMIT_IDIV(8,16,(uint16_t *puAX, uint8_t uDivisor, uint32_t *pfEFlags),                  (puAX,     uDivisor, pfEFlags),
+          DIV_LOAD_U8, DIV_STORE_U8, MULDIV_NEG, MULDIV_MODDIV)
+#  endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+# endif /* !DOXYGEN_RUNNING */
+
+#endif /* (!defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)) && !defined(DOXYGEN_RUNNING) */
+
+
+/*********************************************************************************************************************************
+*   Unary operations.                                                                                                            *
+*********************************************************************************************************************************/
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+
+/** @def IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC
+ * Updates the status bits (CF, PF, AF, ZF, SF, and OF) for an INC or DEC instruction.
+ *
+ * CF is NOT modified for hysterical raisins (allegedly for carrying and
+ * borrowing in arithmetic loops on intel 8008).
+ *
+ * @returns Status bits.
+ * @param   a_pfEFlags      Pointer to the 32-bit EFLAGS value to update.
+ * @param   a_uResult       Unsigned result value.
+ * @param   a_uDst          The original destination value (for AF calc).
+ * @param   a_cBitsWidth    The width of the result (8, 16, 32, 64).
+ * @param   a_OfMethod      0 for INC-style, 1 for DEC-style.
+ */
+#define IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(a_pfEFlags, a_uResult, a_uDst, a_cBitsWidth, a_OfMethod) \
+    do { \
+        uint32_t fEflTmp = *(a_pfEFlags); \
+        fEflTmp &= ~X86_EFL_STATUS_BITS | X86_EFL_CF; \
+        fEflTmp |= g_afParity[(a_uResult) & 0xff]; \
+        fEflTmp |= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \
+        fEflTmp |= X86_EFL_CALC_ZF(a_uResult); \
+        fEflTmp |= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
+        fEflTmp |= X86_EFL_GET_OF_ ## a_cBitsWidth(a_OfMethod == 0 ? (((a_uDst) ^ RT_BIT_64(a_cBitsWidth - 1)) & (a_uResult)) \
+                                                                   : ((a_uDst) & ((a_uResult) ^ RT_BIT_64(a_cBitsWidth - 1))) ); \
+        *(a_pfEFlags) = fEflTmp; \
+    } while (0)
+
+/*
+ * INC
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u64,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = uDst + 1;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 64, 0 /*INC*/);
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u32,(uint32_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint32_t uDst    = *puDst;
+    uint32_t uResult = uDst + 1;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 32, 0 /*INC*/);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u16,(uint16_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint16_t uDst    = *puDst;
+    uint16_t uResult = uDst + 1;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 16, 0 /*INC*/);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u8,(uint8_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint8_t uDst    = *puDst;
+    uint8_t uResult = uDst + 1;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 8, 0 /*INC*/);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * DEC
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u64,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = uDst - 1;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 64, 1 /*INC*/);
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u32,(uint32_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint32_t uDst    = *puDst;
+    uint32_t uResult = uDst - 1;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 32, 1 /*INC*/);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u16,(uint16_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint16_t uDst    = *puDst;
+    uint16_t uResult = uDst - 1;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 16, 1 /*INC*/);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u8,(uint8_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint8_t uDst    = *puDst;
+    uint8_t uResult = uDst - 1;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 8, 1 /*INC*/);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * NOT
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_not_u64,(uint64_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = ~uDst;
+    *puDst = uResult;
+    /* EFLAGS are not modified. */
+    RT_NOREF_PV(pfEFlags);
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_not_u32,(uint32_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint32_t uDst    = *puDst;
+    uint32_t uResult = ~uDst;
+    *puDst = uResult;
+    /* EFLAGS are not modified. */
+    RT_NOREF_PV(pfEFlags);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_not_u16,(uint16_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint16_t uDst    = *puDst;
+    uint16_t uResult = ~uDst;
+    *puDst = uResult;
+    /* EFLAGS are not modified. */
+    RT_NOREF_PV(pfEFlags);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_not_u8,(uint8_t  *puDst,  uint32_t *pfEFlags))
+{
+    uint8_t uDst    = *puDst;
+    uint8_t uResult = ~uDst;
+    *puDst = uResult;
+    /* EFLAGS are not modified. */
+    RT_NOREF_PV(pfEFlags);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*
+ * NEG
+ */
+
+/**
+ * Updates the status bits (CF, PF, AF, ZF, SF, and OF) for an NEG instruction.
+ *
+ * @returns Status bits.
+ * @param   a_pfEFlags      Pointer to the 32-bit EFLAGS value to update.
+ * @param   a_uResult       Unsigned result value.
+ * @param   a_uDst          The original destination value (for AF calc).
+ * @param   a_cBitsWidth    The width of the result (8, 16, 32, 64).
+ */
+#define IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(a_pfEFlags, a_uResult, a_uDst, a_cBitsWidth) \
+    do { \
+        uint32_t fEflTmp = *(a_pfEFlags); \
+        fEflTmp &= ~X86_EFL_STATUS_BITS & ~X86_EFL_CF; \
+        fEflTmp |= ((a_uDst) != 0) << X86_EFL_CF_BIT; \
+        fEflTmp |= g_afParity[(a_uResult) & 0xff]; \
+        fEflTmp |= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \
+        fEflTmp |= X86_EFL_CALC_ZF(a_uResult); \
+        fEflTmp |= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
+        fEflTmp |= X86_EFL_GET_OF_ ## a_cBitsWidth((a_uDst) & (a_uResult)); \
+        *(a_pfEFlags) = fEflTmp; \
+    } while (0)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u64,(uint64_t *puDst, uint32_t *pfEFlags))
+{
+    uint64_t uDst    = *puDst;
+    uint64_t uResult = (uint64_t)0 - uDst;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 64);
+}
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u32,(uint32_t *puDst, uint32_t *pfEFlags))
+{
+    uint32_t uDst    = *puDst;
+    uint32_t uResult = (uint32_t)0 - uDst;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 32);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u16,(uint16_t *puDst, uint32_t *pfEFlags))
+{
+    uint16_t uDst    = *puDst;
+    uint16_t uResult = (uint16_t)0 - uDst;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 16);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u8,(uint8_t *puDst, uint32_t *pfEFlags))
+{
+    uint8_t uDst    = *puDst;
+    uint8_t uResult = (uint8_t)0 - uDst;
+    *puDst = uResult;
+    IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 8);
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+/*
+ * Locked variants.
+ */
+
+/** Emit a function for doing a locked unary operand operation. */
+# define EMIT_LOCKED_UNARY_OP(a_Mnemonic, a_cBitsWidth) \
+    IEM_DECL_IMPL_DEF(void, iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth ## _locked,(uint ## a_cBitsWidth ## _t *puDst, \
+                                                                                      uint32_t *pfEFlags)) \
+    { \
+        uint ## a_cBitsWidth ## _t uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \
+        uint ## a_cBitsWidth ## _t uTmp; \
+        uint32_t fEflTmp; \
+        do \
+        { \
+            uTmp = uOld; \
+            fEflTmp = *pfEFlags; \
+            iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth(&uTmp, &fEflTmp); \
+        } while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uTmp, uOld, &uOld)); \
+        *pfEFlags = fEflTmp; \
+    }
+
+EMIT_LOCKED_UNARY_OP(inc, 64)
+EMIT_LOCKED_UNARY_OP(dec, 64)
+EMIT_LOCKED_UNARY_OP(not, 64)
+EMIT_LOCKED_UNARY_OP(neg, 64)
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_LOCKED_UNARY_OP(inc, 32)
+EMIT_LOCKED_UNARY_OP(dec, 32)
+EMIT_LOCKED_UNARY_OP(not, 32)
+EMIT_LOCKED_UNARY_OP(neg, 32)
+
+EMIT_LOCKED_UNARY_OP(inc, 16)
+EMIT_LOCKED_UNARY_OP(dec, 16)
+EMIT_LOCKED_UNARY_OP(not, 16)
+EMIT_LOCKED_UNARY_OP(neg, 16)
+
+EMIT_LOCKED_UNARY_OP(inc, 8)
+EMIT_LOCKED_UNARY_OP(dec, 8)
+EMIT_LOCKED_UNARY_OP(not, 8)
+EMIT_LOCKED_UNARY_OP(neg, 8)
+# endif
+
+#endif /* !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+/*********************************************************************************************************************************
+*   Shifting and Rotating                                                                                                        *
+*********************************************************************************************************************************/
+
+/*
+ * ROL
+ */
+#define EMIT_ROL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags, a_fnHlp) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rol_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \
+{ \
+    cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
+    if (cShift) \
+    { \
+        if (a_cBitsWidth < 32) \
+            cShift &= a_cBitsWidth - 1; \
+        a_uType const uDst    = *puDst; \
+        a_uType const uResult = a_fnHlp(uDst, cShift); \
+        *puDst = uResult; \
+        \
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement \
+           it the same way as for 1 bit shifts. */ \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        uint32_t fEfl = *pfEFlags; \
+        fEfl &= ~(X86_EFL_CF | X86_EFL_OF); \
+        uint32_t const fCarry = (uResult & X86_EFL_CF); \
+        fEfl |= fCarry; \
+        if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \
+            fEfl |= ((uResult >> (a_cBitsWidth - 1)) ^ fCarry) << X86_EFL_OF_BIT; \
+        else                /* Intel 10980XE: According to the first sub-shift: */ \
+            fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_ROL(64, uint64_t, RT_NOTHING, 1, ASMRotateLeftU64)
+#endif
+EMIT_ROL(64, uint64_t, _intel,     1, ASMRotateLeftU64)
+EMIT_ROL(64, uint64_t, _amd,       0, ASMRotateLeftU64)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_ROL(32, uint32_t, RT_NOTHING, 1, ASMRotateLeftU32)
+#endif
+EMIT_ROL(32, uint32_t, _intel,     1, ASMRotateLeftU32)
+EMIT_ROL(32, uint32_t, _amd,       0, ASMRotateLeftU32)
+
+DECL_FORCE_INLINE(uint16_t) iemAImpl_rol_u16_hlp(uint16_t uValue, uint8_t cShift)
+{
+    return (uValue << cShift) | (uValue >> (16 - cShift));
+}
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_ROL(16, uint16_t, RT_NOTHING, 1, iemAImpl_rol_u16_hlp)
+#endif
+EMIT_ROL(16, uint16_t, _intel,     1, iemAImpl_rol_u16_hlp)
+EMIT_ROL(16, uint16_t, _amd,       0, iemAImpl_rol_u16_hlp)
+
+DECL_FORCE_INLINE(uint8_t) iemAImpl_rol_u8_hlp(uint8_t uValue, uint8_t cShift)
+{
+    return (uValue << cShift) | (uValue >> (8 - cShift));
+}
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_ROL(8,  uint8_t,  RT_NOTHING, 1, iemAImpl_rol_u8_hlp)
+#endif
+EMIT_ROL(8,  uint8_t,  _intel,     1, iemAImpl_rol_u8_hlp)
+EMIT_ROL(8,  uint8_t,  _amd,       0, iemAImpl_rol_u8_hlp)
+
+
+/*
+ * ROR
+ */
+#define EMIT_ROR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags, a_fnHlp) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_ror_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \
+{ \
+    cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
+    if (cShift) \
+    { \
+        if (a_cBitsWidth < 32) \
+            cShift &= a_cBitsWidth - 1; \
+        a_uType const uDst    = *puDst; \
+        a_uType const uResult = a_fnHlp(uDst, cShift); \
+        *puDst = uResult; \
+        \
+        /* Calc EFLAGS:  */ \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        uint32_t fEfl = *pfEFlags; \
+        fEfl &= ~(X86_EFL_CF | X86_EFL_OF); \
+        uint32_t const fCarry = (uResult >> ((a_cBitsWidth) - 1)) & X86_EFL_CF; \
+        fEfl |= fCarry; \
+        if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \
+            fEfl |= (((uResult >> ((a_cBitsWidth) - 2)) ^ fCarry) & 1) << X86_EFL_OF_BIT; \
+        else                /* Intel 10980XE: According to the first sub-shift: */ \
+            fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << (a_cBitsWidth - 1))); \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_ROR(64, uint64_t, RT_NOTHING, 1, ASMRotateRightU64)
+#endif
+EMIT_ROR(64, uint64_t, _intel,     1, ASMRotateRightU64)
+EMIT_ROR(64, uint64_t, _amd,       0, ASMRotateRightU64)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_ROR(32, uint32_t, RT_NOTHING, 1, ASMRotateRightU32)
+#endif
+EMIT_ROR(32, uint32_t, _intel,     1, ASMRotateRightU32)
+EMIT_ROR(32, uint32_t, _amd,       0, ASMRotateRightU32)
+
+DECL_FORCE_INLINE(uint16_t) iemAImpl_ror_u16_hlp(uint16_t uValue, uint8_t cShift)
+{
+    return (uValue >> cShift) | (uValue << (16 - cShift));
+}
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_ROR(16, uint16_t, RT_NOTHING, 1, iemAImpl_ror_u16_hlp)
+#endif
+EMIT_ROR(16, uint16_t, _intel,     1, iemAImpl_ror_u16_hlp)
+EMIT_ROR(16, uint16_t, _amd,       0, iemAImpl_ror_u16_hlp)
+
+DECL_FORCE_INLINE(uint8_t) iemAImpl_ror_u8_hlp(uint8_t uValue, uint8_t cShift)
+{
+    return (uValue >> cShift) | (uValue << (8 - cShift));
+}
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_ROR(8,  uint8_t,  RT_NOTHING, 1, iemAImpl_ror_u8_hlp)
+#endif
+EMIT_ROR(8,  uint8_t,  _intel,     1, iemAImpl_ror_u8_hlp)
+EMIT_ROR(8,  uint8_t,  _amd,       0, iemAImpl_ror_u8_hlp)
+
+
+/*
+ * RCL
+ */
+#define EMIT_RCL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rcl_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \
+{ \
+    cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
+    if (a_cBitsWidth < 32 && a_fIntelFlags) \
+        cShift %= a_cBitsWidth + 1; \
+    if (cShift) \
+    { \
+        if (a_cBitsWidth < 32 && !a_fIntelFlags) \
+            cShift %= a_cBitsWidth + 1; \
+        a_uType const uDst    = *puDst; \
+        a_uType       uResult = uDst << cShift; \
+        if (cShift > 1) \
+            uResult |= uDst >> (a_cBitsWidth + 1 - cShift); \
+        \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        uint32_t fEfl     = *pfEFlags; \
+        uint32_t fInCarry = fEfl & X86_EFL_CF; \
+        uResult |= (a_uType)fInCarry << (cShift - 1); \
+        \
+        *puDst = uResult; \
+        \
+        /* Calc EFLAGS. */ \
+        fEfl &= ~(X86_EFL_CF | X86_EFL_OF); \
+        uint32_t const fOutCarry = a_cBitsWidth >= 32 || a_fIntelFlags || cShift \
+                                 ? (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF : fInCarry; \
+        fEfl |= fOutCarry; \
+        if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \
+            fEfl |= ((uResult >> (a_cBitsWidth - 1)) ^ fOutCarry) << X86_EFL_OF_BIT; \
+        else                /* Intel 10980XE: According to the first sub-shift: */ \
+            fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_RCL(64, uint64_t, RT_NOTHING, 1)
+#endif
+EMIT_RCL(64, uint64_t, _intel,     1)
+EMIT_RCL(64, uint64_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_RCL(32, uint32_t, RT_NOTHING, 1)
+#endif
+EMIT_RCL(32, uint32_t, _intel,     1)
+EMIT_RCL(32, uint32_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_RCL(16, uint16_t, RT_NOTHING, 1)
+#endif
+EMIT_RCL(16, uint16_t, _intel,     1)
+EMIT_RCL(16, uint16_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_RCL(8,  uint8_t,  RT_NOTHING, 1)
+#endif
+EMIT_RCL(8,  uint8_t,  _intel,     1)
+EMIT_RCL(8,  uint8_t,  _amd,       0)
+
+
+/*
+ * RCR
+ */
+#define EMIT_RCR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rcr_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \
+{ \
+    cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
+    if (a_cBitsWidth < 32 && a_fIntelFlags) \
+        cShift %= a_cBitsWidth + 1; \
+    if (cShift) \
+    { \
+        if (a_cBitsWidth < 32 && !a_fIntelFlags) \
+            cShift %= a_cBitsWidth + 1; \
+        a_uType const uDst    = *puDst; \
+        a_uType       uResult = uDst >> cShift; \
+        if (cShift > 1) \
+            uResult |= uDst << (a_cBitsWidth + 1 - cShift); \
+        \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        uint32_t fEfl     = *pfEFlags; \
+        uint32_t fInCarry = fEfl & X86_EFL_CF; \
+        uResult |= (a_uType)fInCarry << (a_cBitsWidth - cShift); \
+        *puDst = uResult; \
+        \
+        /* Calc EFLAGS.  The OF bit is undefined if cShift > 1, we implement \
+           it the same way as for 1 bit shifts. */ \
+        fEfl &= ~(X86_EFL_CF | X86_EFL_OF); \
+        uint32_t const fOutCarry = a_cBitsWidth >= 32 || a_fIntelFlags || cShift \
+                                 ? (uDst >> (cShift - 1)) & X86_EFL_CF : fInCarry; \
+        fEfl |= fOutCarry; \
+        if (!a_fIntelFlags) /* AMD 3990X: XOR two most signficant bits of the result: */ \
+            fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uResult ^ (uResult << 1));  \
+        else                /* Intel 10980XE: same as AMD, but only for the first sub-shift: */ \
+            fEfl |= (fInCarry ^ (uint32_t)(uDst >> (a_cBitsWidth - 1))) << X86_EFL_OF_BIT; \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_RCR(64, uint64_t, RT_NOTHING, 1)
+#endif
+EMIT_RCR(64, uint64_t, _intel,     1)
+EMIT_RCR(64, uint64_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_RCR(32, uint32_t, RT_NOTHING, 1)
+#endif
+EMIT_RCR(32, uint32_t, _intel,     1)
+EMIT_RCR(32, uint32_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_RCR(16, uint16_t, RT_NOTHING, 1)
+#endif
+EMIT_RCR(16, uint16_t, _intel,     1)
+EMIT_RCR(16, uint16_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_RCR(8,  uint8_t,  RT_NOTHING, 1)
+#endif
+EMIT_RCR(8,  uint8_t,  _intel,     1)
+EMIT_RCR(8,  uint8_t,  _amd,       0)
+
+
+/*
+ * SHL
+ */
+#define EMIT_SHL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shl_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \
+{ \
+    cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
+    if (cShift) \
+    { \
+        a_uType const uDst  = *puDst; \
+        a_uType       uResult = uDst << cShift; \
+        *puDst = uResult; \
+        \
+        /* Calc EFLAGS. */ \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
+        uint32_t fCarry = (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF; \
+        fEfl |= fCarry; \
+        if (!a_fIntelFlags) \
+            fEfl |= ((uResult >> (a_cBitsWidth - 1)) ^ fCarry) << X86_EFL_OF_BIT; /* AMD 3990X: Last shift result. */ \
+        else \
+            fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); /* Intel 10980XE: First shift result. */ \
+        fEfl |= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
+        fEfl |= X86_EFL_CALC_ZF(uResult); \
+        fEfl |= g_afParity[uResult & 0xff]; \
+        if (!a_fIntelFlags) \
+            fEfl |= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHL(64, uint64_t, RT_NOTHING, 1)
+#endif
+EMIT_SHL(64, uint64_t, _intel,     1)
+EMIT_SHL(64, uint64_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHL(32, uint32_t, RT_NOTHING, 1)
+#endif
+EMIT_SHL(32, uint32_t, _intel,     1)
+EMIT_SHL(32, uint32_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHL(16, uint16_t, RT_NOTHING, 1)
+#endif
+EMIT_SHL(16, uint16_t, _intel,     1)
+EMIT_SHL(16, uint16_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHL(8,  uint8_t,  RT_NOTHING, 1)
+#endif
+EMIT_SHL(8,  uint8_t,  _intel,     1)
+EMIT_SHL(8,  uint8_t,  _amd,       0)
+
+
+/*
+ * SHR
+ */
+#define EMIT_SHR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shr_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \
+{ \
+    cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
+    if (cShift) \
+    { \
+        a_uType const uDst    = *puDst; \
+        a_uType       uResult = uDst >> cShift; \
+        *puDst = uResult; \
+        \
+        /* Calc EFLAGS. */ \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
+        fEfl |= (uDst >> (cShift - 1)) & X86_EFL_CF; \
+        if (a_fIntelFlags || cShift == 1) /* AMD 3990x does what intel documents; Intel 10980XE does this for all shift counts. */ \
+            fEfl |= (uDst >> (a_cBitsWidth - 1)) << X86_EFL_OF_BIT; \
+        fEfl |= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
+        fEfl |= X86_EFL_CALC_ZF(uResult); \
+        fEfl |= g_afParity[uResult & 0xff]; \
+        if (!a_fIntelFlags) \
+            fEfl |= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHR(64, uint64_t, RT_NOTHING, 1)
+#endif
+EMIT_SHR(64, uint64_t, _intel,     1)
+EMIT_SHR(64, uint64_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHR(32, uint32_t, RT_NOTHING, 1)
+#endif
+EMIT_SHR(32, uint32_t, _intel,     1)
+EMIT_SHR(32, uint32_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHR(16, uint16_t, RT_NOTHING, 1)
+#endif
+EMIT_SHR(16, uint16_t, _intel,     1)
+EMIT_SHR(16, uint16_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHR(8,  uint8_t,  RT_NOTHING, 1)
+#endif
+EMIT_SHR(8,  uint8_t,  _intel,     1)
+EMIT_SHR(8,  uint8_t,  _amd,       0)
+
+
+/*
+ * SAR
+ */
+#define EMIT_SAR(a_cBitsWidth, a_uType, a_iType, a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_sar_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, uint8_t cShift, uint32_t *pfEFlags)) \
+{ \
+    cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
+    if (cShift) \
+    { \
+        a_iType const iDst    = (a_iType)*puDst; \
+        a_uType       uResult = iDst >> cShift; \
+        *puDst = uResult; \
+        \
+        /* Calc EFLAGS. \
+           Note! The OF flag is always zero because the result never differs from the input. */ \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
+        fEfl |= (iDst >> (cShift - 1)) & X86_EFL_CF; \
+        fEfl |= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
+        fEfl |= X86_EFL_CALC_ZF(uResult); \
+        fEfl |= g_afParity[uResult & 0xff]; \
+        if (!a_fIntelFlags) \
+            fEfl |= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SAR(64, uint64_t, int64_t, RT_NOTHING, 1)
+#endif
+EMIT_SAR(64, uint64_t, int64_t, _intel,     1)
+EMIT_SAR(64, uint64_t, int64_t, _amd,       0)
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SAR(32, uint32_t, int32_t, RT_NOTHING, 1)
+#endif
+EMIT_SAR(32, uint32_t, int32_t, _intel,     1)
+EMIT_SAR(32, uint32_t, int32_t, _amd,       0)
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SAR(16, uint16_t, int16_t, RT_NOTHING, 1)
+#endif
+EMIT_SAR(16, uint16_t, int16_t, _intel,     1)
+EMIT_SAR(16, uint16_t, int16_t, _amd,       0)
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SAR(8,  uint8_t,  int8_t,  RT_NOTHING, 1)
+#endif
+EMIT_SAR(8,  uint8_t,  int8_t,  _intel,     1)
+EMIT_SAR(8,  uint8_t,  int8_t,  _amd,       0)
+
+
+/*
+ * SHLD
+ *
+ *  - CF is the last bit shifted out of puDst.
+ *  - AF is always cleared by Intel 10980XE.
+ *  - AF is always set by AMD 3990X.
+ *  - OF is set according to the first shift on Intel 10980XE, it seems.
+ *  - OF is set according to the last sub-shift on AMD 3990X.
+ *  - ZF, SF and PF are calculated according to the result by both vendors.
+ *
+ * For 16-bit shifts the count mask isn't 15, but 31, and the CPU will
+ * pick either the source register or the destination register for input bits
+ * when going beyond 16.  According to https://www.sandpile.org/x86/flags.htm
+ * intel has changed behaviour here several times.  We implement what current
+ * skylake based does for now, we can extend this later as needed.
+ */
+#define EMIT_SHLD(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shld_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, uint8_t cShift, \
+                                                                           uint32_t *pfEFlags)) \
+{ \
+    cShift &= a_cBitsWidth - 1; \
+    if (cShift) \
+    { \
+        a_uType const uDst    = *puDst; \
+        a_uType       uResult = uDst << cShift; \
+        uResult |= uSrc >> (a_cBitsWidth - cShift); \
+        *puDst = uResult; \
+        \
+        /* CALC EFLAGS: */ \
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
+        if (a_fIntelFlags) \
+            /* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \
+            fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1));  \
+        else \
+        {   /* AMD 3990X: Set according to last shift. AF always set. */ \
+            fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth((uDst << (cShift - 1)) ^ uResult); \
+            fEfl |= X86_EFL_AF; \
+        } \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        fEfl |= (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF; /* CF = last bit shifted out */ \
+        fEfl |= g_afParity[uResult & 0xff]; \
+        fEfl |= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
+        fEfl |= X86_EFL_CALC_ZF(uResult); \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHLD(64, uint64_t, RT_NOTHING, 1)
+#endif
+EMIT_SHLD(64, uint64_t, _intel,     1)
+EMIT_SHLD(64, uint64_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHLD(32, uint32_t, RT_NOTHING, 1)
+#endif
+EMIT_SHLD(32, uint32_t, _intel,     1)
+EMIT_SHLD(32, uint32_t, _amd,       0)
+
+#define EMIT_SHLD_16(a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT(iemAImpl_shld_u16,a_Suffix),(uint16_t *puDst, uint16_t uSrc, uint8_t cShift, uint32_t *pfEFlags)) \
+{ \
+    cShift &= 31; \
+    if (cShift) \
+    { \
+        uint16_t const uDst    = *puDst; \
+        uint64_t const uTmp    = a_fIntelFlags \
+                               ? ((uint64_t)uDst << 32) | ((uint32_t)uSrc << 16) | uDst \
+                               : ((uint64_t)uDst << 32) | ((uint32_t)uSrc << 16) | uSrc; \
+        uint16_t const uResult = (uint16_t)((uTmp << cShift) >> 32); \
+        *puDst = uResult; \
+        \
+        /* CALC EFLAGS: */ \
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        if (a_fIntelFlags) \
+        { \
+            fEfl |= (uTmp >> (48 - cShift)) & X86_EFL_CF; /* CF = last bit shifted out of the combined operand */ \
+            /* Intel 6700K & 10980XE: OF is et according to the first shift. AF always cleared. */ \
+            fEfl |= X86_EFL_GET_OF_16(uDst ^ (uDst << 1));  \
+        } \
+        else \
+        { \
+            /* AMD 3990X: OF is set according to last shift, with some weirdness. AF always set. CF = last bit shifted out of uDst. */ \
+            if (cShift < 16) \
+            { \
+                fEfl |= (uDst >> (16 - cShift)) & X86_EFL_CF;  \
+                fEfl |= X86_EFL_GET_OF_16((uDst << (cShift - 1)) ^ uResult); \
+            } \
+            else \
+            { \
+                if (cShift == 16) \
+                    fEfl |= uDst & X86_EFL_CF; \
+                fEfl |= X86_EFL_GET_OF_16((uDst << (cShift - 1)) ^ 0); \
+            } \
+            fEfl |= X86_EFL_AF; \
+        } \
+        fEfl |= g_afParity[uResult & 0xff]; \
+        fEfl |= X86_EFL_CALC_SF(uResult, 16); \
+        fEfl |= X86_EFL_CALC_ZF(uResult); \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHLD_16(RT_NOTHING, 1)
+#endif
+EMIT_SHLD_16(_intel,     1)
+EMIT_SHLD_16(_amd,       0)
+
+
+/*
+ * SHRD
+ *
+ * EFLAGS behaviour seems to be the same as with SHLD:
+ *  - CF is the last bit shifted out of puDst.
+ *  - AF is always cleared by Intel 10980XE.
+ *  - AF is always set by AMD 3990X.
+ *  - OF is set according to the first shift on Intel 10980XE, it seems.
+ *  - OF is set according to the last sub-shift on AMD 3990X.
+ *  - ZF, SF and PF are calculated according to the result by both vendors.
+ *
+ * For 16-bit shifts the count mask isn't 15, but 31, and the CPU will
+ * pick either the source register or the destination register for input bits
+ * when going beyond 16.  According to https://www.sandpile.org/x86/flags.htm
+ * intel has changed behaviour here several times.  We implement what current
+ * skylake based does for now, we can extend this later as needed.
+ */
+#define EMIT_SHRD(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shrd_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, uint8_t cShift, uint32_t *pfEFlags)) \
+{ \
+    cShift &= a_cBitsWidth - 1; \
+    if (cShift) \
+    { \
+        a_uType const uDst    = *puDst; \
+        a_uType       uResult = uDst >> cShift; \
+        uResult |= uSrc << (a_cBitsWidth - cShift); \
+        *puDst = uResult; \
+        \
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        fEfl |= (uDst >> (cShift - 1)) & X86_EFL_CF; \
+        if (a_fIntelFlags) \
+            /* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \
+            fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uSrc << (a_cBitsWidth - 1))); \
+        else \
+        {   /* AMD 3990X: Set according to last shift. AF always set. */ \
+            if (cShift > 1) /* Set according to last shift. */ \
+                fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth((uSrc << (a_cBitsWidth - cShift + 1)) ^ uResult); \
+            else \
+                fEfl |= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ uResult); \
+            fEfl |= X86_EFL_AF; \
+        } \
+        fEfl |= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
+        fEfl |= X86_EFL_CALC_ZF(uResult); \
+        fEfl |= g_afParity[uResult & 0xff]; \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHRD(64, uint64_t, RT_NOTHING, 1)
+#endif
+EMIT_SHRD(64, uint64_t, _intel,     1)
+EMIT_SHRD(64, uint64_t, _amd,       0)
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHRD(32, uint32_t, RT_NOTHING, 1)
+#endif
+EMIT_SHRD(32, uint32_t, _intel,     1)
+EMIT_SHRD(32, uint32_t, _amd,       0)
+
+#define EMIT_SHRD_16(a_Suffix, a_fIntelFlags) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT(iemAImpl_shrd_u16,a_Suffix),(uint16_t *puDst, uint16_t uSrc, uint8_t cShift, uint32_t *pfEFlags)) \
+{ \
+    cShift &= 31; \
+    if (cShift) \
+    { \
+        uint16_t const uDst    = *puDst; \
+        uint64_t const uTmp    = a_fIntelFlags \
+                               ? uDst | ((uint32_t)uSrc << 16) | ((uint64_t)uDst << 32) \
+                               : uDst | ((uint32_t)uSrc << 16) | ((uint64_t)uSrc << 32); \
+        uint16_t const uResult = (uint16_t)(uTmp >> cShift); \
+        *puDst = uResult; \
+        \
+        uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
+        AssertCompile(X86_EFL_CF_BIT == 0); \
+        if (a_fIntelFlags) \
+        { \
+            /* Intel 10980XE: The CF is the last shifted out of the combined uTmp operand. */ \
+            fEfl |= (uTmp >> (cShift - 1)) & X86_EFL_CF; \
+            /* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \
+            fEfl |= X86_EFL_GET_OF_16(uDst ^ (uSrc << 15)); \
+        } \
+        else \
+        { \
+            /* AMD 3990X: CF flag seems to be last bit shifted out of uDst, not the combined uSrc:uSrc:uDst operand. */ \
+            fEfl |= (uDst >> (cShift - 1)) & X86_EFL_CF; \
+            /* AMD 3990X: Set according to last shift. AF always set. */ \
+            if (cShift > 1) /* Set according to last shift. */ \
+                fEfl |= X86_EFL_GET_OF_16((uint16_t)(uTmp >> (cShift - 1)) ^ uResult); \
+            else \
+                fEfl |= X86_EFL_GET_OF_16(uDst ^ uResult); \
+            fEfl |= X86_EFL_AF; \
+        } \
+        fEfl |= X86_EFL_CALC_SF(uResult, 16); \
+        fEfl |= X86_EFL_CALC_ZF(uResult); \
+        fEfl |= g_afParity[uResult & 0xff]; \
+        *pfEFlags = fEfl; \
+    } \
+}
+
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHRD_16(RT_NOTHING, 1)
+#endif
+EMIT_SHRD_16(_intel,     1)
+EMIT_SHRD_16(_amd,       0)
+
+
+/*
+ * RORX (BMI2)
+ */
+#define EMIT_RORX(a_cBitsWidth, a_uType, a_fnHlp) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT(iemAImpl_rorx_u,a_cBitsWidth),(a_uType *puDst, a_uType uSrc, a_uType cShift)) \
+{ \
+    *puDst = a_fnHlp(uSrc, cShift & (a_cBitsWidth - 1)); \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_RORX(64, uint64_t, ASMRotateRightU64)
+#endif
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_RORX(32, uint32_t, ASMRotateRightU32)
+#endif
+
+
+/*
+ * SHLX (BMI2)
+ */
+#define EMIT_SHLX(a_cBitsWidth, a_uType, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shlx_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, a_uType cShift)) \
+{ \
+    cShift &= a_cBitsWidth - 1; \
+    *puDst = uSrc << cShift; \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHLX(64, uint64_t, RT_NOTHING)
+EMIT_SHLX(64, uint64_t, _fallback)
+#endif
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHLX(32, uint32_t, RT_NOTHING)
+EMIT_SHLX(32, uint32_t, _fallback)
+#endif
+
+
+/*
+ * SHRX (BMI2)
+ */
+#define EMIT_SHRX(a_cBitsWidth, a_uType, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shrx_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, a_uType cShift)) \
+{ \
+    cShift &= a_cBitsWidth - 1; \
+    *puDst = uSrc >> cShift; \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHRX(64, uint64_t, RT_NOTHING)
+EMIT_SHRX(64, uint64_t, _fallback)
+#endif
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SHRX(32, uint32_t, RT_NOTHING)
+EMIT_SHRX(32, uint32_t, _fallback)
+#endif
+
+
+/*
+ * SARX (BMI2)
+ */
+#define EMIT_SARX(a_cBitsWidth, a_uType, a_iType, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_sarx_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, a_uType cShift)) \
+{ \
+    cShift &= a_cBitsWidth - 1; \
+    *puDst = (a_iType)uSrc >> cShift; \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SARX(64, uint64_t, int64_t, RT_NOTHING)
+EMIT_SARX(64, uint64_t, int64_t, _fallback)
+#endif
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_SARX(32, uint32_t, int32_t, RT_NOTHING)
+EMIT_SARX(32, uint32_t, int32_t, _fallback)
+#endif
+
+
+/*
+ * PDEP (BMI2)
+ */
+#define EMIT_PDEP(a_cBitsWidth, a_uType, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_pdep_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, a_uType fMask)) \
+{ \
+    a_uType uResult = 0; \
+    for (unsigned iMaskBit = 0, iBit = 0; iMaskBit < a_cBitsWidth; iMaskBit++) \
+        if (fMask & ((a_uType)1 << iMaskBit)) \
+        { \
+            uResult |= ((uSrc >> iBit) & 1) << iMaskBit; \
+            iBit++; \
+        } \
+    *puDst = uResult; \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_PDEP(64, uint64_t, RT_NOTHING)
+#endif
+EMIT_PDEP(64, uint64_t, _fallback)
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_PDEP(32, uint32_t, RT_NOTHING)
+#endif
+EMIT_PDEP(32, uint32_t, _fallback)
+
+/*
+ * PEXT (BMI2)
+ */
+#define EMIT_PEXT(a_cBitsWidth, a_uType, a_Suffix) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_pext_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, a_uType fMask)) \
+{ \
+    a_uType uResult = 0; \
+    for (unsigned iMaskBit = 0, iBit = 0; iMaskBit < a_cBitsWidth; iMaskBit++) \
+        if (fMask & ((a_uType)1 << iMaskBit)) \
+        { \
+            uResult |= ((uSrc >> iMaskBit) & 1) << iBit; \
+            iBit++; \
+        } \
+    *puDst = uResult; \
+}
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_PEXT(64, uint64_t, RT_NOTHING)
+#endif
+EMIT_PEXT(64, uint64_t, _fallback)
+#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) || defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_PEXT(32, uint32_t, RT_NOTHING)
+#endif
+EMIT_PEXT(32, uint32_t, _fallback)
+
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
+
+# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
+/*
+ * BSWAP
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u64,(uint64_t *puDst))
+{
+    *puDst = ASMByteSwapU64(*puDst);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u32,(uint32_t *puDst))
+{
+    *puDst = ASMByteSwapU32(*puDst);
+}
+
+
+/* Note! undocument, so 32-bit arg */
+IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u16,(uint32_t *puDst))
+{
+#if 0
+    *(uint16_t *)puDst = ASMByteSwapU16(*(uint16_t *)puDst);
+#else
+    /* This is the behaviour AMD 3990x (64-bit mode): */
+    *(uint16_t *)puDst = 0;
+#endif
+}
+
+# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
+
+
+
+# if defined(IEM_WITHOUT_ASSEMBLY)
+
+/*
+ * LFENCE, SFENCE & MFENCE.
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_lfence,(void))
+{
+    ASMReadFence();
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sfence,(void))
+{
+    ASMWriteFence();
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_mfence,(void))
+{
+    ASMMemoryFence();
+}
+
+
+#  ifndef RT_ARCH_ARM64
+IEM_DECL_IMPL_DEF(void, iemAImpl_alt_mem_fence,(void))
+{
+    ASMMemoryFence();
+}
+#  endif
+
+# endif
+
+#endif /* !RT_ARCH_AMD64 || IEM_WITHOUT_ASSEMBLY */
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_arpl,(uint16_t *pu16Dst, uint16_t u16Src, uint32_t *pfEFlags))
+{
+    if ((*pu16Dst & X86_SEL_RPL) < (u16Src & X86_SEL_RPL))
+    {
+        *pu16Dst &= X86_SEL_MASK_OFF_RPL;
+        *pu16Dst |= u16Src & X86_SEL_RPL;
+
+        *pfEFlags |= X86_EFL_ZF;
+    }
+    else
+        *pfEFlags &= ~X86_EFL_ZF;
+}
+
+
+#if defined(IEM_WITHOUT_ASSEMBLY)
+
+/*********************************************************************************************************************************
+*   x87 FPU Loads                                                                                                                *
+*********************************************************************************************************************************/
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT32U pr32Val))
+{
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */
+    if (RTFLOAT32U_IS_NORMAL(pr32Val))
+    {
+        pFpuRes->r80Result.sj64.fSign     = pr32Val->s.fSign;
+        pFpuRes->r80Result.sj64.fInteger  = 1;
+        pFpuRes->r80Result.sj64.uFraction = (uint64_t)pr32Val->s.uFraction
+                                         << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS);
+        pFpuRes->r80Result.sj64.uExponent = pr32Val->s.uExponent - RTFLOAT32U_EXP_BIAS + RTFLOAT80U_EXP_BIAS;
+        Assert(RTFLOAT80U_IS_NORMAL(&pFpuRes->r80Result));
+    }
+    else if (RTFLOAT32U_IS_ZERO(pr32Val))
+    {
+        pFpuRes->r80Result.s.fSign     = pr32Val->s.fSign;
+        pFpuRes->r80Result.s.uExponent = 0;
+        pFpuRes->r80Result.s.uMantissa = 0;
+        Assert(RTFLOAT80U_IS_ZERO(&pFpuRes->r80Result));
+    }
+    else if (RTFLOAT32U_IS_SUBNORMAL(pr32Val))
+    {
+        /* Subnormal values gets normalized. */
+        pFpuRes->r80Result.sj64.fSign     = pr32Val->s.fSign;
+        pFpuRes->r80Result.sj64.fInteger  = 1;
+        unsigned const cExtraShift = RTFLOAT32U_FRACTION_BITS - ASMBitLastSetU32(pr32Val->s.uFraction);
+        pFpuRes->r80Result.sj64.uFraction = (uint64_t)pr32Val->s.uFraction
+                                         << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS + cExtraShift + 1);
+        pFpuRes->r80Result.sj64.uExponent = pr32Val->s.uExponent - RTFLOAT32U_EXP_BIAS + RTFLOAT80U_EXP_BIAS - cExtraShift;
+        pFpuRes->FSW |= X86_FSW_DE;
+        if (!(pFpuState->FCW & X86_FCW_DM))
+            pFpuRes->FSW |= X86_FSW_ES | X86_FSW_B; /* The value is still pushed. */
+    }
+    else if (RTFLOAT32U_IS_INF(pr32Val))
+    {
+        pFpuRes->r80Result.s.fSign     = pr32Val->s.fSign;
+        pFpuRes->r80Result.s.uExponent = RTFLOAT80U_EXP_MAX;
+        pFpuRes->r80Result.s.uMantissa = RT_BIT_64(63);
+        Assert(RTFLOAT80U_IS_INF(&pFpuRes->r80Result));
+    }
+    else
+    {
+        /* Signalling and quiet NaNs, both turn into quiet ones when loaded (weird). */
+        Assert(RTFLOAT32U_IS_NAN(pr32Val));
+        pFpuRes->r80Result.sj64.fSign     = pr32Val->s.fSign;
+        pFpuRes->r80Result.sj64.uExponent = RTFLOAT80U_EXP_MAX;
+        pFpuRes->r80Result.sj64.fInteger  = 1;
+        pFpuRes->r80Result.sj64.uFraction = (uint64_t)pr32Val->s.uFraction
+                                         << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS);
+        if (RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val))
+        {
+            pFpuRes->r80Result.sj64.uFraction |= RT_BIT_64(62); /* make quiet */
+            Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result));
+            pFpuRes->FSW |= X86_FSW_IE;
+
+            if (!(pFpuState->FCW & X86_FCW_IM))
+            {
+                /* The value is not pushed. */
+                pFpuRes->FSW &= ~X86_FSW_TOP_MASK;
+                pFpuRes->FSW |= X86_FSW_ES | X86_FSW_B;
+                pFpuRes->r80Result.au64[0] = 0;
+                pFpuRes->r80Result.au16[4] = 0;
+            }
+        }
+        else
+            Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result));
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT64U pr64Val))
+{
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */
+    if (RTFLOAT64U_IS_NORMAL(pr64Val))
+    {
+        pFpuRes->r80Result.sj64.fSign     = pr64Val->s.fSign;
+        pFpuRes->r80Result.sj64.fInteger  = 1;
+        pFpuRes->r80Result.sj64.uFraction = pr64Val->s64.uFraction << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS);
+        pFpuRes->r80Result.sj64.uExponent = pr64Val->s.uExponent - RTFLOAT64U_EXP_BIAS + RTFLOAT80U_EXP_BIAS;
+        Assert(RTFLOAT80U_IS_NORMAL(&pFpuRes->r80Result));
+    }
+    else if (RTFLOAT64U_IS_ZERO(pr64Val))
+    {
+        pFpuRes->r80Result.s.fSign     = pr64Val->s.fSign;
+        pFpuRes->r80Result.s.uExponent = 0;
+        pFpuRes->r80Result.s.uMantissa = 0;
+        Assert(RTFLOAT80U_IS_ZERO(&pFpuRes->r80Result));
+    }
+    else if (RTFLOAT64U_IS_SUBNORMAL(pr64Val))
+    {
+        /* Subnormal values gets normalized. */
+        pFpuRes->r80Result.sj64.fSign     = pr64Val->s.fSign;
+        pFpuRes->r80Result.sj64.fInteger  = 1;
+        unsigned const cExtraShift = RTFLOAT64U_FRACTION_BITS - ASMBitLastSetU64(pr64Val->s64.uFraction);
+        pFpuRes->r80Result.sj64.uFraction = pr64Val->s64.uFraction
+                                         << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS + cExtraShift + 1);
+        pFpuRes->r80Result.sj64.uExponent = pr64Val->s.uExponent - RTFLOAT64U_EXP_BIAS + RTFLOAT80U_EXP_BIAS - cExtraShift;
+        pFpuRes->FSW |= X86_FSW_DE;
+        if (!(pFpuState->FCW & X86_FCW_DM))
+            pFpuRes->FSW |= X86_FSW_ES | X86_FSW_B; /* The value is still pushed. */
+    }
+    else if (RTFLOAT64U_IS_INF(pr64Val))
+    {
+        pFpuRes->r80Result.s.fSign     = pr64Val->s.fSign;
+        pFpuRes->r80Result.s.uExponent = RTFLOAT80U_EXP_MAX;
+        pFpuRes->r80Result.s.uMantissa = RT_BIT_64(63);
+        Assert(RTFLOAT80U_IS_INF(&pFpuRes->r80Result));
+    }
+    else
+    {
+        /* Signalling and quiet NaNs, both turn into quiet ones when loaded (weird). */
+        Assert(RTFLOAT64U_IS_NAN(pr64Val));
+        pFpuRes->r80Result.sj64.fSign     = pr64Val->s.fSign;
+        pFpuRes->r80Result.sj64.uExponent = RTFLOAT80U_EXP_MAX;
+        pFpuRes->r80Result.sj64.fInteger  = 1;
+        pFpuRes->r80Result.sj64.uFraction = pr64Val->s64.uFraction << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS);
+        if (RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val))
+        {
+            pFpuRes->r80Result.sj64.uFraction |= RT_BIT_64(62); /* make quiet */
+            Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result));
+            pFpuRes->FSW |= X86_FSW_IE;
+
+            if (!(pFpuState->FCW & X86_FCW_IM))
+            {
+                /* The value is not pushed. */
+                pFpuRes->FSW &= ~X86_FSW_TOP_MASK;
+                pFpuRes->FSW |= X86_FSW_ES | X86_FSW_B;
+                pFpuRes->r80Result.au64[0] = 0;
+                pFpuRes->r80Result.au16[4] = 0;
+            }
+        }
+        else
+            Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result));
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    pFpuRes->r80Result.au64[0] = pr80Val->au64[0];
+    pFpuRes->r80Result.au16[4] = pr80Val->au16[4];
+    /* Raises no exceptions. */
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fld1,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
+{
+    pFpuRes->r80Result.sj64.fSign       = 0;
+    pFpuRes->r80Result.sj64.uExponent   = 0 + 16383;
+    pFpuRes->r80Result.sj64.fInteger    = 1;
+    pFpuRes->r80Result.sj64.uFraction   = 0;
+
+    /*
+     * FPU status word:
+     *      - TOP is irrelevant, but we must match x86 assembly version.
+     *      - C1 is always cleared as we don't have any stack overflows.
+     *      - C0, C2, and C3 are undefined and Intel 10980XE does not touch them.
+     */
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3));
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fldl2e,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
+{
+    pFpuRes->r80Result.sj64.fSign       = 0;
+    pFpuRes->r80Result.sj64.uExponent   = 0 + 16383;
+    pFpuRes->r80Result.sj64.fInteger    = 1;
+    pFpuRes->r80Result.sj64.uFraction   =    (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
+                                          || (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP
+                                        ? UINT64_C(0x38aa3b295c17f0bc) : UINT64_C(0x38aa3b295c17f0bb);
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fldl2t,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
+{
+    pFpuRes->r80Result.sj64.fSign       = 0;
+    pFpuRes->r80Result.sj64.uExponent   = 1 + 16383;
+    pFpuRes->r80Result.sj64.fInteger    = 1;
+    pFpuRes->r80Result.sj64.uFraction   = (pFpuState->FCW & X86_FCW_RC_MASK) != X86_FCW_RC_UP
+                                        ? UINT64_C(0x549a784bcd1b8afe) : UINT64_C(0x549a784bcd1b8aff);
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fldlg2,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
+{
+    pFpuRes->r80Result.sj64.fSign       = 0;
+    pFpuRes->r80Result.sj64.uExponent   = -2 + 16383;
+    pFpuRes->r80Result.sj64.fInteger    = 1;
+    pFpuRes->r80Result.sj64.uFraction   =    (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
+                                          || (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP
+                                        ? UINT64_C(0x1a209a84fbcff799) : UINT64_C(0x1a209a84fbcff798);
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fldln2,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
+{
+    pFpuRes->r80Result.sj64.fSign       = 0;
+    pFpuRes->r80Result.sj64.uExponent   = -1 + 16383;
+    pFpuRes->r80Result.sj64.fInteger    = 1;
+    pFpuRes->r80Result.sj64.uFraction   =    (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
+                                          || (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP
+                                        ? UINT64_C(0x317217f7d1cf79ac) : UINT64_C(0x317217f7d1cf79ab);
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fldpi,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
+{
+    pFpuRes->r80Result.sj64.fSign       = 0;
+    pFpuRes->r80Result.sj64.uExponent   = 1 + 16383;
+    pFpuRes->r80Result.sj64.fInteger    = 1;
+    pFpuRes->r80Result.sj64.uFraction   =    (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
+                                          || (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP
+                                        ? UINT64_C(0x490fdaa22168c235) : UINT64_C(0x490fdaa22168c234);
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fldz,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
+{
+    pFpuRes->r80Result.s.fSign      = 0;
+    pFpuRes->r80Result.s.uExponent  = 0;
+    pFpuRes->r80Result.s.uMantissa  = 0;
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */
+}
+
+#define EMIT_FILD(a_cBits) \
+IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i ## a_cBits,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, \
+                                                             int ## a_cBits ## _t const *piVal)) \
+{ \
+    int  ## a_cBits ## _t iVal = *piVal; \
+    if (iVal == 0) \
+    { \
+        pFpuRes->r80Result.s.fSign      = 0; \
+        pFpuRes->r80Result.s.uExponent  = 0; \
+        pFpuRes->r80Result.s.uMantissa  = 0; \
+    } \
+    else \
+    { \
+        if (iVal > 0) \
+            pFpuRes->r80Result.s.fSign  = 0; \
+        else \
+        { \
+            pFpuRes->r80Result.s.fSign  = 1; \
+            iVal = -iVal; \
+        } \
+        unsigned const cBits = ASMBitLastSetU ## a_cBits((uint ## a_cBits ## _t)iVal); \
+        pFpuRes->r80Result.s.uExponent  = cBits - 1 + RTFLOAT80U_EXP_BIAS; \
+        pFpuRes->r80Result.s.uMantissa  = (uint64_t)iVal << (RTFLOAT80U_FRACTION_BITS + 1 - cBits); \
+    } \
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */ \
+}
+EMIT_FILD(16)
+EMIT_FILD(32)
+EMIT_FILD(64)
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_d80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTPBCD80U pd80Val))
+{
+    pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); /* see iemAImpl_fld1 */
+    if (   pd80Val->s.abPairs[0] == 0
+        && pd80Val->s.abPairs[1] == 0
+        && pd80Val->s.abPairs[2] == 0
+        && pd80Val->s.abPairs[3] == 0
+        && pd80Val->s.abPairs[4] == 0
+        && pd80Val->s.abPairs[5] == 0
+        && pd80Val->s.abPairs[6] == 0
+        && pd80Val->s.abPairs[7] == 0
+        && pd80Val->s.abPairs[8] == 0)
+    {
+        pFpuRes->r80Result.s.fSign      = pd80Val->s.fSign;
+        pFpuRes->r80Result.s.uExponent  = 0;
+        pFpuRes->r80Result.s.uMantissa  = 0;
+    }
+    else
+    {
+        pFpuRes->r80Result.s.fSign      = pd80Val->s.fSign;
+
+        size_t   cPairs  = RT_ELEMENTS(pd80Val->s.abPairs);
+        while (cPairs > 0 && pd80Val->s.abPairs[cPairs - 1] == 0)
+            cPairs--;
+
+        uint64_t uVal    = 0;
+        uint64_t uFactor = 1;
+        for (size_t iPair = 0; iPair < cPairs; iPair++, uFactor *= 100)
+            uVal += RTPBCD80U_LO_DIGIT(pd80Val->s.abPairs[iPair]) * uFactor
+                  + RTPBCD80U_HI_DIGIT(pd80Val->s.abPairs[iPair]) * uFactor * 10;
+
+        unsigned const cBits = ASMBitLastSetU64(uVal);
+        pFpuRes->r80Result.s.uExponent  = cBits - 1 + RTFLOAT80U_EXP_BIAS;
+        pFpuRes->r80Result.s.uMantissa  = uVal << (RTFLOAT80U_FRACTION_BITS + 1 - cBits);
+    }
+}
+
+
+/*********************************************************************************************************************************
+*   x87 FPU Stores                                                                                                               *
+*********************************************************************************************************************************/
+
+/**
+ * Helper for storing a deconstructed and normal R80 value as a 64-bit one.
+ *
+ * This uses the rounding rules indicated by fFcw and returns updated fFsw.
+ *
+ * @returns Updated FPU status word value.
+ * @param   fSignIn     Incoming sign indicator.
+ * @param   uMantissaIn Incoming mantissa (dot between bit 63 and 62).
+ * @param   iExponentIn Unbiased exponent.
+ * @param   fFcw        The FPU control word.
+ * @param   fFsw        Prepped FPU status word, i.e. exceptions and C1 clear.
+ * @param   pr32Dst     Where to return the output value, if one should be
+ *                      returned.
+ *
+ * @note    Tailored as a helper for iemAImpl_fst_r80_to_r32 right now.
+ * @note    Exact same logic as iemAImpl_StoreNormalR80AsR64.
+ */
+static uint16_t iemAImpl_StoreNormalR80AsR32(bool fSignIn, uint64_t uMantissaIn, int32_t iExponentIn,
+                                             uint16_t fFcw, uint16_t fFsw, PRTFLOAT32U pr32Dst)
+{
+    uint64_t const fRoundingOffMask = RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS) - 1; /* 0x7ff */
+    uint64_t const uRoundingAdd     = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
+                                    ? RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS - 1)  /* 0x400 */
+                                    : (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)
+                                    ? fRoundingOffMask
+                                    : 0;
+    uint64_t       fRoundedOff      = uMantissaIn & fRoundingOffMask;
+
+    /*
+     * Deal with potential overflows/underflows first, optimizing for none.
+     * 0 and MAX are used for special values; MAX-1 may be rounded up to MAX.
+     */
+    int32_t iExponentOut = (int32_t)iExponentIn + RTFLOAT32U_EXP_BIAS;
+    if ((uint32_t)iExponentOut - 1 < (uint32_t)(RTFLOAT32U_EXP_MAX - 3))
+    { /* likely? */ }
+    /*
+     * Underflow if the exponent zero or negative.  This is attempted mapped
+     * to a subnormal number when possible, with some additional trickery ofc.
+     */
+    else if (iExponentOut <= 0)
+    {
+        bool const fIsTiny = iExponentOut < 0
+                          || UINT64_MAX - uMantissaIn > uRoundingAdd;
+        if (!(fFcw & X86_FCW_UM) && fIsTiny)
+            /* Note! 754-1985 sec 7.4 has something about bias adjust of 192 here, not in 2008 & 2019. Perhaps only 8087 & 287? */
+            return fFsw | X86_FSW_UE | X86_FSW_ES | X86_FSW_B;
+
+        if (iExponentOut <= 0)
+        {
+            uMantissaIn = iExponentOut <= -63
+                        ? uMantissaIn != 0
+                        : (uMantissaIn >> (-iExponentOut + 1)) | ((uMantissaIn & (RT_BIT_64(-iExponentOut + 1) - 1)) != 0);
+            fRoundedOff = uMantissaIn & fRoundingOffMask;
+            if (fRoundedOff && fIsTiny)
+                fFsw |= X86_FSW_UE;
+            iExponentOut   = 0;
+        }
+    }
+    /*
+     * Overflow if at or above max exponent value or if we will reach max
+     * when rounding.  Will return +/-zero or +/-max value depending on
+     * whether we're rounding or not.
+     */
+    else if (   iExponentOut >= RTFLOAT32U_EXP_MAX
+             || (   iExponentOut == RTFLOAT32U_EXP_MAX - 1
+                 && UINT64_MAX - uMantissaIn <= uRoundingAdd))
+    {
+        fFsw |= X86_FSW_OE;
+        if (!(fFcw & X86_FCW_OM))
+            return fFsw | X86_FSW_ES | X86_FSW_B;
+        fFsw |= X86_FSW_PE;
+        if (uRoundingAdd)
+            fFsw |= X86_FSW_C1;
+        if (!(fFcw & X86_FCW_PM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+
+        pr32Dst->s.fSign         = fSignIn;
+        if (uRoundingAdd)
+        {   /* Zero */
+            pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX;
+            pr32Dst->s.uFraction = 0;
+        }
+        else
+        {   /* Max */
+            pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX - 1;
+            pr32Dst->s.uFraction = RT_BIT_32(RTFLOAT32U_FRACTION_BITS) - 1;
+        }
+        return fFsw;
+    }
+
+    /*
+     * Normal or subnormal number.
+     */
+    /* Do rounding - just truncate in near mode when midway on an even outcome. */
+    uint64_t uMantissaOut = uMantissaIn;
+    if (   (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_NEAREST
+        || (uMantissaIn & RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS))
+        || fRoundedOff != uRoundingAdd)
+    {
+        uMantissaOut = uMantissaIn + uRoundingAdd;
+        if (uMantissaOut >= uMantissaIn)
+        { /* likely */ }
+        else
+        {
+            uMantissaOut >>= 1; /* (We don't need to add bit 63 here (the integer bit), as it will be chopped off below.) */
+            iExponentOut++;
+            Assert(iExponentOut < RTFLOAT32U_EXP_MAX);  /* checked above */
+            fFsw |= X86_FSW_C1;
+        }
+    }
+    else
+        uMantissaOut = uMantissaIn;
+
+    /* Truncate the mantissa and set the return value. */
+    uMantissaOut >>= RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS;
+
+    pr32Dst->s.uFraction = (uint32_t)uMantissaOut; /* Note! too big for bitfield if normal. */
+    pr32Dst->s.uExponent = iExponentOut;
+    pr32Dst->s.fSign     = fSignIn;
+
+    /* Set status flags realted to rounding. */
+    if (fRoundedOff)
+    {
+        fFsw |= X86_FSW_PE;
+        if (uMantissaOut > (uMantissaIn >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS)))
+            fFsw |= X86_FSW_C1;
+        if (!(fFcw & X86_FCW_PM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+
+    return fFsw;
+}
+
+
+/**
+ * @note Exact same logic as iemAImpl_fst_r80_to_r64.
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                 PRTFLOAT32U pr32Dst, PCRTFLOAT80U pr80Src))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t       fFsw = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3));
+    if (RTFLOAT80U_IS_NORMAL(pr80Src))
+        fFsw = iemAImpl_StoreNormalR80AsR32(pr80Src->s.fSign, pr80Src->s.uMantissa,
+                                            (int32_t)pr80Src->s.uExponent - RTFLOAT80U_EXP_BIAS, fFcw, fFsw, pr32Dst);
+    else if (RTFLOAT80U_IS_ZERO(pr80Src))
+    {
+        pr32Dst->s.fSign      = pr80Src->s.fSign;
+        pr32Dst->s.uExponent  = 0;
+        pr32Dst->s.uFraction  = 0;
+        Assert(RTFLOAT32U_IS_ZERO(pr32Dst));
+    }
+    else if (RTFLOAT80U_IS_INF(pr80Src))
+    {
+        pr32Dst->s.fSign      = pr80Src->s.fSign;
+        pr32Dst->s.uExponent  = RTFLOAT32U_EXP_MAX;
+        pr32Dst->s.uFraction  = 0;
+        Assert(RTFLOAT32U_IS_INF(pr32Dst));
+    }
+    else if (RTFLOAT80U_IS_INDEFINITE(pr80Src))
+    {
+        /* Mapped to +/-QNaN */
+        pr32Dst->s.fSign      = pr80Src->s.fSign;
+        pr32Dst->s.uExponent  = RTFLOAT32U_EXP_MAX;
+        pr32Dst->s.uFraction  = RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1);
+    }
+    else if (RTFLOAT80U_IS_PSEUDO_INF(pr80Src) || RTFLOAT80U_IS_UNNORMAL(pr80Src) || RTFLOAT80U_IS_PSEUDO_NAN(pr80Src))
+    {
+        /* Pseudo-Inf / Pseudo-Nan / Unnormal -> QNaN (during load, probably) */
+        if (fFcw & X86_FCW_IM)
+        {
+            pr32Dst->s.fSign      = 1;
+            pr32Dst->s.uExponent  = RTFLOAT32U_EXP_MAX;
+            pr32Dst->s.uFraction  = RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1);
+            fFsw |= X86_FSW_IE;
+        }
+        else
+            fFsw |= X86_FSW_IE | X86_FSW_ES | X86_FSW_B;;
+    }
+    else if (RTFLOAT80U_IS_NAN(pr80Src))
+    {
+        /* IM applies to signalled NaN input only. Everything is converted to quiet NaN. */
+        if ((fFcw & X86_FCW_IM) || !RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src))
+        {
+            pr32Dst->s.fSign      = pr80Src->s.fSign;
+            pr32Dst->s.uExponent  = RTFLOAT32U_EXP_MAX;
+            pr32Dst->s.uFraction  = (uint32_t)(pr80Src->sj64.uFraction >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS));
+            pr32Dst->s.uFraction |= RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1);
+            if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src))
+                fFsw |= X86_FSW_IE;
+        }
+        else
+            fFsw |= X86_FSW_IE | X86_FSW_ES | X86_FSW_B;
+    }
+    else
+    {
+        /* Denormal values causes both an underflow and precision exception. */
+        Assert(RTFLOAT80U_IS_DENORMAL(pr80Src) || RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Src));
+        if (fFcw & X86_FCW_UM)
+        {
+            pr32Dst->s.fSign     = pr80Src->s.fSign;
+            pr32Dst->s.uExponent = 0;
+            if ((fFcw & X86_FCW_RC_MASK) == (!pr80Src->s.fSign ? X86_FCW_RC_UP : X86_FCW_RC_DOWN))
+            {
+                pr32Dst->s.uFraction = 1;
+                fFsw |= X86_FSW_UE | X86_FSW_PE | X86_FSW_C1;
+                if (!(fFcw & X86_FCW_PM))
+                    fFsw |= X86_FSW_ES | X86_FSW_B;
+            }
+            else
+            {
+                pr32Dst->s.uFraction = 0;
+                fFsw |= X86_FSW_UE | X86_FSW_PE;
+                if (!(fFcw & X86_FCW_PM))
+                    fFsw |= X86_FSW_ES | X86_FSW_B;
+            }
+        }
+        else
+            fFsw |= X86_FSW_UE | X86_FSW_ES | X86_FSW_B;
+    }
+    *pu16FSW = fFsw;
+}
+
+
+/**
+ * Helper for storing a deconstructed and normal R80 value as a 64-bit one.
+ *
+ * This uses the rounding rules indicated by fFcw and returns updated fFsw.
+ *
+ * @returns Updated FPU status word value.
+ * @param   fSignIn     Incoming sign indicator.
+ * @param   uMantissaIn Incoming mantissa (dot between bit 63 and 62).
+ * @param   iExponentIn Unbiased exponent.
+ * @param   fFcw        The FPU control word.
+ * @param   fFsw        Prepped FPU status word, i.e. exceptions and C1 clear.
+ * @param   pr64Dst     Where to return the output value, if one should be
+ *                      returned.
+ *
+ * @note    Tailored as a helper for iemAImpl_fst_r80_to_r64 right now.
+ * @note    Exact same logic as iemAImpl_StoreNormalR80AsR32.
+ */
+static uint16_t iemAImpl_StoreNormalR80AsR64(bool fSignIn, uint64_t uMantissaIn, int32_t iExponentIn,
+                                             uint16_t fFcw, uint16_t fFsw, PRTFLOAT64U pr64Dst)
+{
+    uint64_t const fRoundingOffMask = RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS) - 1; /* 0x7ff */
+    uint32_t const uRoundingAdd     = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
+                                    ? RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS - 1)  /* 0x400 */
+                                    : (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)
+                                    ? fRoundingOffMask
+                                    : 0;
+    uint32_t       fRoundedOff      = uMantissaIn & fRoundingOffMask;
+
+    /*
+     * Deal with potential overflows/underflows first, optimizing for none.
+     * 0 and MAX are used for special values; MAX-1 may be rounded up to MAX.
+     */
+    int32_t iExponentOut = (int32_t)iExponentIn + RTFLOAT64U_EXP_BIAS;
+    if ((uint32_t)iExponentOut - 1 < (uint32_t)(RTFLOAT64U_EXP_MAX - 3))
+    { /* likely? */ }
+    /*
+     * Underflow if the exponent zero or negative.  This is attempted mapped
+     * to a subnormal number when possible, with some additional trickery ofc.
+     */
+    else if (iExponentOut <= 0)
+    {
+        bool const fIsTiny = iExponentOut < 0
+                          || UINT64_MAX - uMantissaIn > uRoundingAdd;
+        if (!(fFcw & X86_FCW_UM) && fIsTiny)
+            /* Note! 754-1985 sec 7.4 has something about bias adjust of 1536 here, not in 2008 & 2019. Perhaps only 8087 & 287? */
+            return fFsw | X86_FSW_UE | X86_FSW_ES | X86_FSW_B;
+
+        if (iExponentOut <= 0)
+        {
+            uMantissaIn  = iExponentOut <= -63
+                         ? uMantissaIn != 0
+                         : (uMantissaIn >> (-iExponentOut + 1)) | ((uMantissaIn & (RT_BIT_64(-iExponentOut + 1) - 1)) != 0);
+            fRoundedOff  = uMantissaIn & fRoundingOffMask;
+            if (fRoundedOff && fIsTiny)
+                fFsw    |= X86_FSW_UE;
+            iExponentOut = 0;
+        }
+    }
+    /*
+     * Overflow if at or above max exponent value or if we will reach max
+     * when rounding.  Will return +/-zero or +/-max value depending on
+     * whether we're rounding or not.
+     */
+    else if (   iExponentOut >= RTFLOAT64U_EXP_MAX
+             || (   iExponentOut == RTFLOAT64U_EXP_MAX - 1
+                 && UINT64_MAX - uMantissaIn <= uRoundingAdd))
+    {
+        fFsw |= X86_FSW_OE;
+        if (!(fFcw & X86_FCW_OM))
+            return fFsw | X86_FSW_ES | X86_FSW_B;
+        fFsw |= X86_FSW_PE;
+        if (uRoundingAdd)
+            fFsw |= X86_FSW_C1;
+        if (!(fFcw & X86_FCW_PM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+
+        pr64Dst->s64.fSign         = fSignIn;
+        if (uRoundingAdd)
+        {   /* Zero */
+            pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX;
+            pr64Dst->s64.uFraction = 0;
+        }
+        else
+        {   /* Max */
+            pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX - 1;
+            pr64Dst->s64.uFraction = RT_BIT_64(RTFLOAT64U_FRACTION_BITS) - 1;
+        }
+        return fFsw;
+    }
+
+    /*
+     * Normal or subnormal number.
+     */
+    /* Do rounding - just truncate in near mode when midway on an even outcome. */
+    uint64_t uMantissaOut = uMantissaIn;
+    if (   (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_NEAREST
+        || (uMantissaIn & RT_BIT_32(RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS))
+        || fRoundedOff != uRoundingAdd)
+    {
+        uMantissaOut = uMantissaIn + uRoundingAdd;
+        if (uMantissaOut >= uMantissaIn)
+        { /* likely */ }
+        else
+        {
+            uMantissaOut >>= 1; /* (We don't need to add bit 63 here (the integer bit), as it will be chopped off below.) */
+            iExponentOut++;
+            Assert(iExponentOut < RTFLOAT64U_EXP_MAX);  /* checked above */
+            fFsw |= X86_FSW_C1;
+        }
+    }
+    else
+        uMantissaOut = uMantissaIn;
+
+    /* Truncate the mantissa and set the return value. */
+    uMantissaOut >>= RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS;
+
+    pr64Dst->s64.uFraction = uMantissaOut; /* Note! too big for bitfield if normal. */
+    pr64Dst->s64.uExponent = iExponentOut;
+    pr64Dst->s64.fSign     = fSignIn;
+
+    /* Set status flags realted to rounding. */
+    if (fRoundedOff)
+    {
+        fFsw |= X86_FSW_PE;
+        if (uMantissaOut > (uMantissaIn >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS)))
+            fFsw |= X86_FSW_C1;
+        if (!(fFcw & X86_FCW_PM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+
+    return fFsw;
+}
+
+
+/**
+ * @note Exact same logic as iemAImpl_fst_r80_to_r32.
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                 PRTFLOAT64U pr64Dst, PCRTFLOAT80U pr80Src))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t       fFsw = (7 << X86_FSW_TOP_SHIFT) | (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3));
+    if (RTFLOAT80U_IS_NORMAL(pr80Src))
+        fFsw = iemAImpl_StoreNormalR80AsR64(pr80Src->s.fSign, pr80Src->s.uMantissa,
+                                            (int32_t)pr80Src->s.uExponent - RTFLOAT80U_EXP_BIAS, fFcw, fFsw, pr64Dst);
+    else if (RTFLOAT80U_IS_ZERO(pr80Src))
+    {
+        pr64Dst->s64.fSign      = pr80Src->s.fSign;
+        pr64Dst->s64.uExponent  = 0;
+        pr64Dst->s64.uFraction  = 0;
+        Assert(RTFLOAT64U_IS_ZERO(pr64Dst));
+    }
+    else if (RTFLOAT80U_IS_INF(pr80Src))
+    {
+        pr64Dst->s64.fSign      = pr80Src->s.fSign;
+        pr64Dst->s64.uExponent  = RTFLOAT64U_EXP_MAX;
+        pr64Dst->s64.uFraction  = 0;
+        Assert(RTFLOAT64U_IS_INF(pr64Dst));
+    }
+    else if (RTFLOAT80U_IS_INDEFINITE(pr80Src))
+    {
+        /* Mapped to +/-QNaN */
+        pr64Dst->s64.fSign      = pr80Src->s.fSign;
+        pr64Dst->s64.uExponent  = RTFLOAT64U_EXP_MAX;
+        pr64Dst->s64.uFraction  = RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1);
+    }
+    else if (RTFLOAT80U_IS_PSEUDO_INF(pr80Src) || RTFLOAT80U_IS_UNNORMAL(pr80Src) || RTFLOAT80U_IS_PSEUDO_NAN(pr80Src))
+    {
+        /* Pseudo-Inf / Pseudo-Nan / Unnormal -> QNaN (during load, probably) */
+        if (fFcw & X86_FCW_IM)
+        {
+            pr64Dst->s64.fSign      = 1;
+            pr64Dst->s64.uExponent  = RTFLOAT64U_EXP_MAX;
+            pr64Dst->s64.uFraction  = RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1);
+            fFsw |= X86_FSW_IE;
+        }
+        else
+            fFsw |= X86_FSW_IE | X86_FSW_ES | X86_FSW_B;;
+    }
+    else if (RTFLOAT80U_IS_NAN(pr80Src))
+    {
+        /* IM applies to signalled NaN input only. Everything is converted to quiet NaN. */
+        if ((fFcw & X86_FCW_IM) || !RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src))
+        {
+            pr64Dst->s64.fSign      = pr80Src->s.fSign;
+            pr64Dst->s64.uExponent  = RTFLOAT64U_EXP_MAX;
+            pr64Dst->s64.uFraction  = pr80Src->sj64.uFraction >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS);
+            pr64Dst->s64.uFraction |= RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1);
+            if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src))
+                fFsw |= X86_FSW_IE;
+        }
+        else
+            fFsw |= X86_FSW_IE | X86_FSW_ES | X86_FSW_B;
+    }
+    else
+    {
+        /* Denormal values causes both an underflow and precision exception. */
+        Assert(RTFLOAT80U_IS_DENORMAL(pr80Src) || RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Src));
+        if (fFcw & X86_FCW_UM)
+        {
+            pr64Dst->s64.fSign     = pr80Src->s.fSign;
+            pr64Dst->s64.uExponent = 0;
+            if ((fFcw & X86_FCW_RC_MASK) == (!pr80Src->s.fSign ? X86_FCW_RC_UP : X86_FCW_RC_DOWN))
+            {
+                pr64Dst->s64.uFraction = 1;
+                fFsw |= X86_FSW_UE | X86_FSW_PE | X86_FSW_C1;
+                if (!(fFcw & X86_FCW_PM))
+                    fFsw |= X86_FSW_ES | X86_FSW_B;
+            }
+            else
+            {
+                pr64Dst->s64.uFraction = 0;
+                fFsw |= X86_FSW_UE | X86_FSW_PE;
+                if (!(fFcw & X86_FCW_PM))
+                    fFsw |= X86_FSW_ES | X86_FSW_B;
+            }
+        }
+        else
+            fFsw |= X86_FSW_UE | X86_FSW_ES | X86_FSW_B;
+    }
+    *pu16FSW = fFsw;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                 PRTFLOAT80U pr80Dst, PCRTFLOAT80U pr80Src))
+{
+    /*
+     * FPU status word:
+     *      - TOP is irrelevant, but we must match x86 assembly version (0).
+     *      - C1 is always cleared as we don't have any stack overflows.
+     *      - C0, C2, and C3 are undefined and Intel 10980XE does not touch them.
+     */
+    *pu16FSW = pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3); /* see iemAImpl_fld1 */
+    *pr80Dst = *pr80Src;
+}
+
+
+/*
+ *
+ * Mantissa:
+ *  63        56        48        40        32        24        16         8         0
+ *  v          v         v         v         v         v         v         v         v
+ *  1[.]111 0000 1111 0000 1111 0000 1111 0000 1111 0000 1111 0000 1111 0000 1111 0000
+ *      \    \    \    \    \    \    \    \    \    \    \    \    \    \    \    \
+ * Exp: 0    4    8    12   16   20   24   28   32   36   40   44   48   52   56   60
+ *
+ * int64_t has the same width, only bit 63 is the sign bit.  So, the max we can map over
+ * are bits 1 thru 63, dropping off bit 0, with an exponent of 62. The number of bits we
+ * drop off from the mantissa increases with decreasing exponent, till an exponent of 0
+ * where we'll drop off all but bit 63.
+ */
+#define EMIT_FIST(a_cBits, a_iType, a_iTypeMin, a_iTypeIndefinite) \
+IEM_DECL_IMPL_DEF(void, iemAImpl_fist_r80_to_i ## a_cBits,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW, \
+                                                           a_iType *piDst, PCRTFLOAT80U pr80Val)) \
+{ \
+    uint16_t const fFcw    = pFpuState->FCW; \
+    uint16_t       fFsw    = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); \
+    bool const     fSignIn = pr80Val->s.fSign; \
+    \
+    /* \
+     * Deal with normal numbers first. \
+     */ \
+    if (RTFLOAT80U_IS_NORMAL(pr80Val)) \
+    { \
+        uint64_t       uMantissa = pr80Val->s.uMantissa; \
+        int32_t        iExponent = (int32_t)pr80Val->s.uExponent - RTFLOAT80U_EXP_BIAS; \
+        \
+        if ((uint32_t)iExponent <= a_cBits - 2) \
+        { \
+            unsigned const cShiftOff        = 63 - iExponent; \
+            uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1; \
+            uint64_t const uRoundingAdd     = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST \
+                                            ? RT_BIT_64(cShiftOff - 1) \
+                                            : (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP) \
+                                            ? fRoundingOffMask \
+                                            : 0; \
+            uint64_t       fRoundedOff      = uMantissa & fRoundingOffMask; \
+            \
+            uMantissa >>= cShiftOff; \
+            uint64_t const uRounding = (fRoundedOff + uRoundingAdd) >> cShiftOff; \
+            uMantissa += uRounding; \
+            if (!(uMantissa & RT_BIT_64(a_cBits - 1))) \
+            { \
+                if (fRoundedOff) \
+                { \
+                    if ((uMantissa & 1) && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST && fRoundedOff == uRoundingAdd) \
+                        uMantissa &= ~(uint64_t)1; /* round to even number if equal distance between up/down. */ \
+                    else if (uRounding) \
+                        fFsw |= X86_FSW_C1; \
+                    fFsw |= X86_FSW_PE; \
+                    if (!(fFcw & X86_FCW_PM)) \
+                        fFsw |= X86_FSW_ES | X86_FSW_B; \
+                } \
+                \
+                if (!fSignIn) \
+                    *piDst = (a_iType)uMantissa; \
+                else \
+                    *piDst = -(a_iType)uMantissa; \
+            } \
+            else \
+            { \
+                /* overflowed after rounding. */ \
+                AssertMsg(iExponent == a_cBits - 2 && uMantissa == RT_BIT_64(a_cBits - 1), \
+                          ("e=%d m=%#RX64 (org %#RX64) s=%d; shift=%d ro=%#RX64 rm=%#RX64 ra=%#RX64\n", iExponent, uMantissa, \
+                          pr80Val->s.uMantissa, fSignIn, cShiftOff, fRoundedOff, fRoundingOffMask, uRoundingAdd)); \
+                \
+                /* Special case for the integer minimum value. */ \
+                if (fSignIn) \
+                { \
+                    *piDst = a_iTypeMin; \
+                    fFsw |= X86_FSW_PE | X86_FSW_C1; \
+                    if (!(fFcw & X86_FCW_PM)) \
+                        fFsw |= X86_FSW_ES | X86_FSW_B; \
+                } \
+                else \
+                { \
+                    fFsw |= X86_FSW_IE; \
+                    if (fFcw & X86_FCW_IM) \
+                        *piDst = a_iTypeMin; \
+                    else \
+                        fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); \
+                } \
+            } \
+        } \
+        /* \
+         * Tiny sub-zero numbers. \
+         */ \
+        else if (iExponent < 0) \
+        { \
+            if (!fSignIn) \
+            { \
+                if (   (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP \
+                    || (iExponent == -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST)) \
+                { \
+                    *piDst = 1; \
+                    fFsw |= X86_FSW_C1; \
+                } \
+                else \
+                    *piDst = 0; \
+            } \
+            else \
+            { \
+                if (   (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP \
+                    || (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_ZERO \
+                    || (iExponent < -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST)) \
+                    *piDst = 0; \
+                else \
+                { \
+                    *piDst = -1; \
+                    fFsw |= X86_FSW_C1; \
+                } \
+            } \
+            fFsw |= X86_FSW_PE; \
+            if (!(fFcw & X86_FCW_PM)) \
+                fFsw |= X86_FSW_ES | X86_FSW_B; \
+        } \
+        /* \
+         * Special MIN case. \
+         */ \
+        else if (   fSignIn && iExponent == a_cBits - 1 \
+                 && ( a_cBits < 64 && (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_DOWN \
+                     ? uMantissa < (RT_BIT_64(63) | RT_BIT_64(65 - a_cBits)) \
+                     : uMantissa == RT_BIT_64(63))) \
+        { \
+            *piDst = a_iTypeMin;  \
+            if (uMantissa & (RT_BIT_64(64 - a_cBits + 1) - 1)) \
+            { \
+                fFsw |= X86_FSW_PE; \
+                if (!(fFcw & X86_FCW_PM)) \
+                    fFsw |= X86_FSW_ES | X86_FSW_B; \
+            } \
+        } \
+        /* \
+         * Too large/small number outside the target integer range. \
+         */ \
+        else \
+        { \
+            fFsw |= X86_FSW_IE; \
+            if (fFcw & X86_FCW_IM) \
+                *piDst = a_iTypeIndefinite; \
+            else \
+                fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); \
+        } \
+    } \
+    /* \
+     * Map both +0 and -0 to integer zero (signless/+). \
+     */ \
+    else if (RTFLOAT80U_IS_ZERO(pr80Val)) \
+        *piDst = 0; \
+    /* \
+     * Denormals are just really tiny sub-zero numbers that are either rounded \
+     * to zero, 1 or -1 depending on sign and rounding control. \
+     */ \
+    else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val) || RTFLOAT80U_IS_DENORMAL(pr80Val)) \
+    { \
+        if ((fFcw & X86_FCW_RC_MASK) != (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)) \
+            *piDst = 0; \
+        else \
+        { \
+            *piDst = fSignIn ? -1 : 1; \
+            fFsw |= X86_FSW_C1; \
+        } \
+        fFsw |= X86_FSW_PE; \
+        if (!(fFcw & X86_FCW_PM)) \
+            fFsw |= X86_FSW_ES | X86_FSW_B; \
+    } \
+    /* \
+     * All other special values are considered invalid arguments and result \
+     * in an IE exception and indefinite value if masked. \
+     */ \
+    else \
+    { \
+        fFsw |= X86_FSW_IE; \
+        if (fFcw & X86_FCW_IM) \
+            *piDst = a_iTypeIndefinite; \
+        else \
+            fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); \
+    } \
+    *pu16FSW = fFsw; \
+}
+EMIT_FIST(64, int64_t, INT64_MIN, X86_FPU_INT64_INDEFINITE)
+EMIT_FIST(32, int32_t, INT32_MIN, X86_FPU_INT32_INDEFINITE)
+EMIT_FIST(16, int16_t, INT16_MIN, X86_FPU_INT16_INDEFINITE)
+
+#endif /*IEM_WITHOUT_ASSEMBLY */
+
+
+/*
+ * The FISTT instruction was added with SSE3 and are a lot simpler than FIST.
+ *
+ * The 16-bit version is a bit peculiar, though, as it seems to be raising IE
+ * as if it was the 32-bit version (i.e. starting with exp 31 instead of 15),
+ * thus the @a a_cBitsIn.
+ */
+#define EMIT_FISTT(a_cBits, a_cBitsIn, a_iType, a_iTypeMin, a_iTypeMax, a_iTypeIndefinite, a_Suffix, a_fIntelVersion) \
+IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_fistt_r80_to_i,a_cBits,a_Suffix),(PCX86FXSTATE pFpuState, uint16_t *pu16FSW, \
+                                                                              a_iType *piDst, PCRTFLOAT80U pr80Val)) \
+{ \
+    uint16_t const fFcw    = pFpuState->FCW; \
+    uint16_t       fFsw    = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)); \
+    bool const     fSignIn = pr80Val->s.fSign; \
+    \
+    /* \
+     * Deal with normal numbers first. \
+     */ \
+    if (RTFLOAT80U_IS_NORMAL(pr80Val)) \
+    { \
+        uint64_t       uMantissa = pr80Val->s.uMantissa; \
+        int32_t        iExponent = (int32_t)pr80Val->s.uExponent - RTFLOAT80U_EXP_BIAS; \
+        \
+        if ((uint32_t)iExponent <= a_cBitsIn - 2) \
+        { \
+            unsigned const cShiftOff        = 63 - iExponent; \
+            uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1; \
+            uint64_t const fRoundedOff      = uMantissa & fRoundingOffMask; \
+            uMantissa >>= cShiftOff; \
+            /*Assert(!(uMantissa & RT_BIT_64(a_cBits - 1)));*/ \
+            if (!fSignIn) \
+                *piDst = (a_iType)uMantissa; \
+            else \
+                *piDst = -(a_iType)uMantissa; \
+            \
+            if (fRoundedOff) \
+            { \
+                fFsw |= X86_FSW_PE; \
+                if (!(fFcw & X86_FCW_PM)) \
+                    fFsw |= X86_FSW_ES | X86_FSW_B; \
+            } \
+        } \
+        /* \
+         * Tiny sub-zero numbers. \
+         */ \
+        else if (iExponent < 0) \
+        { \
+            *piDst = 0; \
+            fFsw |= X86_FSW_PE; \
+            if (!(fFcw & X86_FCW_PM)) \
+                fFsw |= X86_FSW_ES | X86_FSW_B; \
+        } \
+        /* \
+         * Special MIN case. \
+         */ \
+        else if (   fSignIn && iExponent == a_cBits - 1 \
+                 && (a_cBits < 64 \
+                     ? uMantissa < (RT_BIT_64(63) | RT_BIT_64(65 - a_cBits)) \
+                     : uMantissa == RT_BIT_64(63)) ) \
+        { \
+            *piDst = a_iTypeMin;  \
+            if (uMantissa & (RT_BIT_64(64 - a_cBits + 1) - 1)) \
+            { \
+                fFsw |= X86_FSW_PE; \
+                if (!(fFcw & X86_FCW_PM)) \
+                    fFsw |= X86_FSW_ES | X86_FSW_B; \
+            } \
+        } \
+        /* \
+         * Figure this weirdness. \
+         */ \
+        else if (0 /* huh? gone? */ && a_cBits == 16 && fSignIn && iExponent == 31 && uMantissa < UINT64_C(0x8000100000000000) ) \
+        { \
+            *piDst = 0;  \
+            if (uMantissa & (RT_BIT_64(64 - a_cBits + 1) - 1)) \
+            { \
+                fFsw |= X86_FSW_PE; \
+                if (!(fFcw & X86_FCW_PM)) \
+                    fFsw |= X86_FSW_ES | X86_FSW_B; \
+            } \
+        } \
+        /* \
+         * Too large/small number outside the target integer range. \
+         */ \
+        else \
+        { \
+            fFsw |= X86_FSW_IE; \
+            if (fFcw & X86_FCW_IM) \
+                *piDst = a_iTypeIndefinite; \
+            else \
+                fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); \
+        } \
+    } \
+    /* \
+     * Map both +0 and -0 to integer zero (signless/+). \
+     */ \
+    else if (RTFLOAT80U_IS_ZERO(pr80Val)) \
+        *piDst = 0; \
+    /* \
+     * Denormals are just really tiny sub-zero numbers that are trucated to zero. \
+     */ \
+    else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val) || RTFLOAT80U_IS_DENORMAL(pr80Val)) \
+    { \
+        *piDst = 0; \
+        fFsw |= X86_FSW_PE; \
+        if (!(fFcw & X86_FCW_PM)) \
+            fFsw |= X86_FSW_ES | X86_FSW_B; \
+    } \
+    /* \
+     * All other special values are considered invalid arguments and result \
+     * in an IE exception and indefinite value if masked. \
+     */ \
+    else \
+    { \
+        fFsw |= X86_FSW_IE; \
+        if (fFcw & X86_FCW_IM) \
+            *piDst = a_iTypeIndefinite; \
+        else \
+            fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT); \
+    } \
+    *pu16FSW = fFsw; \
+}
+#if defined(IEM_WITHOUT_ASSEMBLY)
+EMIT_FISTT(64, 64, int64_t, INT64_MIN, INT64_MAX, X86_FPU_INT64_INDEFINITE, RT_NOTHING, 1)
+EMIT_FISTT(32, 32, int32_t, INT32_MIN, INT32_MAX, X86_FPU_INT32_INDEFINITE, RT_NOTHING, 1)
+EMIT_FISTT(16, 16, int16_t, INT16_MIN, INT16_MAX, X86_FPU_INT16_INDEFINITE, RT_NOTHING, 1)
+#endif
+EMIT_FISTT(16, 16, int16_t, INT16_MIN, INT16_MAX, X86_FPU_INT16_INDEFINITE, _intel,     1)
+EMIT_FISTT(16, 16, int16_t, INT16_MIN, INT16_MAX, X86_FPU_INT16_INDEFINITE, _amd,       0)
+
+
+#if defined(IEM_WITHOUT_ASSEMBLY)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_d80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
+                                                 PRTPBCD80U pd80Dst, PCRTFLOAT80U pr80Src))
+{
+    /*static RTPBCD80U const s_ad80MaxMin[2] = { RTPBCD80U_INIT_MAX(),   RTPBCD80U_INIT_MIN() };*/
+    static RTPBCD80U const s_ad80Zeros[2]  = { RTPBCD80U_INIT_ZERO(0), RTPBCD80U_INIT_ZERO(1) };
+    static RTPBCD80U const s_ad80One[2]    = { RTPBCD80U_INIT_C(0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,1),
+                                               RTPBCD80U_INIT_C(1, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,1) };
+    static RTPBCD80U const s_d80Indefinite = RTPBCD80U_INIT_INDEFINITE();
+
+    uint16_t const fFcw    = pFpuState->FCW;
+    uint16_t       fFsw    = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3));
+    bool const     fSignIn = pr80Src->s.fSign;
+
+    /*
+     * Deal with normal numbers first.
+     */
+    if (RTFLOAT80U_IS_NORMAL(pr80Src))
+    {
+        uint64_t       uMantissa = pr80Src->s.uMantissa;
+        int32_t        iExponent = (int32_t)pr80Src->s.uExponent - RTFLOAT80U_EXP_BIAS;
+        if (   (uint32_t)iExponent <= 58
+            || ((uint32_t)iExponent == 59 && uMantissa <= UINT64_C(0xde0b6b3a763fffff)) )
+        {
+            unsigned const cShiftOff        = 63 - iExponent;
+            uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1;
+            uint64_t const uRoundingAdd     = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
+                                            ? RT_BIT_64(cShiftOff - 1)
+                                            : (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)
+                                            ? fRoundingOffMask
+                                            : 0;
+            uint64_t       fRoundedOff      = uMantissa & fRoundingOffMask;
+
+            uMantissa >>= cShiftOff;
+            uint64_t const uRounding = (fRoundedOff + uRoundingAdd) >> cShiftOff;
+            uMantissa += uRounding;
+            if (uMantissa <= (uint64_t)RTPBCD80U_MAX)
+            {
+                if (fRoundedOff)
+                {
+                    if ((uMantissa & 1) && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST && fRoundedOff == uRoundingAdd)
+                        uMantissa &= ~(uint64_t)1; /* round to even number if equal distance between up/down. */
+                    else if (uRounding)
+                        fFsw |= X86_FSW_C1;
+                    fFsw |= X86_FSW_PE;
+                    if (!(fFcw & X86_FCW_PM))
+                        fFsw |= X86_FSW_ES | X86_FSW_B;
+                }
+
+                pd80Dst->s.fSign = fSignIn;
+                pd80Dst->s.uPad  = 0;
+                for (size_t iPair = 0; iPair < RT_ELEMENTS(pd80Dst->s.abPairs); iPair++)
+                {
+                    unsigned const uDigits = uMantissa % 100;
+                    uMantissa /= 100;
+                    uint8_t const bLo = uDigits % 10;
+                    uint8_t const bHi = uDigits / 10;
+                    pd80Dst->s.abPairs[iPair] = RTPBCD80U_MAKE_PAIR(bHi, bLo);
+                }
+            }
+            else
+            {
+                /* overflowed after rounding. */
+                fFsw |= X86_FSW_IE;
+                if (fFcw & X86_FCW_IM)
+                    *pd80Dst = s_d80Indefinite;
+                else
+                    fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT);
+            }
+        }
+        /*
+         * Tiny sub-zero numbers.
+         */
+        else if (iExponent < 0)
+        {
+            if (!fSignIn)
+            {
+                if (   (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP
+                    || (iExponent == -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST))
+                {
+                    *pd80Dst = s_ad80One[fSignIn];
+                    fFsw |= X86_FSW_C1;
+                }
+                else
+                    *pd80Dst = s_ad80Zeros[fSignIn];
+            }
+            else
+            {
+                if (   (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP
+                    || (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_ZERO
+                    || (iExponent < -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST))
+                    *pd80Dst = s_ad80Zeros[fSignIn];
+                else
+                {
+                    *pd80Dst = s_ad80One[fSignIn];
+                    fFsw |= X86_FSW_C1;
+                }
+            }
+            fFsw |= X86_FSW_PE;
+            if (!(fFcw & X86_FCW_PM))
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        /*
+         * Too large/small number outside the target integer range.
+         */
+        else
+        {
+            fFsw |= X86_FSW_IE;
+            if (fFcw & X86_FCW_IM)
+                *pd80Dst = s_d80Indefinite;
+            else
+                fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT);
+        }
+    }
+    /*
+     * Map both +0 and -0 to integer zero (signless/+).
+     */
+    else if (RTFLOAT80U_IS_ZERO(pr80Src))
+        *pd80Dst = s_ad80Zeros[fSignIn];
+    /*
+     * Denormals are just really tiny sub-zero numbers that are either rounded
+     * to zero, 1 or -1 depending on sign and rounding control.
+     */
+    else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Src) || RTFLOAT80U_IS_DENORMAL(pr80Src))
+    {
+        if ((fFcw & X86_FCW_RC_MASK) != (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP))
+            *pd80Dst = s_ad80Zeros[fSignIn];
+        else
+        {
+            *pd80Dst = s_ad80One[fSignIn];
+            fFsw |= X86_FSW_C1;
+        }
+        fFsw |= X86_FSW_PE;
+        if (!(fFcw & X86_FCW_PM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+    /*
+     * All other special values are considered invalid arguments and result
+     * in an IE exception and indefinite value if masked.
+     */
+    else
+    {
+        fFsw |= X86_FSW_IE;
+        if (fFcw & X86_FCW_IM)
+            *pd80Dst = s_d80Indefinite;
+        else
+            fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT);
+    }
+    *pu16FSW = fFsw;
+}
+
+
+/*********************************************************************************************************************************
+*   FPU Helpers                                                                                                                  *
+*********************************************************************************************************************************/
+AssertCompileSize(RTFLOAT128U, 16);
+AssertCompileSize(RTFLOAT80U,  10);
+AssertCompileSize(RTFLOAT64U,   8);
+AssertCompileSize(RTFLOAT32U,   4);
+
+/**
+ * Normalizes a possible pseudo-normal value.
+ *
+ * Psuedo-normal values are some oddities from the 8087 & 287 days.  They are
+ * denormals with the J-bit set, so they can simply be rewritten as 2**-16382,
+ * i.e. changing uExponent from 0 to 1.
+ *
+ * This macro will declare a RTFLOAT80U with the name given by
+ * @a a_r80ValNormalized and update the @a a_pr80Val variable to point to it if
+ * a normalization was performed.
+ *
+ * @note This must be applied before calling SoftFloat with a value that couldbe
+ *       a pseudo-denormal, as SoftFloat doesn't handle pseudo-denormals
+ *       correctly.
+ */
+#define IEM_NORMALIZE_PSEUDO_DENORMAL(a_pr80Val, a_r80ValNormalized) \
+    RTFLOAT80U a_r80ValNormalized; \
+    if (RTFLOAT80U_IS_PSEUDO_DENORMAL(a_pr80Val)) \
+    { \
+        a_r80ValNormalized = *a_pr80Val; \
+        a_r80ValNormalized.s.uExponent = 1; \
+        a_pr80Val = &a_r80ValNormalized; \
+    } else do {} while (0)
+
+#ifdef IEM_WITH_FLOAT128_FOR_FPU
+
+DECLINLINE(int) iemFpuF128SetRounding(uint16_t fFcw)
+{
+    int fNew;
+    switch (fFcw & X86_FCW_RC_MASK)
+    {
+        default:
+        case X86_FCW_RC_NEAREST:    fNew = FE_TONEAREST; break;
+        case X86_FCW_RC_ZERO:       fNew = FE_TOWARDZERO; break;
+        case X86_FCW_RC_UP:         fNew = FE_UPWARD; break;
+        case X86_FCW_RC_DOWN:       fNew = FE_DOWNWARD; break;
+    }
+    int fOld = fegetround();
+    fesetround(fNew);
+    return fOld;
+}
+
+
+DECLINLINE(void) iemFpuF128RestoreRounding(int fOld)
+{
+    fesetround(fOld);
+}
+
+DECLINLINE(_Float128) iemFpuF128FromFloat80(PCRTFLOAT80U pr80Val, uint16_t fFcw)
+{
+    RT_NOREF(fFcw);
+    RTFLOAT128U Tmp;
+    Tmp.s2.uSignAndExponent = pr80Val->s2.uSignAndExponent;
+    Tmp.s2.uFractionHigh    = (uint16_t)((pr80Val->s2.uMantissa & (RT_BIT_64(63) - 1)) >> 48);
+    Tmp.s2.uFractionMid     = (uint32_t)((pr80Val->s2.uMantissa & UINT32_MAX) >> 16);
+    Tmp.s2.uFractionLow     = pr80Val->s2.uMantissa << 48;
+    if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val))
+    {
+        Assert(Tmp.s.uExponent == 0);
+        Tmp.s2.uSignAndExponent++;
+    }
+    return *(_Float128 *)&Tmp;
+}
+
+
+DECLINLINE(uint16_t) iemFpuF128ToFloat80(PRTFLOAT80U pr80Dst, _Float128 rd128ValSrc, uint16_t fFcw, uint16_t fFsw)
+{
+    RT_NOREF(fFcw);
+    RTFLOAT128U Tmp;
+    *(_Float128 *)&Tmp = rd128ValSrc;
+    ASMCompilerBarrier();
+    if (RTFLOAT128U_IS_NORMAL(&Tmp))
+    {
+        pr80Dst->s.fSign     = Tmp.s64.fSign;
+        pr80Dst->s.uExponent = Tmp.s64.uExponent;
+        uint64_t uFraction   = Tmp.s64.uFractionHi << (63 - 48)
+                             | Tmp.s64.uFractionLo >> (64 - 15);
+
+        /* Do rounding - just truncate in near mode when midway on an even outcome. */
+        unsigned const cShiftOff        = 64 - 15;
+        uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1;
+        uint64_t const uRoundedOff      = Tmp.s64.uFractionLo & fRoundingOffMask;
+        if (uRoundedOff)
+        {
+            uint64_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
+                                        ? RT_BIT_64(cShiftOff - 1)
+                                        : (fFcw & X86_FCW_RC_MASK) == (Tmp.s64.fSign ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)
+                                        ? fRoundingOffMask
+                                        : 0;
+            if (   (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_NEAREST
+                || (Tmp.s64.uFractionLo & RT_BIT_64(cShiftOff))
+                || uRoundedOff != uRoundingAdd)
+            {
+                if ((uRoundedOff + uRoundingAdd) >> cShiftOff)
+                {
+                    uFraction += 1;
+                    if (!(uFraction & RT_BIT_64(63)))
+                    { /* likely */ }
+                    else
+                    {
+                        uFraction >>= 1;
+                        pr80Dst->s.uExponent++;
+                        if (pr80Dst->s.uExponent == RTFLOAT64U_EXP_MAX)
+                            return fFsw;
+                    }
+                    fFsw |= X86_FSW_C1;
+                }
+            }
+            fFsw |= X86_FSW_PE;
+            if (!(fFcw & X86_FCW_PM))
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        pr80Dst->s.uMantissa = RT_BIT_64(63) | uFraction;
+    }
+    else if (RTFLOAT128U_IS_ZERO(&Tmp))
+    {
+        pr80Dst->s.fSign     = Tmp.s64.fSign;
+        pr80Dst->s.uExponent = 0;
+        pr80Dst->s.uMantissa = 0;
+    }
+    else if (RTFLOAT128U_IS_INF(&Tmp))
+    {
+        pr80Dst->s.fSign     = Tmp.s64.fSign;
+        pr80Dst->s.uExponent = 0;
+        pr80Dst->s.uMantissa = 0;
+    }
+    return fFsw;
+}
+
+
+#else  /* !IEM_WITH_FLOAT128_FOR_FPU - SoftFloat */
+
+/** Initializer for the SoftFloat state structure. */
+# define IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(a_fFcw) \
+    { \
+        softfloat_tininess_afterRounding, \
+          ((a_fFcw) & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST ? (uint8_t)softfloat_round_near_even \
+        : ((a_fFcw) & X86_FCW_RC_MASK) == X86_FCW_RC_UP      ? (uint8_t)softfloat_round_max \
+        : ((a_fFcw) & X86_FCW_RC_MASK) == X86_FCW_RC_DOWN    ? (uint8_t)softfloat_round_min \
+        :                                                      (uint8_t)softfloat_round_minMag, \
+        0, \
+        (uint8_t)((a_fFcw) & X86_FCW_XCPT_MASK), \
+          ((a_fFcw) & X86_FCW_PC_MASK) == X86_FCW_PC_53      ? (uint8_t)64 \
+        : ((a_fFcw) & X86_FCW_PC_MASK) == X86_FCW_PC_24      ? (uint8_t)32 : (uint8_t)80 \
+    }
+
+/** Returns updated FSW from a SoftFloat state and exception mask (FCW). */
+# define IEM_SOFTFLOAT_STATE_TO_FSW(a_fFsw, a_pSoftState, a_fFcw) \
+    (  (a_fFsw) \
+     | (uint16_t)(((a_pSoftState)->exceptionFlags & softfloat_flag_c1) << 2) \
+     | ((a_pSoftState)->exceptionFlags & X86_FSW_XCPT_MASK) \
+     | (  ((a_pSoftState)->exceptionFlags & X86_FSW_XCPT_MASK) & (~(a_fFcw) & X86_FSW_XCPT_MASK) \
+        ? X86_FSW_ES | X86_FSW_B : 0) )
+
+
+DECLINLINE(float128_t) iemFpuSoftF128Precision(float128_t r128, unsigned cBits, uint16_t fFcw = X86_FCW_RC_NEAREST)
+{
+    RT_NOREF(fFcw);
+    Assert(cBits > 64);
+# if 0 /* rounding does not seem to help */
+    uint64_t off = r128.v[0] & (RT_BIT_64(1 + 112 - cBits) - 1);
+    r128.v[0] &= ~(RT_BIT_64(1 + 112 - cBits) - 1);
+    if (off >= RT_BIT_64(1 + 112 - cBits - 1)
+        && (r128.v[0] & RT_BIT_64(1 + 112 - cBits)))
+    {
+        uint64_t uOld = r128.v[0];
+        r128.v[0] += RT_BIT_64(1 + 112 - cBits);
+        if (r128.v[0] < uOld)
+            r128.v[1] += 1;
+    }
+# else
+    r128.v[0] &= ~(RT_BIT_64(1 + 112 - cBits) - 1);
+# endif
+    return r128;
+}
+
+
+DECLINLINE(float128_t) iemFpuSoftF128PrecisionIprt(PCRTFLOAT128U pr128, unsigned cBits, uint16_t fFcw = X86_FCW_RC_NEAREST)
+{
+    RT_NOREF(fFcw);
+    Assert(cBits > 64);
+# if 0 /* rounding does not seem to help, not even on constants */
+    float128_t r128 = { pr128->au64[0], pr128->au64[1] };
+    uint64_t off = r128.v[0] & (RT_BIT_64(1 + 112 - cBits) - 1);
+    r128.v[0] &= ~(RT_BIT_64(1 + 112 - cBits) - 1);
+    if (off >= RT_BIT_64(1 + 112 - cBits - 1)
+        && (r128.v[0] & RT_BIT_64(1 + 112 - cBits)))
+    {
+        uint64_t uOld = r128.v[0];
+        r128.v[0] += RT_BIT_64(1 + 112 - cBits);
+        if (r128.v[0] < uOld)
+            r128.v[1] += 1;
+    }
+    return r128;
+# else
+    float128_t r128 = { { pr128->au64[0] & ~(RT_BIT_64(1 + 112 - cBits) - 1), pr128->au64[1] } };
+    return r128;
+# endif
+}
+
+
+# if 0  /*  unused */
+DECLINLINE(float128_t) iemFpuSoftF128FromIprt(PCRTFLOAT128U pr128)
+{
+    float128_t r128 = { { pr128->au64[0], pr128->au64[1] } };
+    return r128;
+}
+# endif
+
+
+/** Converts a 80-bit floating point value to SoftFloat 128-bit floating point. */
+DECLINLINE(float128_t) iemFpuSoftF128FromFloat80(PCRTFLOAT80U pr80Val)
+{
+    extFloat80_t Tmp;
+    Tmp.signExp = pr80Val->s2.uSignAndExponent;
+    Tmp.signif  = pr80Val->s2.uMantissa;
+    softfloat_state_t Ignored = SOFTFLOAT_STATE_INIT_DEFAULTS();
+    return extF80_to_f128(Tmp, &Ignored);
+}
+
+
+/**
+ * Converts from the packed IPRT 80-bit floating point (RTFLOAT80U) format to
+ * the SoftFloat extended 80-bit floating point format (extFloat80_t).
+ *
+ * This is only a structure format conversion, nothing else.
+ */
+DECLINLINE(extFloat80_t) iemFpuSoftF80FromIprt(PCRTFLOAT80U pr80Val)
+{
+    extFloat80_t Tmp;
+    Tmp.signExp = pr80Val->s2.uSignAndExponent;
+    Tmp.signif  = pr80Val->s2.uMantissa;
+    return Tmp;
+}
+
+
+/**
+ * Converts from SoftFloat extended 80-bit floating point format (extFloat80_t)
+ * to the packed IPRT 80-bit floating point (RTFLOAT80U) format.
+ *
+ * This is only a structure format conversion, nothing else.
+ */
+DECLINLINE(PRTFLOAT80U) iemFpuSoftF80ToIprt(PRTFLOAT80U pr80Dst, extFloat80_t const r80XSrc)
+{
+    pr80Dst->s2.uSignAndExponent = r80XSrc.signExp;
+    pr80Dst->s2.uMantissa        = r80XSrc.signif;
+    return pr80Dst;
+}
+
+
+DECLINLINE(uint16_t) iemFpuSoftF128ToFloat80(PRTFLOAT80U pr80Dst, float128_t r128Src, uint16_t fFcw, uint16_t fFsw)
+{
+    RT_NOREF(fFcw);
+    RTFLOAT128U Tmp;
+    *(float128_t *)&Tmp = r128Src;
+    ASMCompilerBarrier();
+
+    if (RTFLOAT128U_IS_NORMAL(&Tmp))
+    {
+        pr80Dst->s.fSign     = Tmp.s64.fSign;
+        pr80Dst->s.uExponent = Tmp.s64.uExponent;
+        uint64_t uFraction   = Tmp.s64.uFractionHi << (63 - 48)
+                             | Tmp.s64.uFractionLo >> (64 - 15);
+
+        /* Do rounding - just truncate in near mode when midway on an even outcome. */
+        unsigned const cShiftOff        = 64 - 15;
+        uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1;
+        uint64_t const uRoundedOff      = Tmp.s64.uFractionLo & fRoundingOffMask;
+        if (uRoundedOff)
+        {
+            uint64_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
+                                        ? RT_BIT_64(cShiftOff - 1)
+                                        : (fFcw & X86_FCW_RC_MASK) == (Tmp.s64.fSign ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)
+                                        ? fRoundingOffMask
+                                        : 0;
+            if (   (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_NEAREST
+                || (Tmp.s64.uFractionLo & RT_BIT_64(cShiftOff))
+                || uRoundedOff != uRoundingAdd)
+            {
+                if ((uRoundedOff + uRoundingAdd) >> cShiftOff)
+                {
+                    uFraction += 1;
+                    if (!(uFraction & RT_BIT_64(63)))
+                    { /* likely */ }
+                    else
+                    {
+                        uFraction >>= 1;
+                        pr80Dst->s.uExponent++;
+                        if (pr80Dst->s.uExponent == RTFLOAT64U_EXP_MAX)
+                            return fFsw;
+                    }
+                    fFsw |= X86_FSW_C1;
+                }
+            }
+            fFsw |= X86_FSW_PE;
+            if (!(fFcw & X86_FCW_PM))
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+
+        pr80Dst->s.uMantissa = RT_BIT_64(63) | uFraction;
+    }
+    else if (RTFLOAT128U_IS_ZERO(&Tmp))
+    {
+        pr80Dst->s.fSign     = Tmp.s64.fSign;
+        pr80Dst->s.uExponent = 0;
+        pr80Dst->s.uMantissa = 0;
+    }
+    else if (RTFLOAT128U_IS_INF(&Tmp))
+    {
+        pr80Dst->s.fSign     = Tmp.s64.fSign;
+        pr80Dst->s.uExponent = 0x7fff;
+        pr80Dst->s.uMantissa = 0;
+    }
+    return fFsw;
+}
+
+
+/**
+ * Helper for transfering exception and C1 to FSW and setting the result value
+ * accordingly.
+ *
+ * @returns Updated FSW.
+ * @param   pSoftState      The SoftFloat state following the operation.
+ * @param   r80XResult      The result of the SoftFloat operation.
+ * @param   pr80Result      Where to store the result for IEM.
+ * @param   fFcw            The FPU control word.
+ * @param   fFsw            The FSW before the operation, with necessary bits
+ *                          cleared and such.
+ * @param   pr80XcptResult  Alternative return value for use an unmasked \#IE is
+ *                          raised.
+ */
+DECLINLINE(uint16_t) iemFpuSoftStateAndF80ToFswAndIprtResult(softfloat_state_t const *pSoftState, extFloat80_t r80XResult,
+                                                             PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw,
+                                                             PCRTFLOAT80U pr80XcptResult)
+{
+    fFsw |= (pSoftState->exceptionFlags & X86_FSW_XCPT_MASK)
+         | (uint16_t)((pSoftState->exceptionFlags & softfloat_flag_c1) << 2);
+    if (fFsw & ~fFcw & X86_FSW_XCPT_MASK)
+        fFsw |= X86_FSW_ES | X86_FSW_B;
+
+    if (!(fFsw & ~fFcw & (X86_FSW_IE | X86_FSW_DE)))
+        iemFpuSoftF80ToIprt(pr80Result, r80XResult);
+    else
+    {
+        fFsw &= ~(X86_FSW_OE | X86_FSW_UE | X86_FSW_PE | X86_FSW_ZE | X86_FSW_C1);
+        *pr80Result = *pr80XcptResult;
+    }
+    return fFsw;
+}
+
+
+/**
+ * Helper doing polynomial evaluation using Horner's method.
+ *
+ * See https://en.wikipedia.org/wiki/Horner%27s_method for details.
+ */
+float128_t iemFpuSoftF128HornerPoly(float128_t z, PCRTFLOAT128U g_par128HornerConsts, size_t cHornerConsts,
+                                    unsigned cPrecision, softfloat_state_t *pSoftState)
+{
+    Assert(cHornerConsts > 1);
+    size_t     i          = cHornerConsts - 1;
+    float128_t r128Result = iemFpuSoftF128PrecisionIprt(&g_par128HornerConsts[i], cPrecision);
+    while (i-- > 0)
+    {
+        r128Result = iemFpuSoftF128Precision(f128_mul(r128Result, z, pSoftState), cPrecision);
+        r128Result = f128_add(r128Result, iemFpuSoftF128PrecisionIprt(&g_par128HornerConsts[i], cPrecision), pSoftState);
+        r128Result = iemFpuSoftF128Precision(r128Result, cPrecision);
+    }
+    return r128Result;
+}
+
+#endif /* !IEM_WITH_FLOAT128_FOR_FPU - SoftFloat */
+
+
+/**
+ * Composes a normalized and rounded RTFLOAT80U result from a 192 bit wide
+ * mantissa, exponent and sign.
+ *
+ * @returns Updated FSW.
+ * @param   pr80Dst     Where to return the composed value.
+ * @param   fSign       The sign.
+ * @param   puMantissa  The mantissa, 256-bit type but the to 64-bits are
+ *                      ignored and should be zero.  This will probably be
+ *                      modified during normalization and rounding.
+ * @param   iExponent   Unbiased exponent.
+ * @param   fFcw        The FPU control word.
+ * @param   fFsw        The FPU status word.
+ */
+static uint16_t iemFpuFloat80RoundAndComposeFrom192(PRTFLOAT80U pr80Dst, bool fSign, PRTUINT256U puMantissa,
+                                                    int32_t iExponent, uint16_t fFcw, uint16_t fFsw)
+{
+    AssertStmt(puMantissa->QWords.qw3 == 0, puMantissa->QWords.qw3 = 0);
+
+    iExponent += RTFLOAT80U_EXP_BIAS;
+
+    /* Do normalization if necessary and possible. */
+    if (!(puMantissa->QWords.qw2 & RT_BIT_64(63)))
+    {
+        int cShift = 192 - RTUInt256BitCount(puMantissa);
+        if (iExponent > cShift)
+            iExponent -= cShift;
+        else
+        {
+            if (fFcw & X86_FCW_UM)
+            {
+                if (iExponent > 0)
+                    cShift = --iExponent;
+                else
+                    cShift = 0;
+            }
+            iExponent -= cShift;
+        }
+        RTUInt256AssignShiftLeft(puMantissa, cShift);
+    }
+
+    /* Do rounding. */
+    uint64_t uMantissa = puMantissa->QWords.qw2;
+    if (puMantissa->QWords.qw1 || puMantissa->QWords.qw0)
+    {
+        bool fAdd;
+        switch (fFcw & X86_FCW_RC_MASK)
+        {
+            default: /* (for the simple-minded MSC which otherwise things fAdd would be used uninitialized) */
+            case X86_FCW_RC_NEAREST:
+                if (puMantissa->QWords.qw1 & RT_BIT_64(63))
+                {
+                    if (   (uMantissa & 1)
+                        || puMantissa->QWords.qw0 != 0
+                        || puMantissa->QWords.qw1 != RT_BIT_64(63))
+                    {
+                        fAdd = true;
+                        break;
+                    }
+                    uMantissa &= ~(uint64_t)1;
+                }
+                fAdd = false;
+                break;
+            case X86_FCW_RC_ZERO:
+                fAdd = false;
+                break;
+            case X86_FCW_RC_UP:
+                fAdd = !fSign;
+                break;
+            case X86_FCW_RC_DOWN:
+                fAdd = fSign;
+                break;
+        }
+        if (fAdd)
+        {
+            uint64_t const uTmp = uMantissa;
+            uMantissa = uTmp + 1;
+            if (uMantissa < uTmp)
+            {
+                uMantissa >>= 1;
+                uMantissa |= RT_BIT_64(63);
+                iExponent++;
+            }
+            fFsw |= X86_FSW_C1;
+        }
+        fFsw |= X86_FSW_PE;
+        if (!(fFcw & X86_FCW_PM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+
+    /* Check for underflow (denormals). */
+    if (iExponent <= 0)
+    {
+        if (fFcw & X86_FCW_UM)
+        {
+            if (uMantissa & RT_BIT_64(63))
+                uMantissa >>= 1;
+            iExponent = 0;
+        }
+        else
+        {
+            iExponent += RTFLOAT80U_EXP_BIAS_ADJUST;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_UE;
+    }
+    /* Check for overflow */
+    else if (iExponent >= RTFLOAT80U_EXP_MAX)
+    {
+        Assert(iExponent < RTFLOAT80U_EXP_MAX);
+    }
+
+    /* Compose the result. */
+    pr80Dst->s.uMantissa = uMantissa;
+    pr80Dst->s.uExponent = iExponent;
+    pr80Dst->s.fSign     = fSign;
+    return fFsw;
+}
+
+
+/**
+ * See also iemAImpl_fld_r80_from_r32
+ */
+static uint16_t iemAImplConvertR32ToR80(PCRTFLOAT32U pr32Val, PRTFLOAT80U pr80Dst)
+{
+    uint16_t fFsw = 0;
+    if (RTFLOAT32U_IS_NORMAL(pr32Val))
+    {
+        pr80Dst->sj64.fSign     = pr32Val->s.fSign;
+        pr80Dst->sj64.fInteger  = 1;
+        pr80Dst->sj64.uFraction = (uint64_t)pr32Val->s.uFraction
+                               << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS);
+        pr80Dst->sj64.uExponent = pr32Val->s.uExponent - RTFLOAT32U_EXP_BIAS + RTFLOAT80U_EXP_BIAS;
+        Assert(RTFLOAT80U_IS_NORMAL(pr80Dst));
+    }
+    else if (RTFLOAT32U_IS_ZERO(pr32Val))
+    {
+        pr80Dst->s.fSign     = pr32Val->s.fSign;
+        pr80Dst->s.uExponent = 0;
+        pr80Dst->s.uMantissa = 0;
+        Assert(RTFLOAT80U_IS_ZERO(pr80Dst));
+    }
+    else if (RTFLOAT32U_IS_SUBNORMAL(pr32Val))
+    {
+        /* Subnormal -> normalized + X86_FSW_DE return. */
+        pr80Dst->sj64.fSign     = pr32Val->s.fSign;
+        pr80Dst->sj64.fInteger  = 1;
+        unsigned const cExtraShift = RTFLOAT32U_FRACTION_BITS - ASMBitLastSetU32(pr32Val->s.uFraction);
+        pr80Dst->sj64.uFraction = (uint64_t)pr32Val->s.uFraction
+                               << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS + cExtraShift + 1);
+        pr80Dst->sj64.uExponent = pr32Val->s.uExponent - RTFLOAT32U_EXP_BIAS + RTFLOAT80U_EXP_BIAS - cExtraShift;
+        fFsw = X86_FSW_DE;
+    }
+    else if (RTFLOAT32U_IS_INF(pr32Val))
+    {
+        pr80Dst->s.fSign     = pr32Val->s.fSign;
+        pr80Dst->s.uExponent = RTFLOAT80U_EXP_MAX;
+        pr80Dst->s.uMantissa = RT_BIT_64(63);
+        Assert(RTFLOAT80U_IS_INF(pr80Dst));
+    }
+    else
+    {
+        Assert(RTFLOAT32U_IS_NAN(pr32Val));
+        pr80Dst->sj64.fSign     = pr32Val->s.fSign;
+        pr80Dst->sj64.uExponent = RTFLOAT80U_EXP_MAX;
+        pr80Dst->sj64.fInteger  = 1;
+        pr80Dst->sj64.uFraction = (uint64_t)pr32Val->s.uFraction
+                               << (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS);
+        Assert(RTFLOAT80U_IS_NAN(pr80Dst));
+        Assert(RTFLOAT80U_IS_SIGNALLING_NAN(pr80Dst) == RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val));
+    }
+    return fFsw;
+}
+
+
+/**
+ * See also iemAImpl_fld_r80_from_r64
+ */
+static uint16_t iemAImplConvertR64ToR80(PCRTFLOAT64U pr64Val, PRTFLOAT80U pr80Dst)
+{
+    uint16_t fFsw = 0;
+    if (RTFLOAT64U_IS_NORMAL(pr64Val))
+    {
+        pr80Dst->sj64.fSign     = pr64Val->s.fSign;
+        pr80Dst->sj64.fInteger  = 1;
+        pr80Dst->sj64.uFraction = pr64Val->s64.uFraction << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS);
+        pr80Dst->sj64.uExponent = pr64Val->s.uExponent - RTFLOAT64U_EXP_BIAS + RTFLOAT80U_EXP_BIAS;
+        Assert(RTFLOAT80U_IS_NORMAL(pr80Dst));
+    }
+    else if (RTFLOAT64U_IS_ZERO(pr64Val))
+    {
+        pr80Dst->s.fSign     = pr64Val->s.fSign;
+        pr80Dst->s.uExponent = 0;
+        pr80Dst->s.uMantissa = 0;
+        Assert(RTFLOAT80U_IS_ZERO(pr80Dst));
+    }
+    else if (RTFLOAT64U_IS_SUBNORMAL(pr64Val))
+    {
+        /* Subnormal values gets normalized. */
+        pr80Dst->sj64.fSign     = pr64Val->s.fSign;
+        pr80Dst->sj64.fInteger  = 1;
+        unsigned const cExtraShift = RTFLOAT64U_FRACTION_BITS - ASMBitLastSetU64(pr64Val->s64.uFraction);
+        pr80Dst->sj64.uFraction = pr64Val->s64.uFraction
+                               << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS + cExtraShift + 1);
+        pr80Dst->sj64.uExponent = pr64Val->s.uExponent - RTFLOAT64U_EXP_BIAS + RTFLOAT80U_EXP_BIAS - cExtraShift;
+        fFsw = X86_FSW_DE;
+    }
+    else if (RTFLOAT64U_IS_INF(pr64Val))
+    {
+        pr80Dst->s.fSign     = pr64Val->s.fSign;
+        pr80Dst->s.uExponent = RTFLOAT80U_EXP_MAX;
+        pr80Dst->s.uMantissa = RT_BIT_64(63);
+        Assert(RTFLOAT80U_IS_INF(pr80Dst));
+    }
+    else
+    {
+        /* Signalling and quiet NaNs, both turn into quiet ones when loaded (weird). */
+        Assert(RTFLOAT64U_IS_NAN(pr64Val));
+        pr80Dst->sj64.fSign     = pr64Val->s.fSign;
+        pr80Dst->sj64.uExponent = RTFLOAT80U_EXP_MAX;
+        pr80Dst->sj64.fInteger  = 1;
+        pr80Dst->sj64.uFraction = pr64Val->s64.uFraction << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS);
+        Assert(RTFLOAT80U_IS_NAN(pr80Dst));
+        Assert(RTFLOAT80U_IS_SIGNALLING_NAN(pr80Dst) == RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val));
+    }
+    return fFsw;
+}
+
+
+/**
+ * See also EMIT_FILD.
+ */
+#define EMIT_CONVERT_IXX_TO_R80(a_cBits) \
+static PRTFLOAT80U iemAImplConvertI ## a_cBits ## ToR80(int ## a_cBits ## _t iVal, PRTFLOAT80U pr80Dst) \
+{ \
+    if (iVal == 0) \
+    { \
+        pr80Dst->s.fSign      = 0; \
+        pr80Dst->s.uExponent  = 0; \
+        pr80Dst->s.uMantissa  = 0; \
+    } \
+    else \
+    { \
+        if (iVal > 0) \
+            pr80Dst->s.fSign  = 0; \
+        else \
+        { \
+            pr80Dst->s.fSign  = 1; \
+            iVal = -iVal; \
+        } \
+        unsigned const cBits = ASMBitLastSetU ## a_cBits((uint ## a_cBits ## _t)iVal); \
+        pr80Dst->s.uExponent  = cBits - 1 + RTFLOAT80U_EXP_BIAS; \
+        pr80Dst->s.uMantissa  = (uint64_t)iVal << (RTFLOAT80U_FRACTION_BITS + 1 - cBits); \
+    } \
+    return pr80Dst; \
+}
+EMIT_CONVERT_IXX_TO_R80(16)
+EMIT_CONVERT_IXX_TO_R80(32)
+//EMIT_CONVERT_IXX_TO_R80(64)
+
+/** For implementing iemAImpl_fmul_r80_by_r64 and such. */
+#define EMIT_R80_BY_R64(a_Name, a_fnR80ByR80, a_DenormalException) \
+IEM_DECL_IMPL_DEF(void, a_Name,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2)) \
+{ \
+    RTFLOAT80U r80Val2; \
+    uint16_t   fFsw = iemAImplConvertR64ToR80(pr64Val2, &r80Val2); \
+    Assert(!fFsw || fFsw == X86_FSW_DE); \
+    if (fFsw) \
+    { \
+        if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_NAN(pr80Val1) || (a_DenormalException)) \
+            fFsw = 0; \
+        else if (!(pFpuState->FCW & X86_FCW_DM)) \
+        { \
+            pFpuRes->r80Result = *pr80Val1; \
+            pFpuRes->FSW       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT) \
+                               | X86_FSW_DE | X86_FSW_ES | X86_FSW_B; \
+            return; \
+        } \
+    } \
+    a_fnR80ByR80(pFpuState, pFpuRes, pr80Val1, &r80Val2); \
+    pFpuRes->FSW = (pFpuRes->FSW & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT) | fFsw; \
+}
+
+/** For implementing iemAImpl_fmul_r80_by_r32 and such. */
+#define EMIT_R80_BY_R32(a_Name, a_fnR80ByR80, a_DenormalException) \
+IEM_DECL_IMPL_DEF(void, a_Name,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2)) \
+{ \
+    RTFLOAT80U r80Val2; \
+    uint16_t   fFsw = iemAImplConvertR32ToR80(pr32Val2, &r80Val2); \
+    Assert(!fFsw || fFsw == X86_FSW_DE); \
+    if (fFsw) \
+    { \
+        if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_NAN(pr80Val1) || (a_DenormalException)) \
+            fFsw = 0; \
+        else if (!(pFpuState->FCW & X86_FCW_DM)) \
+        { \
+            pFpuRes->r80Result = *pr80Val1; \
+            pFpuRes->FSW       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT) \
+                               | X86_FSW_DE | X86_FSW_ES | X86_FSW_B; \
+            return; \
+        } \
+    } \
+    a_fnR80ByR80(pFpuState, pFpuRes, pr80Val1, &r80Val2); \
+    pFpuRes->FSW = (pFpuRes->FSW & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT) | fFsw; \
+}
+
+/** For implementing iemAImpl_fimul_r80_by_i32 and such. */
+#define EMIT_R80_BY_I32(a_Name, a_fnR80ByR80) \
+IEM_DECL_IMPL_DEF(void, a_Name,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2)) \
+{ \
+    RTFLOAT80U r80Val2; \
+    a_fnR80ByR80(pFpuState, pFpuRes, pr80Val1, iemAImplConvertI32ToR80(*pi32Val2, &r80Val2)); \
+    pFpuRes->FSW = (pFpuRes->FSW & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT); \
+}
+
+/** For implementing iemAImpl_fimul_r80_by_i16 and such. */
+#define EMIT_R80_BY_I16(a_Name, a_fnR80ByR80) \
+IEM_DECL_IMPL_DEF(void, a_Name,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2)) \
+{ \
+    RTFLOAT80U r80Val2; \
+    a_fnR80ByR80(pFpuState, pFpuRes, pr80Val1, iemAImplConvertI16ToR80(*pi16Val2, &r80Val2)); \
+    pFpuRes->FSW = (pFpuRes->FSW & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT);  \
+}
+
+
+
+/*********************************************************************************************************************************
+*   x86 FPU Division Operations                                                                                                  *
+*********************************************************************************************************************************/
+
+/** Worker for iemAImpl_fdiv_r80_by_r80 & iemAImpl_fdivr_r80_by_r80. */
+static uint16_t iemAImpl_fdiv_f80_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result,
+                                             uint16_t fFcw, uint16_t fFsw, PCRTFLOAT80U pr80Val1Org)
+{
+    if (!RTFLOAT80U_IS_ZERO(pr80Val2) || RTFLOAT80U_IS_NAN(pr80Val1) || RTFLOAT80U_IS_INF(pr80Val1))
+    {
+        softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw);
+        extFloat80_t r80XResult = extF80_div(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), &SoftState);
+        return iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, pr80Result, fFcw, fFsw, pr80Val1Org);
+    }
+    if (!RTFLOAT80U_IS_ZERO(pr80Val1))
+    {   /* Div by zero. */
+        if (fFcw & X86_FCW_ZM)
+            *pr80Result = g_ar80Infinity[pr80Val1->s.fSign != pr80Val2->s.fSign];
+        else
+        {
+            *pr80Result = *pr80Val1Org;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_ZE;
+    }
+    else
+    {   /* Invalid operand */
+        if (fFcw & X86_FCW_IM)
+            *pr80Result = g_r80Indefinite;
+        else
+        {
+            *pr80Result = *pr80Val1Org;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    return fFsw;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fdiv_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                  PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT);
+
+    /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */
+    if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2))
+    {
+        if (fFcw & X86_FCW_IM)
+            pFpuRes->r80Result = g_r80Indefinite;
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    /* SoftFloat does not check for denormals and certainly not report them to us. NaNs & /0 trumps denormals. */
+    else if (   (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2) && !RTFLOAT80U_IS_ZERO(pr80Val2))
+             || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1)) )
+    {
+        if (fFcw & X86_FCW_DM)
+        {
+            PCRTFLOAT80U const pr80Val1Org = pr80Val1;
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized);
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized);
+            fFsw = iemAImpl_fdiv_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org);
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_DE;
+    }
+    /* SoftFloat can handle the rest: */
+    else
+        fFsw = iemAImpl_fdiv_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1);
+
+    pFpuRes->FSW = fFsw;
+}
+
+
+EMIT_R80_BY_R64(iemAImpl_fdiv_r80_by_r64,  iemAImpl_fdiv_r80_by_r80, 0)
+EMIT_R80_BY_R32(iemAImpl_fdiv_r80_by_r32,  iemAImpl_fdiv_r80_by_r80, 0)
+EMIT_R80_BY_I32(iemAImpl_fidiv_r80_by_i32, iemAImpl_fdiv_r80_by_r80)
+EMIT_R80_BY_I16(iemAImpl_fidiv_r80_by_i16, iemAImpl_fdiv_r80_by_r80)
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fdivr_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                   PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT);
+
+    /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */
+    if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2))
+    {
+        if (fFcw & X86_FCW_IM)
+            pFpuRes->r80Result = g_r80Indefinite;
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    /* SoftFloat does not check for denormals and certainly not report them to us. NaNs & /0 trumps denormals. */
+    else if (   (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2))
+             || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1) && !RTFLOAT80U_IS_ZERO(pr80Val1)) )
+    {
+        if (fFcw & X86_FCW_DM)
+        {
+            PCRTFLOAT80U const pr80Val1Org = pr80Val1;
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized);
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized);
+            fFsw = iemAImpl_fdiv_f80_r80_worker(pr80Val2, pr80Val1, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org);
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_DE;
+    }
+    /* SoftFloat can handle the rest: */
+    else
+        fFsw = iemAImpl_fdiv_f80_r80_worker(pr80Val2, pr80Val1, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1);
+
+    pFpuRes->FSW = fFsw;
+}
+
+
+EMIT_R80_BY_R64(iemAImpl_fdivr_r80_by_r64,  iemAImpl_fdivr_r80_by_r80, RTFLOAT80U_IS_ZERO(pr80Val1))
+EMIT_R80_BY_R32(iemAImpl_fdivr_r80_by_r32,  iemAImpl_fdivr_r80_by_r80, RTFLOAT80U_IS_ZERO(pr80Val1))
+EMIT_R80_BY_I32(iemAImpl_fidivr_r80_by_i32, iemAImpl_fdivr_r80_by_r80)
+EMIT_R80_BY_I16(iemAImpl_fidivr_r80_by_i16, iemAImpl_fdivr_r80_by_r80)
+
+
+/** Worker for iemAImpl_fprem_r80_by_r80 & iemAImpl_fprem1_r80_by_r80. */
+static uint16_t iemAImpl_fprem_fprem1_r80_by_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result,
+                                                        uint16_t fFcw, uint16_t fFsw, PCRTFLOAT80U pr80Val1Org, bool fLegacyInstr)
+{
+    if (!RTFLOAT80U_IS_ZERO(pr80Val2) || RTFLOAT80U_IS_NAN(pr80Val1) || RTFLOAT80U_IS_INF(pr80Val1))
+    {
+        softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw);
+        uint16_t          fCxFlags  = 0;
+        extFloat80_t r80XResult = extF80_partialRem(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2),
+                                                    fLegacyInstr ? softfloat_round_minMag : softfloat_round_near_even,
+                                                    &fCxFlags, &SoftState);
+        Assert(!(fCxFlags & ~X86_FSW_C_MASK));
+        fFsw = iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, pr80Result, fFcw, fFsw, pr80Val1Org);
+        if (   !(fFsw & X86_FSW_IE)
+            && !RTFLOAT80U_IS_NAN(pr80Result)
+            && !RTFLOAT80U_IS_INDEFINITE(pr80Result))
+        {
+            fFsw &= ~(uint16_t)X86_FSW_C_MASK;
+            fFsw |= fCxFlags & X86_FSW_C_MASK;
+        }
+        return fFsw;
+    }
+
+    /* Invalid operand */
+    if (fFcw & X86_FCW_IM)
+        *pr80Result = g_r80Indefinite;
+    else
+    {
+        *pr80Result = *pr80Val1Org;
+        fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+    return fFsw | X86_FSW_IE;
+}
+
+
+static void iemAImpl_fprem_fprem1_r80_by_r80(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                             PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, bool fLegacyInstr)
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 /*| X86_FSW_C2*/ | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT);
+
+    /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals.
+       In addition, we'd like to handle zero ST(1) now as SoftFloat returns Inf instead
+       of Indefinite.  (Note! There is no #Z like the footnotes to tables 3-31 and 3-32
+       for the FPREM1 & FPREM1 instructions in the intel reference manual claims!) */
+    if (   RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2)
+        || (RTFLOAT80U_IS_ZERO(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1) && !RTFLOAT80U_IS_INDEFINITE(pr80Val1)))
+    {
+        if (fFcw & X86_FCW_IM)
+            pFpuRes->r80Result = g_r80Indefinite;
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    /* SoftFloat does not check for denormals and certainly not report them to us. NaNs & /0 trumps denormals. */
+    else if (   (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2) && !RTFLOAT80U_IS_ZERO(pr80Val2))
+             || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1) && !RTFLOAT80U_IS_INF(pr80Val1)) )
+    {
+        if (fFcw & X86_FCW_DM)
+        {
+            PCRTFLOAT80U const pr80Val1Org = pr80Val1;
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized);
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized);
+            fFsw = iemAImpl_fprem_fprem1_r80_by_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw,
+                                                           pr80Val1Org, fLegacyInstr);
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_DE;
+    }
+    /* SoftFloat can handle the rest: */
+    else
+        fFsw = iemAImpl_fprem_fprem1_r80_by_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw,
+                                                       pr80Val1, fLegacyInstr);
+
+    pFpuRes->FSW = fFsw;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fprem_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                   PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    iemAImpl_fprem_fprem1_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2, true /*fLegacyInstr*/);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fprem1_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                    PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    iemAImpl_fprem_fprem1_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2, false /*fLegacyInstr*/);
+}
+
+
+/*********************************************************************************************************************************
+*   x87 FPU Multiplication Operations                                                                                            *
+*********************************************************************************************************************************/
+
+/** Worker for iemAImpl_fmul_r80_by_r80. */
+static uint16_t iemAImpl_fmul_f80_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result,
+                                             uint16_t fFcw, uint16_t fFsw, PCRTFLOAT80U pr80Val1Org)
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw);
+    extFloat80_t r80XResult = extF80_mul(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), &SoftState);
+    return iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, pr80Result, fFcw, fFsw, pr80Val1Org);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fmul_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                  PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT);
+
+    /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */
+    if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2))
+    {
+        if (fFcw & X86_FCW_IM)
+            pFpuRes->r80Result = g_r80Indefinite;
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    /* SoftFloat does not check for denormals and certainly not report them to us. NaNs trumps denormals. */
+    else if (   (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2))
+             || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1)) )
+    {
+        if (fFcw & X86_FCW_DM)
+        {
+            PCRTFLOAT80U const pr80Val1Org = pr80Val1;
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized);
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized);
+            fFsw = iemAImpl_fmul_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org);
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_DE;
+    }
+    /* SoftFloat can handle the rest: */
+    else
+        fFsw = iemAImpl_fmul_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1);
+
+    pFpuRes->FSW = fFsw;
+}
+
+
+EMIT_R80_BY_R64(iemAImpl_fmul_r80_by_r64,  iemAImpl_fmul_r80_by_r80, 0)
+EMIT_R80_BY_R32(iemAImpl_fmul_r80_by_r32,  iemAImpl_fmul_r80_by_r80, 0)
+EMIT_R80_BY_I32(iemAImpl_fimul_r80_by_i32, iemAImpl_fmul_r80_by_r80)
+EMIT_R80_BY_I16(iemAImpl_fimul_r80_by_i16, iemAImpl_fmul_r80_by_r80)
+
+
+/*********************************************************************************************************************************
+*   x87 FPU Addition                                                                                                             *
+*********************************************************************************************************************************/
+
+/** Worker for iemAImpl_fadd_r80_by_r80. */
+static uint16_t iemAImpl_fadd_f80_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result,
+                                             uint16_t fFcw, uint16_t fFsw, PCRTFLOAT80U pr80Val1Org)
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw);
+    extFloat80_t r80XResult = extF80_add(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), &SoftState);
+    return iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, pr80Result, fFcw, fFsw, pr80Val1Org);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fadd_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                  PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT);
+
+    /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */
+    if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2))
+    {
+        if (fFcw & X86_FCW_IM)
+            pFpuRes->r80Result = g_r80Indefinite;
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    /* SoftFloat does not check for denormals and certainly not report them to us. NaNs trumps denormals. */
+    else if (   (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2))
+             || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1)) )
+    {
+        if (fFcw & X86_FCW_DM)
+        {
+            PCRTFLOAT80U const pr80Val1Org = pr80Val1;
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized);
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized);
+            fFsw = iemAImpl_fadd_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org);
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_DE;
+    }
+    /* SoftFloat can handle the rest: */
+    else
+        fFsw = iemAImpl_fadd_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1);
+
+    pFpuRes->FSW = fFsw;
+}
+
+
+EMIT_R80_BY_R64(iemAImpl_fadd_r80_by_r64,  iemAImpl_fadd_r80_by_r80, 0)
+EMIT_R80_BY_R32(iemAImpl_fadd_r80_by_r32,  iemAImpl_fadd_r80_by_r80, 0)
+EMIT_R80_BY_I32(iemAImpl_fiadd_r80_by_i32, iemAImpl_fadd_r80_by_r80)
+EMIT_R80_BY_I16(iemAImpl_fiadd_r80_by_i16, iemAImpl_fadd_r80_by_r80)
+
+
+/*********************************************************************************************************************************
+*   x87 FPU Subtraction                                                                                                          *
+*********************************************************************************************************************************/
+
+/** Worker for iemAImpl_fsub_r80_by_r80 and iemAImpl_fsubr_r80_by_r80. */
+static uint16_t iemAImpl_fsub_f80_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result,
+                                             uint16_t fFcw, uint16_t fFsw, PCRTFLOAT80U pr80Val1Org)
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw);
+    extFloat80_t r80XResult = extF80_sub(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), &SoftState);
+    return iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, pr80Result, fFcw, fFsw, pr80Val1Org);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fsub_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                  PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT);
+
+    /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */
+    if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2))
+    {
+        if (fFcw & X86_FCW_IM)
+            pFpuRes->r80Result = g_r80Indefinite;
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    /* SoftFloat does not check for denormals and certainly not report them to us. NaNs trumps denormals. */
+    else if (   (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2))
+             || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1)) )
+    {
+        if (fFcw & X86_FCW_DM)
+        {
+            PCRTFLOAT80U const pr80Val1Org = pr80Val1;
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized);
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized);
+            fFsw = iemAImpl_fsub_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org);
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_DE;
+    }
+    /* SoftFloat can handle the rest: */
+    else
+        fFsw = iemAImpl_fsub_f80_r80_worker(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1);
+
+    pFpuRes->FSW = fFsw;
+}
+
+
+EMIT_R80_BY_R64(iemAImpl_fsub_r80_by_r64,  iemAImpl_fsub_r80_by_r80, 0)
+EMIT_R80_BY_R32(iemAImpl_fsub_r80_by_r32,  iemAImpl_fsub_r80_by_r80, 0)
+EMIT_R80_BY_I32(iemAImpl_fisub_r80_by_i32, iemAImpl_fsub_r80_by_r80)
+EMIT_R80_BY_I16(iemAImpl_fisub_r80_by_i16, iemAImpl_fsub_r80_by_r80)
+
+
+/* Same as iemAImpl_fsub_r80_by_r80, but with input operands switched.  */
+IEM_DECL_IMPL_DEF(void, iemAImpl_fsubr_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                   PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT);
+
+    /* SoftFloat does not check for Pseudo-Infinity, Pseudo-Nan and Unnormals. */
+    if (RTFLOAT80U_IS_387_INVALID(pr80Val1) || RTFLOAT80U_IS_387_INVALID(pr80Val2))
+    {
+        if (fFcw & X86_FCW_IM)
+            pFpuRes->r80Result = g_r80Indefinite;
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    /* SoftFloat does not check for denormals and certainly not report them to us. NaNs trumps denormals. */
+    else if (   (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val1) && !RTFLOAT80U_IS_NAN(pr80Val2))
+             || (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val2) && !RTFLOAT80U_IS_NAN(pr80Val1)) )
+    {
+        if (fFcw & X86_FCW_DM)
+        {
+            PCRTFLOAT80U const pr80Val1Org = pr80Val1;
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val1, r80Val1Normalized);
+            IEM_NORMALIZE_PSEUDO_DENORMAL(pr80Val2, r80Val2Normalized);
+            fFsw = iemAImpl_fsub_f80_r80_worker(pr80Val2, pr80Val1, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1Org);
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val1;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_DE;
+    }
+    /* SoftFloat can handle the rest: */
+    else
+        fFsw = iemAImpl_fsub_f80_r80_worker(pr80Val2, pr80Val1, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1);
+
+    pFpuRes->FSW = fFsw;
+}
+
+
+EMIT_R80_BY_R64(iemAImpl_fsubr_r80_by_r64,  iemAImpl_fsubr_r80_by_r80, 0)
+EMIT_R80_BY_R32(iemAImpl_fsubr_r80_by_r32,  iemAImpl_fsubr_r80_by_r80, 0)
+EMIT_R80_BY_I32(iemAImpl_fisubr_r80_by_i32, iemAImpl_fsubr_r80_by_r80)
+EMIT_R80_BY_I16(iemAImpl_fisubr_r80_by_i16, iemAImpl_fsubr_r80_by_r80)
+
+
+/*********************************************************************************************************************************
+*   x87 FPU Trigometric Operations                                                                                               *
+*********************************************************************************************************************************/
+static uint16_t iemAImpl_fpatan_r80_by_r80_normal(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PIEMFPURESULT pFpuRes, uint16_t fFcw, uint16_t fFsw)
+{
+    softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS();
+    extFloat80_t y = iemFpuSoftF80FromIprt(pr80Val1);
+    extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val2);
+    extFloat80_t v;
+    (void)fFcw;
+
+    v = extF80_atan2(y, x, &SoftState);
+
+    iemFpuSoftF80ToIprt(&pFpuRes->r80Result, v);
+    return fFsw;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fpatan_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                    PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3);
+
+    if (RTFLOAT80U_IS_NORMAL(pr80Val1) && RTFLOAT80U_IS_NORMAL(pr80Val2))
+    {
+        fFsw = iemAImpl_fpatan_r80_by_r80_normal(pr80Val1, pr80Val2, pFpuRes, fFcw, fFsw);
+
+        fFsw |= X86_FSW_PE | (7 << X86_FSW_TOP_SHIFT);
+        if (!(fFcw & X86_FCW_PM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+    else
+    {
+        fFsw |= X86_FSW_IE;
+        if (!(fFcw & X86_FCW_IM))
+        {
+            pFpuRes->r80Result = *pr80Val2;
+            fFsw |= X86_FSW_ES | X86_FSW_B | (6 << X86_FSW_TOP_SHIFT);
+        }
+        else
+        {
+            pFpuRes->r80Result = g_r80Indefinite;
+            fFsw |= (7 << X86_FSW_TOP_SHIFT);
+        }
+    }
+
+    pFpuRes->FSW = fFsw;
+}
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fpatan_r80_by_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                          PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    iemAImpl_fpatan_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fpatan_r80_by_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                        PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    iemAImpl_fpatan_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2);
+}
+
+
+#if defined(IEM_WITHOUT_ASSEMBLY)
+static uint16_t iemAImpl_fptan_r80_r80_normal(PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val, uint16_t fFcw, uint16_t fFsw)
+{
+    softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS();
+    extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val);
+    extFloat80_t v;
+    (void)fFcw;
+
+    v = extF80_tan(x, &SoftState);
+
+    iemFpuSoftF80ToIprt(&pFpuResTwo->r80Result1, v);
+    return fFsw;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fptan_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | /*X86_FSW_C2 |*/ X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT);
+
+    if (RTFLOAT80U_IS_ZERO(pr80Val))
+    {
+        pFpuResTwo->r80Result1 = *pr80Val;
+        pFpuResTwo->r80Result2 = g_ar80One[0];
+    }
+    else if (RTFLOAT80U_IS_NORMAL(pr80Val))
+    {
+        if (pr80Val->s.uExponent >= RTFLOAT80U_EXP_BIAS + 63)
+        {
+            fFsw |= X86_FSW_C2 | (7 << X86_FSW_TOP_SHIFT);
+            pFpuResTwo->r80Result1 = *pr80Val;
+        }
+        else
+        {
+            if (pr80Val->s.uExponent <= RTFLOAT80U_EXP_BIAS - 63)
+            {
+                pFpuResTwo->r80Result1 = *pr80Val;
+            }
+            else
+            {
+                fFsw = iemAImpl_fptan_r80_r80_normal(pFpuResTwo, pr80Val, fFcw, fFsw);
+            }
+
+            pFpuResTwo->r80Result2 = g_ar80One[0];
+
+            fFsw |= X86_FSW_PE;
+            if (!(fFcw & X86_FCW_PM))
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else
+    {
+        fFsw |= X86_FSW_IE;
+        if (!(fFcw & X86_FCW_IM))
+            fFsw |= X86_FSW_ES | X86_FSW_B | (7 << X86_FSW_TOP_SHIFT);
+    }
+
+    pFpuResTwo->FSW = fFsw;
+}
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fptan_r80_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
+{
+    iemAImpl_fptan_r80_r80(pFpuState, pFpuResTwo, pr80Val);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fptan_r80_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
+{
+    iemAImpl_fptan_r80_r80(pFpuState, pFpuResTwo, pr80Val);
+}
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+static uint16_t iemAImpl_fsin_r80_normal(PCRTFLOAT80U pr80Val, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw)
+{
+    softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS();
+    extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val);
+    extFloat80_t v;
+    (void)fFcw;
+
+    v = extF80_sin(x, &SoftState);
+
+    iemFpuSoftF80ToIprt(pr80Result, v);
+
+    return fFsw;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fsin_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | /*X86_FSW_C2 |*/ X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT);
+
+    if (RTFLOAT80U_IS_ZERO(pr80Val))
+    {
+        pFpuRes->r80Result = *pr80Val;
+    }
+    else if (RTFLOAT80U_IS_NORMAL(pr80Val))
+    {
+        if (pr80Val->s.uExponent >= RTFLOAT80U_EXP_BIAS + 63)
+        {
+            fFsw |= X86_FSW_C2;
+            pFpuRes->r80Result = *pr80Val;
+        }
+        else
+        {
+            if (pr80Val->s.uExponent <= RTFLOAT80U_EXP_BIAS - 63)
+            {
+                pFpuRes->r80Result = *pr80Val;
+            }
+            else
+            {
+                fFsw = iemAImpl_fsin_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw);
+            }
+            fFsw |= X86_FSW_PE;
+            if (!(fFcw & X86_FCW_PM))
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else if (RTFLOAT80U_IS_INF(pr80Val))
+    {
+        fFsw |= X86_FSW_IE;
+        if (!(fFcw & X86_FCW_IM))
+        {
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+            pFpuRes->r80Result = *pr80Val;
+        }
+        else
+        {
+            pFpuRes->r80Result = g_r80Indefinite;
+        }
+    }
+    else if (RTFLOAT80U_IS_DENORMAL(pr80Val))
+    {
+        fFsw |= X86_FSW_DE;
+
+        if (fFcw & X86_FCW_DM)
+        {
+            if (fFcw & X86_FCW_UM)
+            {
+                pFpuRes->r80Result = *pr80Val;
+            }
+            else
+            {
+                /* Underflow signalling as described at 7.4 section of 1985 IEEE 754*/
+                uint64_t uMantissa = pr80Val->s.uMantissa;
+                uint32_t uExponent = ASMBitLastSetU64(uMantissa);
+
+                uExponent = 64 - uExponent;
+                uMantissa <<= uExponent;
+                uExponent = RTFLOAT128U_EXP_BIAS_ADJUST - uExponent + 1;
+
+                pFpuRes->r80Result.s.fSign = pr80Val->s.fSign;
+                pFpuRes->r80Result.s.uMantissa = uMantissa;
+                pFpuRes->r80Result.s.uExponent = uExponent;
+            }
+
+            fFsw |= X86_FSW_UE | X86_FSW_PE;
+
+            if ((fFcw & X86_FCW_UM) && (fFcw & X86_FCW_PM))
+            {
+                /* All the exceptions are masked. */
+            }
+            else
+            {
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+            }
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val;
+
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val))
+    {
+        pFpuRes->r80Result = *pr80Val;
+        fFsw |= X86_FSW_DE;
+
+        if (fFcw & X86_FCW_DM)
+        {
+            if (fFcw & X86_FCW_PM)
+            {
+                fFsw |= X86_FSW_PE;
+            }
+            else
+            {
+                fFsw |= X86_FSW_ES | X86_FSW_B | X86_FSW_PE;
+            }
+
+            pFpuRes->r80Result.sj64.uExponent = 1;
+        }
+        else
+        {
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    } else if (   RTFLOAT80U_IS_QUIET_NAN(pr80Val)
+             || RTFLOAT80U_IS_INDEFINITE(pr80Val))
+    {
+        pFpuRes->r80Result = *pr80Val;
+    } else {
+        if (   (   RTFLOAT80U_IS_UNNORMAL(pr80Val)
+                || RTFLOAT80U_IS_PSEUDO_NAN(pr80Val))
+            && (fFcw & X86_FCW_IM))
+            pFpuRes->r80Result = g_r80Indefinite;
+        else
+        {
+            pFpuRes->r80Result = *pr80Val;
+            if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val) && (fFcw & X86_FCW_IM))
+                pFpuRes->r80Result.s.uMantissa |= RT_BIT_64(62); /* make it quiet */
+        }
+
+        fFsw |= X86_FSW_IE;
+        if (!(fFcw & X86_FCW_IM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+
+    pFpuRes->FSW = fFsw;
+}
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fsin_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    iemAImpl_fsin_r80(pFpuState, pFpuRes, pr80Val);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fsin_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    iemAImpl_fsin_r80(pFpuState, pFpuRes, pr80Val);
+}
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+static uint16_t iemAImpl_fcos_r80_normal(PCRTFLOAT80U pr80Val, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw)
+{
+    softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS();
+    extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val);
+    extFloat80_t v;
+    (void)fFcw;
+
+    v = extF80_cos(x, &SoftState);
+
+    iemFpuSoftF80ToIprt(pr80Result, v);
+
+    return fFsw;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fcos_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | /*X86_FSW_C2 |*/ X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT);
+
+    if (RTFLOAT80U_IS_ZERO(pr80Val))
+    {
+        pFpuRes->r80Result = g_ar80One[0];
+    }
+    else if (RTFLOAT80U_IS_NORMAL(pr80Val))
+    {
+        if (pr80Val->s.uExponent >= RTFLOAT80U_EXP_BIAS + 63)
+        {
+            fFsw |= X86_FSW_C2;
+            pFpuRes->r80Result = *pr80Val;
+        }
+        else
+        {
+            if (pr80Val->s.uExponent <= RTFLOAT80U_EXP_BIAS - 63)
+            {
+                pFpuRes->r80Result = g_ar80One[0];
+
+            }
+            else
+            {
+                fFsw = iemAImpl_fcos_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw);
+                fFsw |= X86_FSW_C1; // TBD: If the inexact result was rounded up (C1 is set) or “not rounded up” (C1 is cleared).
+            }
+            fFsw |= X86_FSW_PE;
+            if (!(fFcw & X86_FCW_PM))
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else if (RTFLOAT80U_IS_INF(pr80Val))
+    {
+        fFsw |= X86_FSW_IE;
+        if (!(fFcw & X86_FCW_IM))
+        {
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+            pFpuRes->r80Result = *pr80Val;
+        }
+        else
+        {
+            pFpuRes->r80Result = g_r80Indefinite;
+        }
+    }
+    else if (RTFLOAT80U_IS_DENORMAL(pr80Val) || RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val))
+    {
+        fFsw |= X86_FSW_DE;
+
+        if (fFcw & X86_FCW_DM)
+        {
+            pFpuRes->r80Result = g_ar80One[0];
+
+            if (fFcw & X86_FCW_PM)
+            {
+                fFsw |= X86_FSW_PE;
+            }
+            else
+            {
+                fFsw |= X86_FSW_PE | X86_FSW_ES | X86_FSW_B;
+            }
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    } else if (   RTFLOAT80U_IS_QUIET_NAN(pr80Val)
+             || RTFLOAT80U_IS_INDEFINITE(pr80Val))
+    {
+        pFpuRes->r80Result = *pr80Val;
+    } else {
+        if (   (   RTFLOAT80U_IS_UNNORMAL(pr80Val)
+                || RTFLOAT80U_IS_PSEUDO_NAN(pr80Val))
+            && (fFcw & X86_FCW_IM))
+            pFpuRes->r80Result = g_r80Indefinite;
+        else
+        {
+            pFpuRes->r80Result = *pr80Val;
+            if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val) && (fFcw & X86_FCW_IM))
+                pFpuRes->r80Result.s.uMantissa |= RT_BIT_64(62); /* make it quiet */
+        }
+
+        fFsw |= X86_FSW_IE;
+        if (!(fFcw & X86_FCW_IM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+
+    pFpuRes->FSW = fFsw;
+}
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fcos_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    iemAImpl_fcos_r80(pFpuState, pFpuRes, pr80Val);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fcos_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    iemAImpl_fcos_r80(pFpuState, pFpuRes, pr80Val);
+}
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+static uint16_t iemAImpl_fsincos_r80_r80_normal(PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val, uint16_t fFcw, uint16_t fFsw)
+{
+    softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS();
+    extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val);
+    extFloat80_t r80Sin, r80Cos;
+    (void)fFcw;
+
+    extF80_sincos(x, &r80Sin, &r80Cos, &SoftState);
+
+    iemFpuSoftF80ToIprt(&pFpuResTwo->r80Result1, r80Sin);
+    iemFpuSoftF80ToIprt(&pFpuResTwo->r80Result2, r80Cos);
+
+    return fFsw;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fsincos_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | /*X86_FSW_C2 |*/ X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT);
+
+    if (RTFLOAT80U_IS_ZERO(pr80Val))
+    {
+        pFpuResTwo->r80Result1 = *pr80Val;
+        pFpuResTwo->r80Result2 = g_ar80One[0];
+        fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT);
+    }
+    else if (RTFLOAT80U_IS_NORMAL(pr80Val))
+    {
+        if (pr80Val->s.uExponent >= RTFLOAT80U_EXP_BIAS + 63)
+        {
+            fFsw |= X86_FSW_C2;
+
+            if (fFcw & X86_FCW_IM)
+            {
+                pFpuResTwo->r80Result1 = g_r80Indefinite;
+            }
+            else
+            {
+                pFpuResTwo->r80Result1 = g_ar80Zero[0];
+            }
+
+            pFpuResTwo->r80Result2 = *pr80Val;
+        }
+        else
+        {
+            fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT);
+
+            if (pr80Val->s.uExponent <= RTFLOAT80U_EXP_BIAS - 63)
+            {
+                pFpuResTwo->r80Result1 = *pr80Val;
+                pFpuResTwo->r80Result2 = g_ar80One[0];
+            }
+            else
+            {
+                fFsw = iemAImpl_fsincos_r80_r80_normal(pFpuResTwo, pr80Val, fFcw, fFsw);
+                fFsw |= X86_FSW_C1; // TBD: If the inexact result was rounded up (C1 is set) or “not rounded up” (C1 is cleared).
+            }
+            fFsw |= X86_FSW_PE;
+            if (!(fFcw & X86_FCW_PM))
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val))
+    {
+        fFsw |= X86_FSW_DE;
+
+        if (fFcw & X86_FCW_DM)
+        {
+            pFpuResTwo->r80Result1 = *pr80Val;
+            pFpuResTwo->r80Result2 = g_ar80One[0];
+            fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT);
+
+            if (fFcw & X86_FCW_PM)
+            {
+                fFsw |= X86_FSW_PE;
+            }
+            else
+            {
+                fFsw |= X86_FSW_PE | X86_FSW_ES | X86_FSW_B;
+            }
+
+            pFpuResTwo->r80Result1.sj64.uExponent = 1;
+        }
+        else
+        {
+            pFpuResTwo->r80Result1 = g_ar80Zero[0];
+            pFpuResTwo->r80Result2 = *pr80Val;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else if (RTFLOAT80U_IS_DENORMAL(pr80Val))
+    {
+        fFsw |= X86_FSW_DE;
+
+        if (fFcw & X86_FCW_DM)
+        {
+            pFpuResTwo->r80Result2 = g_ar80One[0];
+
+            if (fFcw & X86_FCW_UM)
+            {
+                pFpuResTwo->r80Result1 = *pr80Val;
+            }
+            else
+            {
+                /* Underflow signalling as described at 7.4 section of 1985 IEEE 754*/
+                uint64_t uMantissa = pr80Val->s.uMantissa;
+                uint32_t uExponent = ASMBitLastSetU64(uMantissa);
+
+                uExponent = 64 - uExponent;
+                uMantissa <<= uExponent;
+                uExponent = RTFLOAT128U_EXP_BIAS_ADJUST - uExponent + 1;
+
+                pFpuResTwo->r80Result1.s.fSign = pr80Val->s.fSign;
+                pFpuResTwo->r80Result1.s.uMantissa = uMantissa;
+                pFpuResTwo->r80Result1.s.uExponent = uExponent;
+            }
+
+            fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT);
+            fFsw |= X86_FSW_UE | X86_FSW_PE;
+
+            if ((fFcw & X86_FCW_UM) && (fFcw & X86_FCW_PM))
+            {
+                /* All the exceptions are masked. */
+            }
+            else
+            {
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+            }
+        }
+        else
+        {
+            pFpuResTwo->r80Result1 = g_ar80Zero[0];
+            pFpuResTwo->r80Result2 = *pr80Val;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else if (RTFLOAT80U_IS_QUIET_NAN(pr80Val) || RTFLOAT80U_IS_INDEFINITE(pr80Val))
+    {
+        pFpuResTwo->r80Result1 = *pr80Val;
+        pFpuResTwo->r80Result2 = *pr80Val;
+        fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT);
+    }
+    else if (RTFLOAT80U_IS_UNNORMAL(pr80Val) || RTFLOAT80U_IS_PSEUDO_NAN(pr80Val))
+    {
+        if (fFcw & X86_FCW_IM)
+        {
+            pFpuResTwo->r80Result1 = g_r80Indefinite;
+            pFpuResTwo->r80Result2 = g_r80Indefinite;
+            fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT);
+        }
+        else
+        {
+            pFpuResTwo->r80Result1 = g_ar80Zero[0];
+            pFpuResTwo->r80Result2 = *pr80Val;
+        }
+
+        fFsw |= X86_FSW_IE;
+        if (!(fFcw & X86_FCW_IM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+    else if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val))
+    {
+        pFpuResTwo->r80Result1 = *pr80Val;
+        pFpuResTwo->r80Result2 = *pr80Val;
+
+        if (fFcw & X86_FCW_IM)
+        {
+            pFpuResTwo->r80Result1.s.uMantissa |= RT_BIT_64(62); /* make it quiet */
+            pFpuResTwo->r80Result2.s.uMantissa |= RT_BIT_64(62);
+            fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT);
+        }
+        else
+        {
+            pFpuResTwo->r80Result1 = g_ar80Zero[0];
+            pFpuResTwo->r80Result2 = *pr80Val;
+        }
+
+        fFsw |= X86_FSW_IE;
+        if (!(fFcw & X86_FCW_IM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+    else if (RTFLOAT80U_IS_INF(pr80Val))
+    {
+        if (fFcw & X86_FCW_IM)
+        {
+            pFpuResTwo->r80Result1 = g_r80Indefinite;
+            pFpuResTwo->r80Result2 = g_r80Indefinite;
+            fFsw &= ~X86_FSW_TOP_MASK | (6 << X86_FSW_TOP_SHIFT);
+        }
+        else
+        {
+            pFpuResTwo->r80Result1 = g_ar80Zero[0];
+            pFpuResTwo->r80Result2 = *pr80Val;
+        }
+
+        fFsw |= X86_FSW_IE;
+        if (!(fFcw & X86_FCW_IM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+
+    pFpuResTwo->FSW = fFsw;
+}
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fsincos_r80_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
+{
+    iemAImpl_fsincos_r80_r80(pFpuState, pFpuResTwo, pr80Val);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fsincos_r80_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
+{
+    iemAImpl_fsincos_r80_r80(pFpuState, pFpuResTwo, pr80Val);
+}
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+
+/*********************************************************************************************************************************
+*   x87 FPU Compare and Testing Operations                                                                                       *
+*********************************************************************************************************************************/
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_ftst_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val))
+{
+    uint16_t fFsw = (7 << X86_FSW_TOP_SHIFT);
+
+    if (RTFLOAT80U_IS_ZERO(pr80Val))
+        fFsw |= X86_FSW_C3;
+    else if (RTFLOAT80U_IS_NORMAL(pr80Val) || RTFLOAT80U_IS_INF(pr80Val))
+        fFsw |= pr80Val->s.fSign ? X86_FSW_C0 : 0;
+    else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val))
+    {
+        fFsw |= pr80Val->s.fSign ? X86_FSW_C0 | X86_FSW_DE : X86_FSW_DE;
+        if (!(pFpuState->FCW & X86_FCW_DM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+    else
+    {
+        fFsw |= X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3 | X86_FSW_IE;
+        if (!(pFpuState->FCW & X86_FCW_IM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+
+    *pu16Fsw = fFsw;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fxam_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val))
+{
+    RT_NOREF(pFpuState);
+    uint16_t fFsw = (7 << X86_FSW_TOP_SHIFT);
+
+    /* C1 = sign bit (always, even if empty Intel says). */
+    if (pr80Val->s.fSign)
+        fFsw |= X86_FSW_C1;
+
+    /* Classify the value in C0, C2, C3. */
+    if (!(pFpuState->FTW & RT_BIT_32(X86_FSW_TOP_GET(pFpuState->FSW))))
+        fFsw |= X86_FSW_C0 | X86_FSW_C3; /* empty */
+    else if (RTFLOAT80U_IS_NORMAL(pr80Val))
+        fFsw |= X86_FSW_C2;
+    else if (RTFLOAT80U_IS_ZERO(pr80Val))
+        fFsw |= X86_FSW_C3;
+    else if (RTFLOAT80U_IS_QUIET_OR_SIGNALLING_NAN(pr80Val))
+        fFsw |= X86_FSW_C0;
+    else if (RTFLOAT80U_IS_INF(pr80Val))
+        fFsw |= X86_FSW_C0 | X86_FSW_C2;
+    else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val))
+        fFsw |= X86_FSW_C2 | X86_FSW_C3;
+    /* whatever else: 0 */
+
+    *pu16Fsw = fFsw;
+}
+
+
+/**
+ * Worker for fcom, fucom, and friends.
+ */
+static uint16_t iemAImpl_fcom_r80_by_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2,
+                                                uint16_t fFcw, uint16_t fFsw, bool fIeOnAllNaNs)
+{
+    /*
+     * Unpack the values.
+     */
+    bool const fSign1      = pr80Val1->s.fSign;
+    int32_t    iExponent1  = pr80Val1->s.uExponent;
+    uint64_t   uMantissa1  = pr80Val1->s.uMantissa;
+
+    bool const fSign2      = pr80Val2->s.fSign;
+    int32_t    iExponent2  = pr80Val2->s.uExponent;
+    uint64_t   uMantissa2  = pr80Val2->s.uMantissa;
+
+    /*
+     * Check for invalid inputs.
+     */
+    if (   RTFLOAT80U_IS_387_INVALID_EX(uMantissa1, iExponent1)
+        || RTFLOAT80U_IS_387_INVALID_EX(uMantissa2, iExponent2))
+    {
+         if (!(fFcw & X86_FCW_IM))
+             fFsw |= X86_FSW_ES | X86_FSW_B;
+         return fFsw | X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3 | X86_FSW_IE;
+    }
+
+    /*
+     * Check for NaNs and indefinites, they are all unordered and trumps #DE.
+     */
+    if (   RTFLOAT80U_IS_INDEFINITE_OR_QUIET_OR_SIGNALLING_NAN_EX(uMantissa1, iExponent1)
+        || RTFLOAT80U_IS_INDEFINITE_OR_QUIET_OR_SIGNALLING_NAN_EX(uMantissa2, iExponent2))
+    {
+        if (   fIeOnAllNaNs
+            || RTFLOAT80U_IS_SIGNALLING_NAN_EX(uMantissa1, iExponent1)
+            || RTFLOAT80U_IS_SIGNALLING_NAN_EX(uMantissa2, iExponent2))
+        {
+            fFsw |= X86_FSW_IE;
+            if (!(fFcw & X86_FCW_IM))
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        return fFsw | X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3;
+    }
+
+    /*
+     * Normalize the values.
+     */
+    if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL_EX(uMantissa1, iExponent1))
+    {
+        if (RTFLOAT80U_IS_PSEUDO_DENORMAL_EX(uMantissa1, iExponent1))
+            iExponent1 = 1;
+        else
+        {
+            iExponent1 = 64 - ASMBitLastSetU64(uMantissa1);
+            uMantissa1 <<= iExponent1;
+            iExponent1 = 1 - iExponent1;
+        }
+        fFsw |= X86_FSW_DE;
+        if (!(fFcw & X86_FCW_DM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+
+    if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL_EX(uMantissa2, iExponent2))
+    {
+        if (RTFLOAT80U_IS_PSEUDO_DENORMAL_EX(uMantissa2, iExponent2))
+            iExponent2 = 1;
+        else
+        {
+            iExponent2 = 64 - ASMBitLastSetU64(uMantissa2);
+            uMantissa2 <<= iExponent2;
+            iExponent2 = 1 - iExponent2;
+        }
+        fFsw |= X86_FSW_DE;
+        if (!(fFcw & X86_FCW_DM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+
+    /*
+     * Test if equal (val1 == val2):
+     */
+    if (   uMantissa1 == uMantissa2
+        && iExponent1 == iExponent2
+        && (   fSign1 == fSign2
+            || (uMantissa1 == 0 && iExponent1 == 0) /* ignore sign for zero */ ) )
+        fFsw |= X86_FSW_C3;
+    /*
+     * Test if less than (val1 < val2):
+     */
+    else if (fSign1 && !fSign2)
+        fFsw |= X86_FSW_C0;
+    else if (fSign1 == fSign2)
+    {
+        /* Zeros are problematic, however at the most one can be zero here. */
+        if (RTFLOAT80U_IS_ZERO_EX(uMantissa1, iExponent1))
+            return !fSign1 ? fFsw | X86_FSW_C0 : fFsw;
+        if (RTFLOAT80U_IS_ZERO_EX(uMantissa2, iExponent2))
+            return fSign1  ? fFsw | X86_FSW_C0 : fFsw;
+
+        if (  fSign1
+            ^ (   iExponent1 < iExponent2
+               || (   iExponent1 == iExponent2
+                   && uMantissa1 < uMantissa2 ) ) )
+        fFsw |= X86_FSW_C0;
+    }
+    /* else: No flags set if greater. */
+
+    return fFsw;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pfFsw,
+                                                  PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    *pfFsw = iemAImpl_fcom_r80_by_r80_worker(pr80Val1, pr80Val2, pFpuState->FCW, 6 << X86_FSW_TOP_SHIFT, true /*fIeOnAllNaNs*/);
+}
+
+
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fucom_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pfFsw,
+                                                   PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    *pfFsw = iemAImpl_fcom_r80_by_r80_worker(pr80Val1, pr80Val2, pFpuState->FCW, 6 << X86_FSW_TOP_SHIFT, false /*fIeOnAllNaNs*/);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r64,(PCX86FXSTATE pFpuState, uint16_t *pfFsw,
+                                                  PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
+{
+    RTFLOAT80U r80Val2;
+    uint16_t   fFsw = iemAImplConvertR64ToR80(pr64Val2, &r80Val2);
+    Assert(!fFsw || fFsw == X86_FSW_DE);
+    *pfFsw = iemAImpl_fcom_r80_by_r80_worker(pr80Val1, &r80Val2, pFpuState->FCW, 7 << X86_FSW_TOP_SHIFT, true /*fIeOnAllNaNs*/);
+    if (fFsw != 0 && !(*pfFsw & X86_FSW_IE))
+    {
+        if (!(pFpuState->FCW & X86_FCW_DM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        *pfFsw |= fFsw;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r32,(PCX86FXSTATE pFpuState, uint16_t *pfFsw,
+                                                  PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
+{
+    RTFLOAT80U r80Val2;
+    uint16_t   fFsw = iemAImplConvertR32ToR80(pr32Val2, &r80Val2);
+    Assert(!fFsw || fFsw == X86_FSW_DE);
+    *pfFsw = iemAImpl_fcom_r80_by_r80_worker(pr80Val1, &r80Val2, pFpuState->FCW, 7 << X86_FSW_TOP_SHIFT, true /*fIeOnAllNaNs*/);
+    if (fFsw != 0 && !(*pfFsw & X86_FSW_IE))
+    {
+        if (!(pFpuState->FCW & X86_FCW_DM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        *pfFsw |= fFsw;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i32,(PCX86FXSTATE pFpuState, uint16_t *pfFsw,
+                                                   PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
+{
+    RTFLOAT80U r80Val2;
+    iemAImpl_fcom_r80_by_r80(pFpuState, pfFsw, pr80Val1, iemAImplConvertI32ToR80(*pi32Val2, &r80Val2));
+    *pfFsw = (*pfFsw & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i16,(PCX86FXSTATE pFpuState, uint16_t *pfFsw,
+                                                   PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
+{
+    RTFLOAT80U r80Val2;
+    iemAImpl_fcom_r80_by_r80(pFpuState, pfFsw, pr80Val1, iemAImplConvertI16ToR80(*pi16Val2, &r80Val2));
+    *pfFsw = (*pfFsw & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT);
+}
+
+
+/**
+ * Worker for fcomi & fucomi.
+ */
+static uint32_t iemAImpl_fcomi_r80_by_r80_worker(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2,
+                                                 uint16_t fFcw, uint16_t fFswIn, bool fIeOnAllNaNs, uint16_t *pfFsw)
+{
+    uint16_t fFsw    = iemAImpl_fcom_r80_by_r80_worker(pr80Val1, pr80Val2, fFcw, 6 << X86_FSW_TOP_SHIFT, fIeOnAllNaNs);
+    uint32_t fEflags = ((fFsw & X86_FSW_C3) >> (X86_FSW_C3_BIT - X86_EFL_ZF_BIT))
+                     | ((fFsw & X86_FSW_C2) >> (X86_FSW_C2_BIT - X86_EFL_PF_BIT))
+                     | ((fFsw & X86_FSW_C0) >> (X86_FSW_C0_BIT - X86_EFL_CF_BIT));
+
+    /* Note! C1 is not cleared as per docs! Everything is preserved. */
+    *pfFsw = (fFsw & ~X86_FSW_C_MASK) | (fFswIn & X86_FSW_C_MASK);
+    return fEflags | X86_EFL_IF | X86_EFL_RA1_MASK;
+}
+
+
+IEM_DECL_IMPL_DEF(uint32_t, iemAImpl_fcomi_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pfFsw,
+                                                       PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    return iemAImpl_fcomi_r80_by_r80_worker(pr80Val1, pr80Val2, pFpuState->FCW, pFpuState->FSW, true /*fIeOnAllNaNs*/, pfFsw);
+}
+
+
+IEM_DECL_IMPL_DEF(uint32_t, iemAImpl_fucomi_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pfFsw,
+                                                        PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    return iemAImpl_fcomi_r80_by_r80_worker(pr80Val1, pr80Val2, pFpuState->FCW, pFpuState->FSW, false /*fIeOnAllNaNs*/, pfFsw);
+}
+
+
+/*********************************************************************************************************************************
+*   x87 FPU Other Operations                                                                                                     *
+*********************************************************************************************************************************/
+
+/**
+ * Helper for iemAImpl_frndint_r80, called both on normal and denormal numbers.
+ */
+static uint16_t iemAImpl_frndint_r80_normal(PCRTFLOAT80U pr80Val, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw)
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw);
+    iemFpuSoftF80ToIprt(pr80Result, extF80_roundToInt(iemFpuSoftF80FromIprt(pr80Val), SoftState.roundingMode,
+                                                      true /*exact / generate #PE */, &SoftState));
+    return IEM_SOFTFLOAT_STATE_TO_FSW(fFsw, &SoftState, fFcw);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_frndint_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT);
+
+    if (RTFLOAT80U_IS_NORMAL(pr80Val))
+        fFsw = iemAImpl_frndint_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw);
+    else if (   RTFLOAT80U_IS_ZERO(pr80Val)
+             || RTFLOAT80U_IS_QUIET_NAN(pr80Val)
+             || RTFLOAT80U_IS_INDEFINITE(pr80Val)
+             || RTFLOAT80U_IS_INF(pr80Val))
+        pFpuRes->r80Result = *pr80Val;
+    else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val))
+    {
+        fFsw |= X86_FSW_DE;
+        if (fFcw & X86_FCW_DM)
+            fFsw = iemAImpl_frndint_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw);
+        else
+        {
+            pFpuRes->r80Result = *pr80Val;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else
+    {
+        if (fFcw & X86_FCW_IM)
+        {
+            if (!RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val))
+                pFpuRes->r80Result = g_r80Indefinite;
+            else
+            {
+                pFpuRes->r80Result = *pr80Val;
+                pFpuRes->r80Result.s.uMantissa |= RT_BIT_64(62); /* make it quiet */
+            }
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    pFpuRes->FSW = fFsw;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fscale_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                    PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    /* The SoftFloat worker function extF80_scale_extF80 is of our creation, so
+       it does everything we need it to do. */
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT);
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw);
+    extFloat80_t r80XResult = extF80_scale_extF80(iemFpuSoftF80FromIprt(pr80Val1), iemFpuSoftF80FromIprt(pr80Val2), &SoftState);
+    pFpuRes->FSW = iemFpuSoftStateAndF80ToFswAndIprtResult(&SoftState, r80XResult, &pFpuRes->r80Result, fFcw, fFsw, pr80Val1);
+}
+
+
+/**
+ * Helper for iemAImpl_fsqrt_r80, called both on normal and denormal numbers.
+ */
+static uint16_t iemAImpl_fsqrt_r80_normal(PCRTFLOAT80U pr80Val, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw)
+{
+    Assert(!pr80Val->s.fSign);
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_FCW(fFcw);
+    iemFpuSoftF80ToIprt(pr80Result, extF80_sqrt(iemFpuSoftF80FromIprt(pr80Val), &SoftState));
+    return IEM_SOFTFLOAT_STATE_TO_FSW(fFsw, &SoftState, fFcw);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fsqrt_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT);
+
+    if (RTFLOAT80U_IS_NORMAL(pr80Val) && !pr80Val->s.fSign)
+        fFsw = iemAImpl_fsqrt_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw);
+    else if (   RTFLOAT80U_IS_ZERO(pr80Val)
+             || RTFLOAT80U_IS_QUIET_NAN(pr80Val)
+             || RTFLOAT80U_IS_INDEFINITE(pr80Val)
+             || (RTFLOAT80U_IS_INF(pr80Val) && !pr80Val->s.fSign))
+        pFpuRes->r80Result = *pr80Val;
+    else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val) && !pr80Val->s.fSign) /* Negative denormals only generate #IE! */
+    {
+        fFsw |= X86_FSW_DE;
+        if (fFcw & X86_FCW_DM)
+            fFsw = iemAImpl_fsqrt_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw);
+        else
+        {
+            pFpuRes->r80Result = *pr80Val;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else
+    {
+        if (fFcw & X86_FCW_IM)
+        {
+            if (!RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val))
+                pFpuRes->r80Result = g_r80Indefinite;
+            else
+            {
+                pFpuRes->r80Result = *pr80Val;
+                pFpuRes->r80Result.s.uMantissa |= RT_BIT_64(62); /* make it quiet */
+            }
+        }
+        else
+        {
+            pFpuRes->r80Result = *pr80Val;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    pFpuRes->FSW = fFsw;
+}
+
+
+/**
+ * @code{.unparsed}
+ *          x          x * ln2
+ * f(x) = 2   - 1  =  e         - 1
+ *
+ * @endcode
+ *
+ * We can approximate e^x by a Taylor/Maclaurin series (see
+ * https://en.wikipedia.org/wiki/Taylor_series#Exponential_function):
+ * @code{.unparsed}
+ *        n        0     1     2     3     4
+ *  inf  x        x     x     x     x     x
+ *  SUM ----- =  --- + --- + --- + --- + --- + ...
+ *  n=0   n!      0!    1!    2!    3!    4!
+ *
+ *                              2     3     4
+ *                             x     x     x
+ *            =   1  +  x  +  --- + --- + --- + ...
+ *                             2!    3!    4!
+ * @endcode
+ *
+ * Given z = x * ln2, we get:
+ * @code{.unparsed}
+ *                      2     3     4           n
+ *    z                z     z     z           z
+ *  e   - 1  =  z  +  --- + --- + --- + ... + ---
+ *                     2!    3!    4!          n!
+ * @endcode
+ *
+ * Wanting to use Horner's method, we move one z outside and get:
+ * @code{.unparsed}
+ *                            2     3           (n-1)
+ *                     z     z     z           z
+ *       =  z ( 1  +  --- + --- + --- + ... + -------  )
+ *                     2!    3!    4!          n!
+ * @endcode
+ *
+ * The constants we need for using Horner's methods are 1 and 1 / n!.
+ *
+ * For very tiny x values, we can get away with f(x) = x * ln 2, because
+ * because we don't have the necessary precision to represent 1.0 + z/3 + ...
+ * and can approximate it to be 1.0.  For a visual demonstration of this
+ * check out https://www.desmos.com/calculator/vidcdxizd9 (for as long
+ * as it valid), plotting f(x) = 2^x - 1 and f(x) = x * ln2.
+ *
+ *
+ * As constant accuracy goes, figure 0.1 "80387 Block Diagram" in the "80387
+ * Data Sheet" (order 231920-002; Appendix E in 80387 PRM 231917-001; Military
+ * i387SX 271166-002), indicates that constants are 67-bit (constant rom block)
+ * and the internal mantissa size is 68-bit (mantissa adder & barrel shifter
+ * blocks).  (The one bit difference is probably an implicit one missing from
+ * the constant ROM.)  A paper on division and sqrt on the AMD-K7 by Stuart F.
+ * Oberman states that it internally used a 68 bit mantissa with a 18-bit
+ * exponent.
+ *
+ * However, even when sticking to 67 constants / 68 mantissas, I have not yet
+ * successfully reproduced the exact results from an Intel 10980XE, there is
+ * always a portition of rounding differences.  Not going to spend too much time
+ * on getting this 100% the same, at least not now.
+ *
+ * P.S. If someone are really curious about 8087 and its contstants:
+ * http://www.righto.com/2020/05/extracting-rom-constants-from-8087-math.html
+ *
+ *
+ * @param   pr80Val     The exponent value (x), less than 1.0, greater than
+ *                      -1.0 and not zero. This can be a normal, denormal
+ *                      or pseudo-denormal value.
+ * @param   pr80Result  Where to return the result.
+ * @param   fFcw        FPU control word.
+ * @param   fFsw        FPU status word.
+ */
+static uint16_t iemAImpl_f2xm1_r80_normal(PCRTFLOAT80U pr80Val, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw)
+{
+    /* As mentioned above, we can skip the expensive polynomial calculation
+       as it will be close enough to 1.0 that it makes no difference.
+
+       The cutoff point for intel 10980XE is exponents >= -69.  Intel
+       also seems to be using a 67-bit or 68-bit constant value, and we get
+       a smattering of rounding differences if we go for higher precision. */
+    if (pr80Val->s.uExponent <= RTFLOAT80U_EXP_BIAS - 69)
+    {
+        RTUINT256U u256;
+        RTUInt128MulByU64Ex(&u256, &g_u128Ln2MantissaIntel, pr80Val->s.uMantissa);
+        u256.QWords.qw0 |= 1; /* force #PE */
+        fFsw = iemFpuFloat80RoundAndComposeFrom192(pr80Result, pr80Val->s.fSign, &u256,
+                                                   !RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val) && !RTFLOAT80U_IS_DENORMAL(pr80Val)
+                                                   ? (int32_t)pr80Val->s.uExponent - RTFLOAT80U_EXP_BIAS
+                                                   : 1 - RTFLOAT80U_EXP_BIAS,
+                                                   fFcw, fFsw);
+    }
+    else
+    {
+#ifdef IEM_WITH_FLOAT128_FOR_FPU
+        /* This approach is not good enough for small values, we end up with zero. */
+        int const fOldRounding = iemFpuF128SetRounding(fFcw);
+        _Float128 rd128Val     = iemFpuF128FromFloat80(pr80Val, fFcw);
+        _Float128 rd128Result  = powf128(2.0L, rd128Val);
+        rd128Result -= 1.0L;
+        fFsw = iemFpuF128ToFloat80(pr80Result, rd128Result, fFcw, fFsw);
+        iemFpuF128RestoreRounding(fOldRounding);
+
+# else
+        softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS();
+        float128_t const x = iemFpuSoftF128FromFloat80(pr80Val);
+
+        /* As mentioned above, enforce 68-bit internal mantissa width to better
+           match the Intel 10980XE results. */
+        unsigned const cPrecision = 68;
+
+        /* first calculate z = x * ln2 */
+        float128_t z = iemFpuSoftF128Precision(f128_mul(x, iemFpuSoftF128PrecisionIprt(&g_r128Ln2, cPrecision), &SoftState),
+                                               cPrecision);
+
+        /* Then do the polynomial evaluation. */
+        float128_t r = iemFpuSoftF128HornerPoly(z, g_ar128F2xm1HornerConsts, RT_ELEMENTS(g_ar128F2xm1HornerConsts),
+                                                cPrecision, &SoftState);
+        r = f128_mul(z, r, &SoftState);
+
+        /* Output the result. */
+        fFsw = iemFpuSoftF128ToFloat80(pr80Result, r, fFcw, fFsw);
+# endif
+    }
+    return fFsw;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_f2xm1_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT);
+
+    if (RTFLOAT80U_IS_NORMAL(pr80Val))
+    {
+        if (pr80Val->s.uExponent < RTFLOAT80U_EXP_BIAS)
+            fFsw = iemAImpl_f2xm1_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw);
+        else
+        {
+            /* Special case:
+                   2^+1.0 - 1.0 = 1.0
+                   2^-1.0 - 1.0 = -0.5 */
+            if (   pr80Val->s.uExponent == RTFLOAT80U_EXP_BIAS
+                && pr80Val->s.uMantissa == RT_BIT_64(63))
+            {
+                pFpuRes->r80Result.s.uMantissa = RT_BIT_64(63);
+                pFpuRes->r80Result.s.uExponent = RTFLOAT80U_EXP_BIAS - pr80Val->s.fSign;
+                pFpuRes->r80Result.s.fSign     = pr80Val->s.fSign;
+            }
+            /* ST(0) > 1.0 || ST(0) < -1.0: undefined behavior */
+            /** @todo 287 is documented to only accept values 0 <= ST(0) <= 0.5. */
+            else
+                pFpuRes->r80Result = *pr80Val;
+            fFsw |= X86_FSW_PE;
+            if (!(fFcw & X86_FCW_PM))
+                fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else if (   RTFLOAT80U_IS_ZERO(pr80Val)
+             || RTFLOAT80U_IS_QUIET_NAN(pr80Val)
+             || RTFLOAT80U_IS_INDEFINITE(pr80Val))
+        pFpuRes->r80Result = *pr80Val;
+    else if (RTFLOAT80U_IS_INF(pr80Val))
+        pFpuRes->r80Result = pr80Val->s.fSign ? g_ar80One[1] : *pr80Val;
+    else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val))
+    {
+        fFsw |= X86_FSW_DE;
+        if (fFcw & X86_FCW_DM)
+            fFsw = iemAImpl_f2xm1_r80_normal(pr80Val, &pFpuRes->r80Result, fFcw, fFsw);
+        else
+        {
+            pFpuRes->r80Result = *pr80Val;
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+        }
+    }
+    else
+    {
+        if (   (   RTFLOAT80U_IS_UNNORMAL(pr80Val)
+                || RTFLOAT80U_IS_PSEUDO_NAN(pr80Val))
+            && (fFcw & X86_FCW_IM))
+            pFpuRes->r80Result = g_r80Indefinite;
+        else
+        {
+            pFpuRes->r80Result = *pr80Val;
+            if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val) && (fFcw & X86_FCW_IM))
+                pFpuRes->r80Result.s.uMantissa |= RT_BIT_64(62); /* make it quiet */
+        }
+        fFsw |= X86_FSW_IE;
+        if (!(fFcw & X86_FCW_IM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+    pFpuRes->FSW = fFsw;
+}
+
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_f2xm1_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    iemAImpl_f2xm1_r80(pFpuState, pFpuRes, pr80Val);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_f2xm1_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    iemAImpl_f2xm1_r80(pFpuState, pFpuRes, pr80Val);
+}
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fabs_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    pFpuRes->FSW = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT);
+    pFpuRes->r80Result         = *pr80Val;
+    pFpuRes->r80Result.s.fSign = 0;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fchs_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
+{
+    pFpuRes->FSW = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (7 << X86_FSW_TOP_SHIFT);
+    pFpuRes->r80Result         = *pr80Val;
+    pFpuRes->r80Result.s.fSign = !pr80Val->s.fSign;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fxtract_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = (pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3)) | (6 << X86_FSW_TOP_SHIFT);
+
+    if (RTFLOAT80U_IS_NORMAL(pr80Val))
+    {
+        softfloat_state_t Ignored = SOFTFLOAT_STATE_INIT_DEFAULTS();
+        iemFpuSoftF80ToIprt(&pFpuResTwo->r80Result1, i32_to_extF80((int32_t)pr80Val->s.uExponent - RTFLOAT80U_EXP_BIAS, &Ignored));
+
+        pFpuResTwo->r80Result2.s.fSign     = pr80Val->s.fSign;
+        pFpuResTwo->r80Result2.s.uExponent = RTFLOAT80U_EXP_BIAS;
+        pFpuResTwo->r80Result2.s.uMantissa = pr80Val->s.uMantissa;
+    }
+    else if (RTFLOAT80U_IS_ZERO(pr80Val))
+    {
+        fFsw |= X86_FSW_ZE;
+        if (fFcw & X86_FCW_ZM)
+        {
+            pFpuResTwo->r80Result1 = g_ar80Infinity[1];
+            pFpuResTwo->r80Result2 = *pr80Val;
+        }
+        else
+        {
+            pFpuResTwo->r80Result2 = *pr80Val;
+            fFsw = X86_FSW_ES | X86_FSW_B | (fFsw & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT);
+        }
+    }
+    else if (RTFLOAT80U_IS_DENORMAL_OR_PSEUDO_DENORMAL(pr80Val))
+    {
+        fFsw |= X86_FSW_DE;
+        if (fFcw & X86_FCW_DM)
+        {
+            pFpuResTwo->r80Result2.s.fSign     = pr80Val->s.fSign;
+            pFpuResTwo->r80Result2.s.uExponent = RTFLOAT80U_EXP_BIAS;
+            pFpuResTwo->r80Result2.s.uMantissa = pr80Val->s.uMantissa;
+            int32_t iExponent = -16382;
+            while (!(pFpuResTwo->r80Result2.s.uMantissa & RT_BIT_64(63)))
+            {
+                pFpuResTwo->r80Result2.s.uMantissa <<= 1;
+                iExponent--;
+            }
+
+            softfloat_state_t Ignored = SOFTFLOAT_STATE_INIT_DEFAULTS();
+            iemFpuSoftF80ToIprt(&pFpuResTwo->r80Result1, i32_to_extF80(iExponent, &Ignored));
+        }
+        else
+        {
+            pFpuResTwo->r80Result2 = *pr80Val;
+            fFsw = X86_FSW_ES | X86_FSW_B | (fFsw & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT);
+        }
+    }
+    else if (   RTFLOAT80U_IS_QUIET_NAN(pr80Val)
+             || RTFLOAT80U_IS_INDEFINITE(pr80Val))
+    {
+        pFpuResTwo->r80Result1 = *pr80Val;
+        pFpuResTwo->r80Result2 = *pr80Val;
+    }
+    else if (RTFLOAT80U_IS_INF(pr80Val))
+    {
+        pFpuResTwo->r80Result1 = g_ar80Infinity[0];
+        pFpuResTwo->r80Result2 = *pr80Val;
+    }
+    else
+    {
+        if (fFcw & X86_FCW_IM)
+        {
+            if (!RTFLOAT80U_IS_SIGNALLING_NAN(pr80Val))
+                pFpuResTwo->r80Result1 = g_r80Indefinite;
+            else
+            {
+                pFpuResTwo->r80Result1 = *pr80Val;
+                pFpuResTwo->r80Result1.s.uMantissa |= RT_BIT_64(62); /* make it quiet */
+            }
+            pFpuResTwo->r80Result2 = pFpuResTwo->r80Result1;
+        }
+        else
+        {
+            pFpuResTwo->r80Result2 = *pr80Val;
+            fFsw = X86_FSW_ES | X86_FSW_B | (fFsw & ~X86_FSW_TOP_MASK) | (7 << X86_FSW_TOP_SHIFT);
+        }
+        fFsw |= X86_FSW_IE;
+    }
+    pFpuResTwo->FSW = fFsw;
+}
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+#if defined(IEM_WITHOUT_ASSEMBLY)
+
+static uint16_t iemAImpl_fyl2x_r80_by_r80_normal(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw)
+{
+    softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS();
+    extFloat80_t y = iemFpuSoftF80FromIprt(pr80Val1);
+    extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val2);
+    extFloat80_t v;
+    (void)fFcw;
+
+    v = extF80_ylog2x(y, x, &SoftState);
+    iemFpuSoftF80ToIprt(pr80Result, v);
+
+    return fFsw;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2x_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                   PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3);
+
+    if (RTFLOAT80U_IS_NORMAL(pr80Val1) && RTFLOAT80U_IS_NORMAL(pr80Val2) && !pr80Val2->s.fSign)
+    {
+        fFsw |= iemAImpl_fyl2x_r80_by_r80_normal(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw);
+
+        fFsw |= X86_FSW_PE | (7 << X86_FSW_TOP_SHIFT);
+        if (!(fFcw & X86_FCW_PM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+    else
+    {
+        fFsw |= X86_FSW_IE;
+
+        if (!(fFcw & X86_FCW_IM))
+        {
+            pFpuRes->r80Result = *pr80Val2;
+            fFsw |= X86_FSW_ES | X86_FSW_B | (6 << X86_FSW_TOP_SHIFT);
+        }
+        else
+        {
+            pFpuRes->r80Result = g_r80Indefinite;
+            fFsw |= (7 << X86_FSW_TOP_SHIFT);
+        }
+    }
+
+    pFpuRes->FSW = fFsw;
+}
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2x_r80_by_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                         PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    iemAImpl_fyl2x_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2x_r80_by_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                       PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    iemAImpl_fyl2x_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2);
+}
+
+#if defined(IEM_WITHOUT_ASSEMBLY)
+
+static uint16_t iemAImpl_fyl2xp1_r80_by_r80_normal(PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2, PRTFLOAT80U pr80Result, uint16_t fFcw, uint16_t fFsw)
+{
+    softfloat_state_t SoftState = SOFTFLOAT_STATE_INIT_DEFAULTS();
+    extFloat80_t y = iemFpuSoftF80FromIprt(pr80Val1);
+    extFloat80_t x = iemFpuSoftF80FromIprt(pr80Val2);
+    extFloat80_t v;
+    (void)fFcw;
+
+    v = extF80_ylog2xp1(y, x, &SoftState);
+    iemFpuSoftF80ToIprt(pr80Result, v);
+
+    return fFsw;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2xp1_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                     PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    uint16_t const fFcw = pFpuState->FCW;
+    uint16_t fFsw       = pFpuState->FSW & (X86_FSW_C0 | X86_FSW_C2 | X86_FSW_C3);
+
+    if (RTFLOAT80U_IS_NORMAL(pr80Val1) && RTFLOAT80U_IS_NORMAL(pr80Val2) && pr80Val2->s.uExponent < RTFLOAT80U_EXP_BIAS)
+    {
+        fFsw = iemAImpl_fyl2xp1_r80_by_r80_normal(pr80Val1, pr80Val2, &pFpuRes->r80Result, fFcw, fFsw);
+
+        fFsw |= X86_FSW_PE | (7 << X86_FSW_TOP_SHIFT);
+        if (!(fFcw & X86_FCW_PM))
+            fFsw |= X86_FSW_ES | X86_FSW_B;
+    }
+    else
+    {
+        fFsw |= X86_FSW_IE;
+
+        if (!(fFcw & X86_FCW_IM))
+        {
+            pFpuRes->r80Result = *pr80Val2;
+            fFsw |= X86_FSW_ES | X86_FSW_B | (6 << X86_FSW_TOP_SHIFT);
+        }
+        else
+        {
+            pFpuRes->r80Result = g_r80Indefinite;
+            fFsw |= (7 << X86_FSW_TOP_SHIFT);
+        }
+    }
+
+    pFpuRes->FSW = fFsw;
+}
+
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2xp1_r80_by_r80_intel,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                           PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    iemAImpl_fyl2xp1_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2);
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2xp1_r80_by_r80_amd,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
+                                                         PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
+{
+    iemAImpl_fyl2xp1_r80_by_r80(pFpuState, pFpuRes, pr80Val1, pr80Val2);
+}
+
+
+/*********************************************************************************************************************************
+*   MMX, SSE & AVX                                                                                                               *
+*********************************************************************************************************************************/
+
+#ifdef IEM_WITH_VEX
+
+/*
+ * VMOVSLDUP
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc))
+{
+    pXState->x87.aXMM[iYRegDst].au32[0] = pXState->x87.aXMM[iYRegSrc].au32[0];
+    pXState->x87.aXMM[iYRegDst].au32[1] = pXState->x87.aXMM[iYRegSrc].au32[0];
+    pXState->x87.aXMM[iYRegDst].au32[2] = pXState->x87.aXMM[iYRegSrc].au32[2];
+    pXState->x87.aXMM[iYRegDst].au32[3] = pXState->x87.aXMM[iYRegSrc].au32[2];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[0];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[0];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[2];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[2];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc))
+{
+    pXState->x87.aXMM[iYRegDst].au32[0]       = pSrc->au32[0];
+    pXState->x87.aXMM[iYRegDst].au32[1]       = pSrc->au32[0];
+    pXState->x87.aXMM[iYRegDst].au32[2]       = pSrc->au32[2];
+    pXState->x87.aXMM[iYRegDst].au32[3]       = pSrc->au32[2];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pSrc->au32[4];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pSrc->au32[4];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pSrc->au32[6];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pSrc->au32[6];
+}
+
+#endif /* IEM_WITH_VEX */
+
+
+#ifdef IEM_WITH_VEX
+
+/*
+ * VMOVSHDUP
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovshdup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc))
+{
+    pXState->x87.aXMM[iYRegDst].au32[0] = pXState->x87.aXMM[iYRegSrc].au32[1];
+    pXState->x87.aXMM[iYRegDst].au32[1] = pXState->x87.aXMM[iYRegSrc].au32[1];
+    pXState->x87.aXMM[iYRegDst].au32[2] = pXState->x87.aXMM[iYRegSrc].au32[3];
+    pXState->x87.aXMM[iYRegDst].au32[3] = pXState->x87.aXMM[iYRegSrc].au32[3];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[1];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[1];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[3];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovshdup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc))
+{
+    pXState->x87.aXMM[iYRegDst].au32[0]       = pSrc->au32[1];
+    pXState->x87.aXMM[iYRegDst].au32[1]       = pSrc->au32[1];
+    pXState->x87.aXMM[iYRegDst].au32[2]       = pSrc->au32[3];
+    pXState->x87.aXMM[iYRegDst].au32[3]       = pSrc->au32[3];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pSrc->au32[5];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pSrc->au32[5];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pSrc->au32[7];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pSrc->au32[7];
+}
+
+#endif /* IEM_WITH_VEX */
+
+
+#ifdef IEM_WITH_VEX
+
+/*
+ * VMOVDDUP
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc))
+{
+    pXState->x87.aXMM[iYRegDst].au64[0] = pXState->x87.aXMM[iYRegSrc].au64[0];
+    pXState->x87.aXMM[iYRegDst].au64[1] = pXState->x87.aXMM[iYRegSrc].au64[0];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au64[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au64[0];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au64[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au64[0];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc))
+{
+    pXState->x87.aXMM[iYRegDst].au64[0]       = pSrc->au64[0];
+    pXState->x87.aXMM[iYRegDst].au64[1]       = pSrc->au64[0];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au64[0] = pSrc->au64[2];
+    pXState->u.YmmHi.aYmmHi[iYRegDst].au64[1] = pSrc->au64[2];
+}
+
+#endif /* IEM_WITH_VEX */
+
+
+/*
+ * PAND / VPAND / PANDPS / VPANDPS / PANDPD / VPANDPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pand_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    *puDst &= *puSrc;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pand_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    puDst->au64[0] &= puSrc->au64[0];
+    puDst->au64[1] &= puSrc->au64[1];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpand_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                      PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] & puSrc2->au64[0];
+    puDst->au64[1] = puSrc1->au64[1] & puSrc2->au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpand_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                      PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] & puSrc2->au64[0];
+    puDst->au64[1] = puSrc1->au64[1] & puSrc2->au64[1];
+    puDst->au64[2] = puSrc1->au64[2] & puSrc2->au64[2];
+    puDst->au64[3] = puSrc1->au64[3] & puSrc2->au64[3];
+}
+
+
+/*
+ * PANDN / VPANDN / PANDNPS / VPANDNPS / PANDNPD / VPANDNPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pandn_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    *puDst = ~*puDst & *puSrc;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pandn_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    puDst->au64[0] = ~puDst->au64[0] & puSrc->au64[0];
+    puDst->au64[1] = ~puDst->au64[1] & puSrc->au64[1];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpandn_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                       PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = ~puSrc1->au64[0] & puSrc2->au64[0];
+    puDst->au64[1] = ~puSrc1->au64[1] & puSrc2->au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpandn_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                       PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = ~puSrc1->au64[0] & puSrc2->au64[0];
+    puDst->au64[1] = ~puSrc1->au64[1] & puSrc2->au64[1];
+    puDst->au64[2] = ~puSrc1->au64[2] & puSrc2->au64[2];
+    puDst->au64[3] = ~puSrc1->au64[3] & puSrc2->au64[3];
+}
+
+
+/*
+ * POR / VPOR / PORPS / VPORPS / PORPD / VPORPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_por_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    *puDst |= *puSrc;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_por_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    puDst->au64[0] |= puSrc->au64[0];
+    puDst->au64[1] |= puSrc->au64[1];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpor_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                     PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] | puSrc2->au64[0];
+    puDst->au64[1] = puSrc1->au64[1] | puSrc2->au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpor_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                     PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] | puSrc2->au64[0];
+    puDst->au64[1] = puSrc1->au64[1] | puSrc2->au64[1];
+    puDst->au64[2] = puSrc1->au64[2] | puSrc2->au64[2];
+    puDst->au64[3] = puSrc1->au64[3] | puSrc2->au64[3];
+}
+
+
+/*
+ * PXOR / VPXOR / PXORPS / VPXORPS / PXORPD / VPXORPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pxor_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    *puDst ^= *puSrc;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pxor_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    puDst->au64[0] ^= puSrc->au64[0];
+    puDst->au64[1] ^= puSrc->au64[1];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpxor_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                      PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] ^ puSrc2->au64[0];
+    puDst->au64[1] = puSrc1->au64[1] ^ puSrc2->au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpxor_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                      PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] ^ puSrc2->au64[0];
+    puDst->au64[1] = puSrc1->au64[1] ^ puSrc2->au64[1];
+    puDst->au64[2] = puSrc1->au64[2] ^ puSrc2->au64[2];
+    puDst->au64[3] = puSrc1->au64[3] ^ puSrc2->au64[3];
+}
+
+
+/*
+ * PCMPEQB / VPCMPEQB
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au8[0] = uSrc1.au8[0] == uSrc2.au8[0] ? 0xff : 0;
+    uDst.au8[1] = uSrc1.au8[1] == uSrc2.au8[1] ? 0xff : 0;
+    uDst.au8[2] = uSrc1.au8[2] == uSrc2.au8[2] ? 0xff : 0;
+    uDst.au8[3] = uSrc1.au8[3] == uSrc2.au8[3] ? 0xff : 0;
+    uDst.au8[4] = uSrc1.au8[4] == uSrc2.au8[4] ? 0xff : 0;
+    uDst.au8[5] = uSrc1.au8[5] == uSrc2.au8[5] ? 0xff : 0;
+    uDst.au8[6] = uSrc1.au8[6] == uSrc2.au8[6] ? 0xff : 0;
+    uDst.au8[7] = uSrc1.au8[7] == uSrc2.au8[7] ? 0xff : 0;
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au8[0]  = uSrc1.au8[0]  == puSrc->au8[0]  ? UINT8_MAX : 0;
+    puDst->au8[1]  = uSrc1.au8[1]  == puSrc->au8[1]  ? UINT8_MAX : 0;
+    puDst->au8[2]  = uSrc1.au8[2]  == puSrc->au8[2]  ? UINT8_MAX : 0;
+    puDst->au8[3]  = uSrc1.au8[3]  == puSrc->au8[3]  ? UINT8_MAX : 0;
+    puDst->au8[4]  = uSrc1.au8[4]  == puSrc->au8[4]  ? UINT8_MAX : 0;
+    puDst->au8[5]  = uSrc1.au8[5]  == puSrc->au8[5]  ? UINT8_MAX : 0;
+    puDst->au8[6]  = uSrc1.au8[6]  == puSrc->au8[6]  ? UINT8_MAX : 0;
+    puDst->au8[7]  = uSrc1.au8[7]  == puSrc->au8[7]  ? UINT8_MAX : 0;
+    puDst->au8[8]  = uSrc1.au8[8]  == puSrc->au8[8]  ? UINT8_MAX : 0;
+    puDst->au8[9]  = uSrc1.au8[9]  == puSrc->au8[9]  ? UINT8_MAX : 0;
+    puDst->au8[10] = uSrc1.au8[10] == puSrc->au8[10] ? UINT8_MAX : 0;
+    puDst->au8[11] = uSrc1.au8[11] == puSrc->au8[11] ? UINT8_MAX : 0;
+    puDst->au8[12] = uSrc1.au8[12] == puSrc->au8[12] ? UINT8_MAX : 0;
+    puDst->au8[13] = uSrc1.au8[13] == puSrc->au8[13] ? UINT8_MAX : 0;
+    puDst->au8[14] = uSrc1.au8[14] == puSrc->au8[14] ? UINT8_MAX : 0;
+    puDst->au8[15] = uSrc1.au8[15] == puSrc->au8[15] ? UINT8_MAX : 0;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                         PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au8[0]  = puSrc1->au8[0]  == puSrc2->au8[0]  ? UINT8_MAX : 0;
+    puDst->au8[1]  = puSrc1->au8[1]  == puSrc2->au8[1]  ? UINT8_MAX : 0;
+    puDst->au8[2]  = puSrc1->au8[2]  == puSrc2->au8[2]  ? UINT8_MAX : 0;
+    puDst->au8[3]  = puSrc1->au8[3]  == puSrc2->au8[3]  ? UINT8_MAX : 0;
+    puDst->au8[4]  = puSrc1->au8[4]  == puSrc2->au8[4]  ? UINT8_MAX : 0;
+    puDst->au8[5]  = puSrc1->au8[5]  == puSrc2->au8[5]  ? UINT8_MAX : 0;
+    puDst->au8[6]  = puSrc1->au8[6]  == puSrc2->au8[6]  ? UINT8_MAX : 0;
+    puDst->au8[7]  = puSrc1->au8[7]  == puSrc2->au8[7]  ? UINT8_MAX : 0;
+    puDst->au8[8]  = puSrc1->au8[8]  == puSrc2->au8[8]  ? UINT8_MAX : 0;
+    puDst->au8[9]  = puSrc1->au8[9]  == puSrc2->au8[9]  ? UINT8_MAX : 0;
+    puDst->au8[10] = puSrc1->au8[10] == puSrc2->au8[10] ? UINT8_MAX : 0;
+    puDst->au8[11] = puSrc1->au8[11] == puSrc2->au8[11] ? UINT8_MAX : 0;
+    puDst->au8[12] = puSrc1->au8[12] == puSrc2->au8[12] ? UINT8_MAX : 0;
+    puDst->au8[13] = puSrc1->au8[13] == puSrc2->au8[13] ? UINT8_MAX : 0;
+    puDst->au8[14] = puSrc1->au8[14] == puSrc2->au8[14] ? UINT8_MAX : 0;
+    puDst->au8[15] = puSrc1->au8[15] == puSrc2->au8[15] ? UINT8_MAX : 0;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                         PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au8[0]  = puSrc1->au8[0]  == puSrc2->au8[0]  ? UINT8_MAX : 0;
+    puDst->au8[1]  = puSrc1->au8[1]  == puSrc2->au8[1]  ? UINT8_MAX : 0;
+    puDst->au8[2]  = puSrc1->au8[2]  == puSrc2->au8[2]  ? UINT8_MAX : 0;
+    puDst->au8[3]  = puSrc1->au8[3]  == puSrc2->au8[3]  ? UINT8_MAX : 0;
+    puDst->au8[4]  = puSrc1->au8[4]  == puSrc2->au8[4]  ? UINT8_MAX : 0;
+    puDst->au8[5]  = puSrc1->au8[5]  == puSrc2->au8[5]  ? UINT8_MAX : 0;
+    puDst->au8[6]  = puSrc1->au8[6]  == puSrc2->au8[6]  ? UINT8_MAX : 0;
+    puDst->au8[7]  = puSrc1->au8[7]  == puSrc2->au8[7]  ? UINT8_MAX : 0;
+    puDst->au8[8]  = puSrc1->au8[8]  == puSrc2->au8[8]  ? UINT8_MAX : 0;
+    puDst->au8[9]  = puSrc1->au8[9]  == puSrc2->au8[9]  ? UINT8_MAX : 0;
+    puDst->au8[10] = puSrc1->au8[10] == puSrc2->au8[10] ? UINT8_MAX : 0;
+    puDst->au8[11] = puSrc1->au8[11] == puSrc2->au8[11] ? UINT8_MAX : 0;
+    puDst->au8[12] = puSrc1->au8[12] == puSrc2->au8[12] ? UINT8_MAX : 0;
+    puDst->au8[13] = puSrc1->au8[13] == puSrc2->au8[13] ? UINT8_MAX : 0;
+    puDst->au8[14] = puSrc1->au8[14] == puSrc2->au8[14] ? UINT8_MAX : 0;
+    puDst->au8[15] = puSrc1->au8[15] == puSrc2->au8[15] ? UINT8_MAX : 0;
+    puDst->au8[16] = puSrc1->au8[16] == puSrc2->au8[16] ? UINT8_MAX : 0;
+    puDst->au8[17] = puSrc1->au8[17] == puSrc2->au8[17] ? UINT8_MAX : 0;
+    puDst->au8[18] = puSrc1->au8[18] == puSrc2->au8[18] ? UINT8_MAX : 0;
+    puDst->au8[19] = puSrc1->au8[19] == puSrc2->au8[19] ? UINT8_MAX : 0;
+    puDst->au8[20] = puSrc1->au8[20] == puSrc2->au8[20] ? UINT8_MAX : 0;
+    puDst->au8[21] = puSrc1->au8[21] == puSrc2->au8[21] ? UINT8_MAX : 0;
+    puDst->au8[22] = puSrc1->au8[22] == puSrc2->au8[22] ? UINT8_MAX : 0;
+    puDst->au8[23] = puSrc1->au8[23] == puSrc2->au8[23] ? UINT8_MAX : 0;
+    puDst->au8[24] = puSrc1->au8[24] == puSrc2->au8[24] ? UINT8_MAX : 0;
+    puDst->au8[25] = puSrc1->au8[25] == puSrc2->au8[25] ? UINT8_MAX : 0;
+    puDst->au8[26] = puSrc1->au8[26] == puSrc2->au8[26] ? UINT8_MAX : 0;
+    puDst->au8[27] = puSrc1->au8[27] == puSrc2->au8[27] ? UINT8_MAX : 0;
+    puDst->au8[28] = puSrc1->au8[28] == puSrc2->au8[28] ? UINT8_MAX : 0;
+    puDst->au8[29] = puSrc1->au8[29] == puSrc2->au8[29] ? UINT8_MAX : 0;
+    puDst->au8[30] = puSrc1->au8[30] == puSrc2->au8[30] ? UINT8_MAX : 0;
+    puDst->au8[31] = puSrc1->au8[31] == puSrc2->au8[31] ? UINT8_MAX : 0;
+}
+
+
+/*
+ * PCMPEQW / VPCMPEQW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au16[0] = uSrc1.au16[0] == uSrc2.au16[0] ? UINT16_MAX : 0;
+    uDst.au16[1] = uSrc1.au16[1] == uSrc2.au16[1] ? UINT16_MAX : 0;
+    uDst.au16[2] = uSrc1.au16[2] == uSrc2.au16[2] ? UINT16_MAX : 0;
+    uDst.au16[3] = uSrc1.au16[3] == uSrc2.au16[3] ? UINT16_MAX : 0;
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au16[0] = uSrc1.au16[0] == puSrc->au16[0] ? UINT16_MAX : 0;
+    puDst->au16[1] = uSrc1.au16[1] == puSrc->au16[1] ? UINT16_MAX : 0;
+    puDst->au16[2] = uSrc1.au16[2] == puSrc->au16[2] ? UINT16_MAX : 0;
+    puDst->au16[3] = uSrc1.au16[3] == puSrc->au16[3] ? UINT16_MAX : 0;
+    puDst->au16[4] = uSrc1.au16[4] == puSrc->au16[4] ? UINT16_MAX : 0;
+    puDst->au16[5] = uSrc1.au16[5] == puSrc->au16[5] ? UINT16_MAX : 0;
+    puDst->au16[6] = uSrc1.au16[6] == puSrc->au16[6] ? UINT16_MAX : 0;
+    puDst->au16[7] = uSrc1.au16[7] == puSrc->au16[7] ? UINT16_MAX : 0;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                         PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au16[0] = puSrc1->au16[0] == puSrc2->au16[0] ? UINT16_MAX : 0;
+    puDst->au16[1] = puSrc1->au16[1] == puSrc2->au16[1] ? UINT16_MAX : 0;
+    puDst->au16[2] = puSrc1->au16[2] == puSrc2->au16[2] ? UINT16_MAX : 0;
+    puDst->au16[3] = puSrc1->au16[3] == puSrc2->au16[3] ? UINT16_MAX : 0;
+    puDst->au16[4] = puSrc1->au16[4] == puSrc2->au16[4] ? UINT16_MAX : 0;
+    puDst->au16[5] = puSrc1->au16[5] == puSrc2->au16[5] ? UINT16_MAX : 0;
+    puDst->au16[6] = puSrc1->au16[6] == puSrc2->au16[6] ? UINT16_MAX : 0;
+    puDst->au16[7] = puSrc1->au16[7] == puSrc2->au16[7] ? UINT16_MAX : 0;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                         PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au16[0]  = puSrc1->au16[0]  == puSrc2->au16[0]  ? UINT16_MAX : 0;
+    puDst->au16[1]  = puSrc1->au16[1]  == puSrc2->au16[1]  ? UINT16_MAX : 0;
+    puDst->au16[2]  = puSrc1->au16[2]  == puSrc2->au16[2]  ? UINT16_MAX : 0;
+    puDst->au16[3]  = puSrc1->au16[3]  == puSrc2->au16[3]  ? UINT16_MAX : 0;
+    puDst->au16[4]  = puSrc1->au16[4]  == puSrc2->au16[4]  ? UINT16_MAX : 0;
+    puDst->au16[5]  = puSrc1->au16[5]  == puSrc2->au16[5]  ? UINT16_MAX : 0;
+    puDst->au16[6]  = puSrc1->au16[6]  == puSrc2->au16[6]  ? UINT16_MAX : 0;
+    puDst->au16[7]  = puSrc1->au16[7]  == puSrc2->au16[7]  ? UINT16_MAX : 0;
+    puDst->au16[8]  = puSrc1->au16[8]  == puSrc2->au16[8]  ? UINT16_MAX : 0;
+    puDst->au16[9]  = puSrc1->au16[9]  == puSrc2->au16[9]  ? UINT16_MAX : 0;
+    puDst->au16[10] = puSrc1->au16[10] == puSrc2->au16[10] ? UINT16_MAX : 0;
+    puDst->au16[11] = puSrc1->au16[11] == puSrc2->au16[11] ? UINT16_MAX : 0;
+    puDst->au16[12] = puSrc1->au16[12] == puSrc2->au16[12] ? UINT16_MAX : 0;
+    puDst->au16[13] = puSrc1->au16[13] == puSrc2->au16[13] ? UINT16_MAX : 0;
+    puDst->au16[14] = puSrc1->au16[14] == puSrc2->au16[14] ? UINT16_MAX : 0;
+    puDst->au16[15] = puSrc1->au16[15] == puSrc2->au16[15] ? UINT16_MAX : 0;
+}
+
+
+/*
+ * PCMPEQD / VPCMPEQD.
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqd_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au32[0] = uSrc1.au32[0] == uSrc2.au32[0] ? UINT32_MAX : 0;
+    uDst.au32[1] = uSrc1.au32[1] == uSrc2.au32[1] ? UINT32_MAX : 0;
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au32[0] = uSrc1.au32[0] == puSrc->au32[0] ? UINT32_MAX : 0;
+    puDst->au32[1] = uSrc1.au32[1] == puSrc->au32[1] ? UINT32_MAX : 0;
+    puDst->au32[2] = uSrc1.au32[2] == puSrc->au32[2] ? UINT32_MAX : 0;
+    puDst->au32[3] = uSrc1.au32[3] == puSrc->au32[3] ? UINT32_MAX : 0;
+}
+
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                         PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au32[0] = puSrc1->au32[0] == puSrc2->au32[0] ? UINT32_MAX : 0;
+    puDst->au32[1] = puSrc1->au32[1] == puSrc2->au32[1] ? UINT32_MAX : 0;
+    puDst->au32[2] = puSrc1->au32[2] == puSrc2->au32[2] ? UINT32_MAX : 0;
+    puDst->au32[3] = puSrc1->au32[3] == puSrc2->au32[3] ? UINT32_MAX : 0;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                         PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au32[0] = puSrc1->au32[0] == puSrc2->au32[0] ? UINT32_MAX : 0;
+    puDst->au32[1] = puSrc1->au32[1] == puSrc2->au32[1] ? UINT32_MAX : 0;
+    puDst->au32[2] = puSrc1->au32[2] == puSrc2->au32[2] ? UINT32_MAX : 0;
+    puDst->au32[3] = puSrc1->au32[3] == puSrc2->au32[3] ? UINT32_MAX : 0;
+    puDst->au32[4] = puSrc1->au32[4] == puSrc2->au32[4] ? UINT32_MAX : 0;
+    puDst->au32[5] = puSrc1->au32[5] == puSrc2->au32[5] ? UINT32_MAX : 0;
+    puDst->au32[6] = puSrc1->au32[6] == puSrc2->au32[6] ? UINT32_MAX : 0;
+    puDst->au32[7] = puSrc1->au32[7] == puSrc2->au32[7] ? UINT32_MAX : 0;
+}
+
+
+/*
+ * PCMPEQQ / VPCMPEQQ.
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqq_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au64[0] = uSrc1.au64[0] == puSrc->au64[0] ? UINT64_MAX : 0;
+    puDst->au64[1] = uSrc1.au64[1] == puSrc->au64[1] ? UINT64_MAX : 0;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqq_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                         PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] == puSrc2->au64[0] ? UINT64_MAX : 0;
+    puDst->au64[1] = puSrc1->au64[1] == puSrc2->au64[1] ? UINT64_MAX : 0;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpeqq_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                         PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] == puSrc2->au64[0] ? UINT64_MAX : 0;
+    puDst->au64[1] = puSrc1->au64[1] == puSrc2->au64[1] ? UINT64_MAX : 0;
+    puDst->au64[2] = puSrc1->au64[2] == puSrc2->au64[2] ? UINT64_MAX : 0;
+    puDst->au64[3] = puSrc1->au64[3] == puSrc2->au64[3] ? UINT64_MAX : 0;
+}
+
+
+/*
+ * PCMPGTB / VPCMPGTB
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au8[0] = uSrc1.ai8[0] > uSrc2.ai8[0] ? UINT8_MAX : 0;
+    uDst.au8[1] = uSrc1.ai8[1] > uSrc2.ai8[1] ? UINT8_MAX : 0;
+    uDst.au8[2] = uSrc1.ai8[2] > uSrc2.ai8[2] ? UINT8_MAX : 0;
+    uDst.au8[3] = uSrc1.ai8[3] > uSrc2.ai8[3] ? UINT8_MAX : 0;
+    uDst.au8[4] = uSrc1.ai8[4] > uSrc2.ai8[4] ? UINT8_MAX : 0;
+    uDst.au8[5] = uSrc1.ai8[5] > uSrc2.ai8[5] ? UINT8_MAX : 0;
+    uDst.au8[6] = uSrc1.ai8[6] > uSrc2.ai8[6] ? UINT8_MAX : 0;
+    uDst.au8[7] = uSrc1.ai8[7] > uSrc2.ai8[7] ? UINT8_MAX : 0;
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au8[0]  = uSrc1.ai8[0]  > puSrc->ai8[0]  ? UINT8_MAX : 0;
+    puDst->au8[1]  = uSrc1.ai8[1]  > puSrc->ai8[1]  ? UINT8_MAX : 0;
+    puDst->au8[2]  = uSrc1.ai8[2]  > puSrc->ai8[2]  ? UINT8_MAX : 0;
+    puDst->au8[3]  = uSrc1.ai8[3]  > puSrc->ai8[3]  ? UINT8_MAX : 0;
+    puDst->au8[4]  = uSrc1.ai8[4]  > puSrc->ai8[4]  ? UINT8_MAX : 0;
+    puDst->au8[5]  = uSrc1.ai8[5]  > puSrc->ai8[5]  ? UINT8_MAX : 0;
+    puDst->au8[6]  = uSrc1.ai8[6]  > puSrc->ai8[6]  ? UINT8_MAX : 0;
+    puDst->au8[7]  = uSrc1.ai8[7]  > puSrc->ai8[7]  ? UINT8_MAX : 0;
+    puDst->au8[8]  = uSrc1.ai8[8]  > puSrc->ai8[8]  ? UINT8_MAX : 0;
+    puDst->au8[9]  = uSrc1.ai8[9]  > puSrc->ai8[9]  ? UINT8_MAX : 0;
+    puDst->au8[10] = uSrc1.ai8[10] > puSrc->ai8[10] ? UINT8_MAX : 0;
+    puDst->au8[11] = uSrc1.ai8[11] > puSrc->ai8[11] ? UINT8_MAX : 0;
+    puDst->au8[12] = uSrc1.ai8[12] > puSrc->ai8[12] ? UINT8_MAX : 0;
+    puDst->au8[13] = uSrc1.ai8[13] > puSrc->ai8[13] ? UINT8_MAX : 0;
+    puDst->au8[14] = uSrc1.ai8[14] > puSrc->ai8[14] ? UINT8_MAX : 0;
+    puDst->au8[15] = uSrc1.ai8[15] > puSrc->ai8[15] ? UINT8_MAX : 0;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                         PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au8[0]  = puSrc1->ai8[0]  > puSrc2->ai8[0]  ? UINT8_MAX : 0;
+    puDst->au8[1]  = puSrc1->ai8[1]  > puSrc2->ai8[1]  ? UINT8_MAX : 0;
+    puDst->au8[2]  = puSrc1->ai8[2]  > puSrc2->ai8[2]  ? UINT8_MAX : 0;
+    puDst->au8[3]  = puSrc1->ai8[3]  > puSrc2->ai8[3]  ? UINT8_MAX : 0;
+    puDst->au8[4]  = puSrc1->ai8[4]  > puSrc2->ai8[4]  ? UINT8_MAX : 0;
+    puDst->au8[5]  = puSrc1->ai8[5]  > puSrc2->ai8[5]  ? UINT8_MAX : 0;
+    puDst->au8[6]  = puSrc1->ai8[6]  > puSrc2->ai8[6]  ? UINT8_MAX : 0;
+    puDst->au8[7]  = puSrc1->ai8[7]  > puSrc2->ai8[7]  ? UINT8_MAX : 0;
+    puDst->au8[8]  = puSrc1->ai8[8]  > puSrc2->ai8[8]  ? UINT8_MAX : 0;
+    puDst->au8[9]  = puSrc1->ai8[9]  > puSrc2->ai8[9]  ? UINT8_MAX : 0;
+    puDst->au8[10] = puSrc1->ai8[10] > puSrc2->ai8[10] ? UINT8_MAX : 0;
+    puDst->au8[11] = puSrc1->ai8[11] > puSrc2->ai8[11] ? UINT8_MAX : 0;
+    puDst->au8[12] = puSrc1->ai8[12] > puSrc2->ai8[12] ? UINT8_MAX : 0;
+    puDst->au8[13] = puSrc1->ai8[13] > puSrc2->ai8[13] ? UINT8_MAX : 0;
+    puDst->au8[14] = puSrc1->ai8[14] > puSrc2->ai8[14] ? UINT8_MAX : 0;
+    puDst->au8[15] = puSrc1->ai8[15] > puSrc2->ai8[15] ? UINT8_MAX : 0;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                         PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au8[0]  = puSrc1->ai8[0]  > puSrc2->ai8[0]  ? UINT8_MAX : 0;
+    puDst->au8[1]  = puSrc1->ai8[1]  > puSrc2->ai8[1]  ? UINT8_MAX : 0;
+    puDst->au8[2]  = puSrc1->ai8[2]  > puSrc2->ai8[2]  ? UINT8_MAX : 0;
+    puDst->au8[3]  = puSrc1->ai8[3]  > puSrc2->ai8[3]  ? UINT8_MAX : 0;
+    puDst->au8[4]  = puSrc1->ai8[4]  > puSrc2->ai8[4]  ? UINT8_MAX : 0;
+    puDst->au8[5]  = puSrc1->ai8[5]  > puSrc2->ai8[5]  ? UINT8_MAX : 0;
+    puDst->au8[6]  = puSrc1->ai8[6]  > puSrc2->ai8[6]  ? UINT8_MAX : 0;
+    puDst->au8[7]  = puSrc1->ai8[7]  > puSrc2->ai8[7]  ? UINT8_MAX : 0;
+    puDst->au8[8]  = puSrc1->ai8[8]  > puSrc2->ai8[8]  ? UINT8_MAX : 0;
+    puDst->au8[9]  = puSrc1->ai8[9]  > puSrc2->ai8[9]  ? UINT8_MAX : 0;
+    puDst->au8[10] = puSrc1->ai8[10] > puSrc2->ai8[10] ? UINT8_MAX : 0;
+    puDst->au8[11] = puSrc1->ai8[11] > puSrc2->ai8[11] ? UINT8_MAX : 0;
+    puDst->au8[12] = puSrc1->ai8[12] > puSrc2->ai8[12] ? UINT8_MAX : 0;
+    puDst->au8[13] = puSrc1->ai8[13] > puSrc2->ai8[13] ? UINT8_MAX : 0;
+    puDst->au8[14] = puSrc1->ai8[14] > puSrc2->ai8[14] ? UINT8_MAX : 0;
+    puDst->au8[15] = puSrc1->ai8[15] > puSrc2->ai8[15] ? UINT8_MAX : 0;
+    puDst->au8[16] = puSrc1->ai8[16] > puSrc2->ai8[16] ? UINT8_MAX : 0;
+    puDst->au8[17] = puSrc1->ai8[17] > puSrc2->ai8[17] ? UINT8_MAX : 0;
+    puDst->au8[18] = puSrc1->ai8[18] > puSrc2->ai8[18] ? UINT8_MAX : 0;
+    puDst->au8[19] = puSrc1->ai8[19] > puSrc2->ai8[19] ? UINT8_MAX : 0;
+    puDst->au8[20] = puSrc1->ai8[20] > puSrc2->ai8[20] ? UINT8_MAX : 0;
+    puDst->au8[21] = puSrc1->ai8[21] > puSrc2->ai8[21] ? UINT8_MAX : 0;
+    puDst->au8[22] = puSrc1->ai8[22] > puSrc2->ai8[22] ? UINT8_MAX : 0;
+    puDst->au8[23] = puSrc1->ai8[23] > puSrc2->ai8[23] ? UINT8_MAX : 0;
+    puDst->au8[24] = puSrc1->ai8[24] > puSrc2->ai8[24] ? UINT8_MAX : 0;
+    puDst->au8[25] = puSrc1->ai8[25] > puSrc2->ai8[25] ? UINT8_MAX : 0;
+    puDst->au8[26] = puSrc1->ai8[26] > puSrc2->ai8[26] ? UINT8_MAX : 0;
+    puDst->au8[27] = puSrc1->ai8[27] > puSrc2->ai8[27] ? UINT8_MAX : 0;
+    puDst->au8[28] = puSrc1->ai8[28] > puSrc2->ai8[28] ? UINT8_MAX : 0;
+    puDst->au8[29] = puSrc1->ai8[29] > puSrc2->ai8[29] ? UINT8_MAX : 0;
+    puDst->au8[30] = puSrc1->ai8[30] > puSrc2->ai8[30] ? UINT8_MAX : 0;
+    puDst->au8[31] = puSrc1->ai8[31] > puSrc2->ai8[31] ? UINT8_MAX : 0;
+}
+
+
+/*
+ * PCMPGTW / VPCMPGTW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au16[0] = uSrc1.ai16[0] > uSrc2.ai16[0] ? UINT16_MAX : 0;
+    uDst.au16[1] = uSrc1.ai16[1] > uSrc2.ai16[1] ? UINT16_MAX : 0;
+    uDst.au16[2] = uSrc1.ai16[2] > uSrc2.ai16[2] ? UINT16_MAX : 0;
+    uDst.au16[3] = uSrc1.ai16[3] > uSrc2.ai16[3] ? UINT16_MAX : 0;
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au16[0] = uSrc1.ai16[0] > puSrc->ai16[0] ? UINT16_MAX : 0;
+    puDst->au16[1] = uSrc1.ai16[1] > puSrc->ai16[1] ? UINT16_MAX : 0;
+    puDst->au16[2] = uSrc1.ai16[2] > puSrc->ai16[2] ? UINT16_MAX : 0;
+    puDst->au16[3] = uSrc1.ai16[3] > puSrc->ai16[3] ? UINT16_MAX : 0;
+    puDst->au16[4] = uSrc1.ai16[4] > puSrc->ai16[4] ? UINT16_MAX : 0;
+    puDst->au16[5] = uSrc1.ai16[5] > puSrc->ai16[5] ? UINT16_MAX : 0;
+    puDst->au16[6] = uSrc1.ai16[6] > puSrc->ai16[6] ? UINT16_MAX : 0;
+    puDst->au16[7] = uSrc1.ai16[7] > puSrc->ai16[7] ? UINT16_MAX : 0;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                         PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au16[0] = puSrc1->ai16[0] > puSrc2->ai16[0] ? UINT16_MAX : 0;
+    puDst->au16[1] = puSrc1->ai16[1] > puSrc2->ai16[1] ? UINT16_MAX : 0;
+    puDst->au16[2] = puSrc1->ai16[2] > puSrc2->ai16[2] ? UINT16_MAX : 0;
+    puDst->au16[3] = puSrc1->ai16[3] > puSrc2->ai16[3] ? UINT16_MAX : 0;
+    puDst->au16[4] = puSrc1->ai16[4] > puSrc2->ai16[4] ? UINT16_MAX : 0;
+    puDst->au16[5] = puSrc1->ai16[5] > puSrc2->ai16[5] ? UINT16_MAX : 0;
+    puDst->au16[6] = puSrc1->ai16[6] > puSrc2->ai16[6] ? UINT16_MAX : 0;
+    puDst->au16[7] = puSrc1->ai16[7] > puSrc2->ai16[7] ? UINT16_MAX : 0;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                         PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au16[0]  = puSrc1->ai16[0]  > puSrc2->ai16[0]  ? UINT16_MAX : 0;
+    puDst->au16[1]  = puSrc1->ai16[1]  > puSrc2->ai16[1]  ? UINT16_MAX : 0;
+    puDst->au16[2]  = puSrc1->ai16[2]  > puSrc2->ai16[2]  ? UINT16_MAX : 0;
+    puDst->au16[3]  = puSrc1->ai16[3]  > puSrc2->ai16[3]  ? UINT16_MAX : 0;
+    puDst->au16[4]  = puSrc1->ai16[4]  > puSrc2->ai16[4]  ? UINT16_MAX : 0;
+    puDst->au16[5]  = puSrc1->ai16[5]  > puSrc2->ai16[5]  ? UINT16_MAX : 0;
+    puDst->au16[6]  = puSrc1->ai16[6]  > puSrc2->ai16[6]  ? UINT16_MAX : 0;
+    puDst->au16[7]  = puSrc1->ai16[7]  > puSrc2->ai16[7]  ? UINT16_MAX : 0;
+    puDst->au16[8]  = puSrc1->ai16[8]  > puSrc2->ai16[8]  ? UINT16_MAX : 0;
+    puDst->au16[9]  = puSrc1->ai16[9]  > puSrc2->ai16[9]  ? UINT16_MAX : 0;
+    puDst->au16[10] = puSrc1->ai16[10] > puSrc2->ai16[10] ? UINT16_MAX : 0;
+    puDst->au16[11] = puSrc1->ai16[11] > puSrc2->ai16[11] ? UINT16_MAX : 0;
+    puDst->au16[12] = puSrc1->ai16[12] > puSrc2->ai16[12] ? UINT16_MAX : 0;
+    puDst->au16[13] = puSrc1->ai16[13] > puSrc2->ai16[13] ? UINT16_MAX : 0;
+    puDst->au16[14] = puSrc1->ai16[14] > puSrc2->ai16[14] ? UINT16_MAX : 0;
+    puDst->au16[15] = puSrc1->ai16[15] > puSrc2->ai16[15] ? UINT16_MAX : 0;
+}
+
+
+/*
+ * PCMPGTD / VPCMPGTD.
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtd_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au32[0] = uSrc1.ai32[0] > uSrc2.ai32[0] ? UINT32_MAX : 0;
+    uDst.au32[1] = uSrc1.ai32[1] > uSrc2.ai32[1] ? UINT32_MAX : 0;
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au32[0] = uSrc1.ai32[0] > puSrc->ai32[0] ? UINT32_MAX : 0;
+    puDst->au32[1] = uSrc1.ai32[1] > puSrc->ai32[1] ? UINT32_MAX : 0;
+    puDst->au32[2] = uSrc1.ai32[2] > puSrc->ai32[2] ? UINT32_MAX : 0;
+    puDst->au32[3] = uSrc1.ai32[3] > puSrc->ai32[3] ? UINT32_MAX : 0;
+}
+
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                         PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au32[0] = puSrc1->ai32[0] > puSrc2->ai32[0] ? UINT32_MAX : 0;
+    puDst->au32[1] = puSrc1->ai32[1] > puSrc2->ai32[1] ? UINT32_MAX : 0;
+    puDst->au32[2] = puSrc1->ai32[2] > puSrc2->ai32[2] ? UINT32_MAX : 0;
+    puDst->au32[3] = puSrc1->ai32[3] > puSrc2->ai32[3] ? UINT32_MAX : 0;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                         PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au32[0] = puSrc1->ai32[0] > puSrc2->ai32[0] ? UINT32_MAX : 0;
+    puDst->au32[1] = puSrc1->ai32[1] > puSrc2->ai32[1] ? UINT32_MAX : 0;
+    puDst->au32[2] = puSrc1->ai32[2] > puSrc2->ai32[2] ? UINT32_MAX : 0;
+    puDst->au32[3] = puSrc1->ai32[3] > puSrc2->ai32[3] ? UINT32_MAX : 0;
+    puDst->au32[4] = puSrc1->ai32[4] > puSrc2->ai32[4] ? UINT32_MAX : 0;
+    puDst->au32[5] = puSrc1->ai32[5] > puSrc2->ai32[5] ? UINT32_MAX : 0;
+    puDst->au32[6] = puSrc1->ai32[6] > puSrc2->ai32[6] ? UINT32_MAX : 0;
+    puDst->au32[7] = puSrc1->ai32[7] > puSrc2->ai32[7] ? UINT32_MAX : 0;
+}
+
+
+/*
+ * PCMPGTQ / VPCMPGTQ.
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpgtq_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au64[0] = uSrc1.ai64[0] > puSrc->ai64[0] ? UINT64_MAX : 0;
+    puDst->au64[1] = uSrc1.ai64[1] > puSrc->ai64[1] ? UINT64_MAX : 0;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtq_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                         PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->ai64[0] > puSrc2->ai64[0] ? UINT64_MAX : 0;
+    puDst->au64[1] = puSrc1->ai64[1] > puSrc2->ai64[1] ? UINT64_MAX : 0;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpcmpgtq_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                         PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->ai64[0] > puSrc2->ai64[0] ? UINT64_MAX : 0;
+    puDst->au64[1] = puSrc1->ai64[1] > puSrc2->ai64[1] ? UINT64_MAX : 0;
+    puDst->au64[2] = puSrc1->ai64[2] > puSrc2->ai64[2] ? UINT64_MAX : 0;
+    puDst->au64[3] = puSrc1->ai64[3] > puSrc2->ai64[3] ? UINT64_MAX : 0;
+}
+
+
+/*
+ * PADDB / VPADDB
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au8[0] = uSrc1.au8[0] + uSrc2.au8[0];
+    uDst.au8[1] = uSrc1.au8[1] + uSrc2.au8[1];
+    uDst.au8[2] = uSrc1.au8[2] + uSrc2.au8[2];
+    uDst.au8[3] = uSrc1.au8[3] + uSrc2.au8[3];
+    uDst.au8[4] = uSrc1.au8[4] + uSrc2.au8[4];
+    uDst.au8[5] = uSrc1.au8[5] + uSrc2.au8[5];
+    uDst.au8[6] = uSrc1.au8[6] + uSrc2.au8[6];
+    uDst.au8[7] = uSrc1.au8[7] + uSrc2.au8[7];
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au8[0]  = uSrc1.au8[0]  + puSrc->au8[0];
+    puDst->au8[1]  = uSrc1.au8[1]  + puSrc->au8[1];
+    puDst->au8[2]  = uSrc1.au8[2]  + puSrc->au8[2];
+    puDst->au8[3]  = uSrc1.au8[3]  + puSrc->au8[3];
+    puDst->au8[4]  = uSrc1.au8[4]  + puSrc->au8[4];
+    puDst->au8[5]  = uSrc1.au8[5]  + puSrc->au8[5];
+    puDst->au8[6]  = uSrc1.au8[6]  + puSrc->au8[6];
+    puDst->au8[7]  = uSrc1.au8[7]  + puSrc->au8[7];
+    puDst->au8[8]  = uSrc1.au8[8]  + puSrc->au8[8];
+    puDst->au8[9]  = uSrc1.au8[9]  + puSrc->au8[9];
+    puDst->au8[10] = uSrc1.au8[10] + puSrc->au8[10];
+    puDst->au8[11] = uSrc1.au8[11] + puSrc->au8[11];
+    puDst->au8[12] = uSrc1.au8[12] + puSrc->au8[12];
+    puDst->au8[13] = uSrc1.au8[13] + puSrc->au8[13];
+    puDst->au8[14] = uSrc1.au8[14] + puSrc->au8[14];
+    puDst->au8[15] = uSrc1.au8[15] + puSrc->au8[15];
+}
+
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                       PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au8[0]  = puSrc1->au8[0]  + puSrc2->au8[0];
+    puDst->au8[1]  = puSrc1->au8[1]  + puSrc2->au8[1];
+    puDst->au8[2]  = puSrc1->au8[2]  + puSrc2->au8[2];
+    puDst->au8[3]  = puSrc1->au8[3]  + puSrc2->au8[3];
+    puDst->au8[4]  = puSrc1->au8[4]  + puSrc2->au8[4];
+    puDst->au8[5]  = puSrc1->au8[5]  + puSrc2->au8[5];
+    puDst->au8[6]  = puSrc1->au8[6]  + puSrc2->au8[6];
+    puDst->au8[7]  = puSrc1->au8[7]  + puSrc2->au8[7];
+    puDst->au8[8]  = puSrc1->au8[8]  + puSrc2->au8[8];
+    puDst->au8[9]  = puSrc1->au8[9]  + puSrc2->au8[9];
+    puDst->au8[10] = puSrc1->au8[10] + puSrc2->au8[10];
+    puDst->au8[11] = puSrc1->au8[11] + puSrc2->au8[11];
+    puDst->au8[12] = puSrc1->au8[12] + puSrc2->au8[12];
+    puDst->au8[13] = puSrc1->au8[13] + puSrc2->au8[13];
+    puDst->au8[14] = puSrc1->au8[14] + puSrc2->au8[14];
+    puDst->au8[15] = puSrc1->au8[15] + puSrc2->au8[15];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                       PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au8[0]  = puSrc1->au8[0]  + puSrc2->au8[0];
+    puDst->au8[1]  = puSrc1->au8[1]  + puSrc2->au8[1];
+    puDst->au8[2]  = puSrc1->au8[2]  + puSrc2->au8[2];
+    puDst->au8[3]  = puSrc1->au8[3]  + puSrc2->au8[3];
+    puDst->au8[4]  = puSrc1->au8[4]  + puSrc2->au8[4];
+    puDst->au8[5]  = puSrc1->au8[5]  + puSrc2->au8[5];
+    puDst->au8[6]  = puSrc1->au8[6]  + puSrc2->au8[6];
+    puDst->au8[7]  = puSrc1->au8[7]  + puSrc2->au8[7];
+    puDst->au8[8]  = puSrc1->au8[8]  + puSrc2->au8[8];
+    puDst->au8[9]  = puSrc1->au8[9]  + puSrc2->au8[9];
+    puDst->au8[10] = puSrc1->au8[10] + puSrc2->au8[10];
+    puDst->au8[11] = puSrc1->au8[11] + puSrc2->au8[11];
+    puDst->au8[12] = puSrc1->au8[12] + puSrc2->au8[12];
+    puDst->au8[13] = puSrc1->au8[13] + puSrc2->au8[13];
+    puDst->au8[14] = puSrc1->au8[14] + puSrc2->au8[14];
+    puDst->au8[15] = puSrc1->au8[15] + puSrc2->au8[15];
+    puDst->au8[16] = puSrc1->au8[16] + puSrc2->au8[16];
+    puDst->au8[17] = puSrc1->au8[17] + puSrc2->au8[17];
+    puDst->au8[18] = puSrc1->au8[18] + puSrc2->au8[18];
+    puDst->au8[19] = puSrc1->au8[19] + puSrc2->au8[19];
+    puDst->au8[20] = puSrc1->au8[20] + puSrc2->au8[20];
+    puDst->au8[21] = puSrc1->au8[21] + puSrc2->au8[21];
+    puDst->au8[22] = puSrc1->au8[22] + puSrc2->au8[22];
+    puDst->au8[23] = puSrc1->au8[23] + puSrc2->au8[23];
+    puDst->au8[24] = puSrc1->au8[24] + puSrc2->au8[24];
+    puDst->au8[25] = puSrc1->au8[25] + puSrc2->au8[25];
+    puDst->au8[26] = puSrc1->au8[26] + puSrc2->au8[26];
+    puDst->au8[27] = puSrc1->au8[27] + puSrc2->au8[27];
+    puDst->au8[28] = puSrc1->au8[28] + puSrc2->au8[28];
+    puDst->au8[29] = puSrc1->au8[29] + puSrc2->au8[29];
+    puDst->au8[30] = puSrc1->au8[30] + puSrc2->au8[30];
+    puDst->au8[31] = puSrc1->au8[31] + puSrc2->au8[31];
+}
+
+
+/*
+ * PADDSB / VPADDSB
+ */
+#define SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(a_iWord) \
+        ( (uint16_t)((a_iWord) + 0x80) <= (uint16_t)0xff  \
+          ? (uint8_t)(a_iWord) \
+          : (uint8_t)0x7f + (uint8_t)(((a_iWord) >> 15) & 1) ) /* 0x7f = INT8_MAX; 0x80 = INT8_MIN; source bit 15 = sign */
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddsb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au8[0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[0] + uSrc2.ai8[0]);
+    uDst.au8[1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[1] + uSrc2.ai8[1]);
+    uDst.au8[2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[2] + uSrc2.ai8[2]);
+    uDst.au8[3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[3] + uSrc2.ai8[3]);
+    uDst.au8[4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[4] + uSrc2.ai8[4]);
+    uDst.au8[5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[5] + uSrc2.ai8[5]);
+    uDst.au8[6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[6] + uSrc2.ai8[6]);
+    uDst.au8[7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[7] + uSrc2.ai8[7]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddsb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au8[0]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[0]  + puSrc->ai8[0]);
+    puDst->au8[1]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[1]  + puSrc->ai8[1]);
+    puDst->au8[2]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[2]  + puSrc->ai8[2]);
+    puDst->au8[3]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[3]  + puSrc->ai8[3]);
+    puDst->au8[4]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[4]  + puSrc->ai8[4]);
+    puDst->au8[5]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[5]  + puSrc->ai8[5]);
+    puDst->au8[6]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[6]  + puSrc->ai8[6]);
+    puDst->au8[7]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[7]  + puSrc->ai8[7]);
+    puDst->au8[8]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[8]  + puSrc->ai8[8]);
+    puDst->au8[9]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[9]  + puSrc->ai8[9]);
+    puDst->au8[10] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[10] + puSrc->ai8[10]);
+    puDst->au8[11] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[11] + puSrc->ai8[11]);
+    puDst->au8[12] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[12] + puSrc->ai8[12]);
+    puDst->au8[13] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[13] + puSrc->ai8[13]);
+    puDst->au8[14] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[14] + puSrc->ai8[14]);
+    puDst->au8[15] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[15] + puSrc->ai8[15]);
+}
+
+#endif
+
+
+/*
+ * PADDSB / VPADDSB
+ */
+#define SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(a_uWord) \
+        ( (uint16_t)(a_uWord) <= (uint16_t)0xff  \
+          ? (uint8_t)(a_uWord) \
+          : (uint8_t)0xff ) /* 0xff = UINT8_MAX */
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddusb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au8[0] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[0] + uSrc2.au8[0]);
+    uDst.au8[1] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[1] + uSrc2.au8[1]);
+    uDst.au8[2] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[2] + uSrc2.au8[2]);
+    uDst.au8[3] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[3] + uSrc2.au8[3]);
+    uDst.au8[4] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[4] + uSrc2.au8[4]);
+    uDst.au8[5] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[5] + uSrc2.au8[5]);
+    uDst.au8[6] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[6] + uSrc2.au8[6]);
+    uDst.au8[7] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[7] + uSrc2.au8[7]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddusb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au8[0]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[0]  + puSrc->au8[0]);
+    puDst->au8[1]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[1]  + puSrc->au8[1]);
+    puDst->au8[2]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[2]  + puSrc->au8[2]);
+    puDst->au8[3]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[3]  + puSrc->au8[3]);
+    puDst->au8[4]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[4]  + puSrc->au8[4]);
+    puDst->au8[5]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[5]  + puSrc->au8[5]);
+    puDst->au8[6]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[6]  + puSrc->au8[6]);
+    puDst->au8[7]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[7]  + puSrc->au8[7]);
+    puDst->au8[8]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[8]  + puSrc->au8[8]);
+    puDst->au8[9]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[9]  + puSrc->au8[9]);
+    puDst->au8[10] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[10] + puSrc->au8[10]);
+    puDst->au8[11] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[11] + puSrc->au8[11]);
+    puDst->au8[12] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[12] + puSrc->au8[12]);
+    puDst->au8[13] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[13] + puSrc->au8[13]);
+    puDst->au8[14] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[14] + puSrc->au8[14]);
+    puDst->au8[15] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au8[15] + puSrc->au8[15]);
+}
+
+#endif
+
+
+/*
+ * PADDW / VPADDW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au16[0] = uSrc1.au16[0] + uSrc2.au16[0];
+    uDst.au16[1] = uSrc1.au16[1] + uSrc2.au16[1];
+    uDst.au16[2] = uSrc1.au16[2] + uSrc2.au16[2];
+    uDst.au16[3] = uSrc1.au16[3] + uSrc2.au16[3];
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au16[0] = uSrc1.au16[0] + puSrc->au16[0];
+    puDst->au16[1] = uSrc1.au16[1] + puSrc->au16[1];
+    puDst->au16[2] = uSrc1.au16[2] + puSrc->au16[2];
+    puDst->au16[3] = uSrc1.au16[3] + puSrc->au16[3];
+    puDst->au16[4] = uSrc1.au16[4] + puSrc->au16[4];
+    puDst->au16[5] = uSrc1.au16[5] + puSrc->au16[5];
+    puDst->au16[6] = uSrc1.au16[6] + puSrc->au16[6];
+    puDst->au16[7] = uSrc1.au16[7] + puSrc->au16[7];
+}
+
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                       PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au16[0] = puSrc1->au16[0] + puSrc2->au16[0];
+    puDst->au16[1] = puSrc1->au16[1] + puSrc2->au16[1];
+    puDst->au16[2] = puSrc1->au16[2] + puSrc2->au16[2];
+    puDst->au16[3] = puSrc1->au16[3] + puSrc2->au16[3];
+    puDst->au16[4] = puSrc1->au16[4] + puSrc2->au16[4];
+    puDst->au16[5] = puSrc1->au16[5] + puSrc2->au16[5];
+    puDst->au16[6] = puSrc1->au16[6] + puSrc2->au16[6];
+    puDst->au16[7] = puSrc1->au16[7] + puSrc2->au16[7];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                       PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au16[0]  = puSrc1->au16[0]  + puSrc2->au16[0];
+    puDst->au16[1]  = puSrc1->au16[1]  + puSrc2->au16[1];
+    puDst->au16[2]  = puSrc1->au16[2]  + puSrc2->au16[2];
+    puDst->au16[3]  = puSrc1->au16[3]  + puSrc2->au16[3];
+    puDst->au16[4]  = puSrc1->au16[4]  + puSrc2->au16[4];
+    puDst->au16[5]  = puSrc1->au16[5]  + puSrc2->au16[5];
+    puDst->au16[6]  = puSrc1->au16[6]  + puSrc2->au16[6];
+    puDst->au16[7]  = puSrc1->au16[7]  + puSrc2->au16[7];
+    puDst->au16[8]  = puSrc1->au16[8]  + puSrc2->au16[8];
+    puDst->au16[9]  = puSrc1->au16[9]  + puSrc2->au16[9];
+    puDst->au16[10] = puSrc1->au16[10] + puSrc2->au16[10];
+    puDst->au16[11] = puSrc1->au16[11] + puSrc2->au16[11];
+    puDst->au16[12] = puSrc1->au16[12] + puSrc2->au16[12];
+    puDst->au16[13] = puSrc1->au16[13] + puSrc2->au16[13];
+    puDst->au16[14] = puSrc1->au16[14] + puSrc2->au16[14];
+    puDst->au16[15] = puSrc1->au16[15] + puSrc2->au16[15];
+}
+
+
+/*
+ * PADDSW / VPADDSW
+ */
+#define SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(a_iDword) \
+        ( (uint32_t)((a_iDword) + 0x8000) <= (uint16_t)0xffff  \
+          ? (uint16_t)(a_iDword) \
+          : (uint16_t)0x7fff + (uint16_t)(((a_iDword) >> 31) & 1) ) /* 0x7fff = INT16_MAX; 0x8000 = INT16_MIN; source bit 31 = sign */
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddsw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] + uSrc2.ai16[0]);
+    uDst.au16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[1] + uSrc2.ai16[1]);
+    uDst.au16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] + uSrc2.ai16[2]);
+    uDst.au16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[3] + uSrc2.ai16[3]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddsw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] + puSrc->ai16[0]);
+    puDst->au16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[1] + puSrc->ai16[1]);
+    puDst->au16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] + puSrc->ai16[2]);
+    puDst->au16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[3] + puSrc->ai16[3]);
+    puDst->au16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[4] + puSrc->ai16[4]);
+    puDst->au16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[5] + puSrc->ai16[5]);
+    puDst->au16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[6] + puSrc->ai16[6]);
+    puDst->au16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[7] + puSrc->ai16[7]);
+}
+
+#endif
+
+
+/*
+ * PADDUSW / VPADDUSW
+ */
+#define SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(a_uDword) \
+        ( (uint32_t)(a_uDword) <= (uint16_t)0xffff  \
+          ? (uint16_t)(a_uDword) \
+          : (uint16_t)0xffff ) /* 0xffff = UINT16_MAX */
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddusw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au16[0] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[0] + uSrc2.au16[0]);
+    uDst.au16[1] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[1] + uSrc2.au16[1]);
+    uDst.au16[2] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[2] + uSrc2.au16[2]);
+    uDst.au16[3] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[3] + uSrc2.au16[3]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddusw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au16[0] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[0] + puSrc->au16[0]);
+    puDst->au16[1] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[1] + puSrc->au16[1]);
+    puDst->au16[2] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[2] + puSrc->au16[2]);
+    puDst->au16[3] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[3] + puSrc->au16[3]);
+    puDst->au16[4] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[4] + puSrc->au16[4]);
+    puDst->au16[5] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[5] + puSrc->au16[5]);
+    puDst->au16[6] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[6] + puSrc->au16[6]);
+    puDst->au16[7] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au16[7] + puSrc->au16[7]);
+}
+
+#endif
+
+
+/*
+ * PADDD / VPADDD.
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddd_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au32[0] = uSrc1.au32[0] + uSrc2.au32[0];
+    uDst.au32[1] = uSrc1.au32[1] + uSrc2.au32[1];
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au32[0] = uSrc1.au32[0] + puSrc->au32[0];
+    puDst->au32[1] = uSrc1.au32[1] + puSrc->au32[1];
+    puDst->au32[2] = uSrc1.au32[2] + puSrc->au32[2];
+    puDst->au32[3] = uSrc1.au32[3] + puSrc->au32[3];
+}
+
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                       PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au32[0] = puSrc1->au32[0] + puSrc2->au32[0];
+    puDst->au32[1] = puSrc1->au32[1] + puSrc2->au32[1];
+    puDst->au32[2] = puSrc1->au32[2] + puSrc2->au32[2];
+    puDst->au32[3] = puSrc1->au32[3] + puSrc2->au32[3];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                       PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au32[0] = puSrc1->au32[0] + puSrc2->au32[0];
+    puDst->au32[1] = puSrc1->au32[1] + puSrc2->au32[1];
+    puDst->au32[2] = puSrc1->au32[2] + puSrc2->au32[2];
+    puDst->au32[3] = puSrc1->au32[3] + puSrc2->au32[3];
+    puDst->au32[4] = puSrc1->au32[4] + puSrc2->au32[4];
+    puDst->au32[5] = puSrc1->au32[5] + puSrc2->au32[5];
+    puDst->au32[6] = puSrc1->au32[6] + puSrc2->au32[6];
+    puDst->au32[7] = puSrc1->au32[7] + puSrc2->au32[7];
+}
+
+
+/*
+ * PADDQ / VPADDQ.
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddq_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    *puDst = *puDst + *puSrc;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_paddq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au64[0] = uSrc1.au64[0] + puSrc->au64[0];
+    puDst->au64[1] = uSrc1.au64[1] + puSrc->au64[1];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddq_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                       PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] + puSrc2->au64[0];
+    puDst->au64[1] = puSrc1->au64[1] + puSrc2->au64[1];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpaddq_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                       PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] + puSrc2->au64[0];
+    puDst->au64[1] = puSrc1->au64[1] + puSrc2->au64[1];
+    puDst->au64[2] = puSrc1->au64[2] + puSrc2->au64[2];
+    puDst->au64[3] = puSrc1->au64[3] + puSrc2->au64[3];
+}
+
+
+/*
+ * PSUBB / VPSUBB
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au8[0] = uSrc1.au8[0] - uSrc2.au8[0];
+    uDst.au8[1] = uSrc1.au8[1] - uSrc2.au8[1];
+    uDst.au8[2] = uSrc1.au8[2] - uSrc2.au8[2];
+    uDst.au8[3] = uSrc1.au8[3] - uSrc2.au8[3];
+    uDst.au8[4] = uSrc1.au8[4] - uSrc2.au8[4];
+    uDst.au8[5] = uSrc1.au8[5] - uSrc2.au8[5];
+    uDst.au8[6] = uSrc1.au8[6] - uSrc2.au8[6];
+    uDst.au8[7] = uSrc1.au8[7] - uSrc2.au8[7];
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au8[0]  = uSrc1.au8[0]  - puSrc->au8[0];
+    puDst->au8[1]  = uSrc1.au8[1]  - puSrc->au8[1];
+    puDst->au8[2]  = uSrc1.au8[2]  - puSrc->au8[2];
+    puDst->au8[3]  = uSrc1.au8[3]  - puSrc->au8[3];
+    puDst->au8[4]  = uSrc1.au8[4]  - puSrc->au8[4];
+    puDst->au8[5]  = uSrc1.au8[5]  - puSrc->au8[5];
+    puDst->au8[6]  = uSrc1.au8[6]  - puSrc->au8[6];
+    puDst->au8[7]  = uSrc1.au8[7]  - puSrc->au8[7];
+    puDst->au8[8]  = uSrc1.au8[8]  - puSrc->au8[8];
+    puDst->au8[9]  = uSrc1.au8[9]  - puSrc->au8[9];
+    puDst->au8[10] = uSrc1.au8[10] - puSrc->au8[10];
+    puDst->au8[11] = uSrc1.au8[11] - puSrc->au8[11];
+    puDst->au8[12] = uSrc1.au8[12] - puSrc->au8[12];
+    puDst->au8[13] = uSrc1.au8[13] - puSrc->au8[13];
+    puDst->au8[14] = uSrc1.au8[14] - puSrc->au8[14];
+    puDst->au8[15] = uSrc1.au8[15] - puSrc->au8[15];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                       PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au8[0]  = puSrc1->au8[0]  - puSrc2->au8[0];
+    puDst->au8[1]  = puSrc1->au8[1]  - puSrc2->au8[1];
+    puDst->au8[2]  = puSrc1->au8[2]  - puSrc2->au8[2];
+    puDst->au8[3]  = puSrc1->au8[3]  - puSrc2->au8[3];
+    puDst->au8[4]  = puSrc1->au8[4]  - puSrc2->au8[4];
+    puDst->au8[5]  = puSrc1->au8[5]  - puSrc2->au8[5];
+    puDst->au8[6]  = puSrc1->au8[6]  - puSrc2->au8[6];
+    puDst->au8[7]  = puSrc1->au8[7]  - puSrc2->au8[7];
+    puDst->au8[8]  = puSrc1->au8[8]  - puSrc2->au8[8];
+    puDst->au8[9]  = puSrc1->au8[9]  - puSrc2->au8[9];
+    puDst->au8[10] = puSrc1->au8[10] - puSrc2->au8[10];
+    puDst->au8[11] = puSrc1->au8[11] - puSrc2->au8[11];
+    puDst->au8[12] = puSrc1->au8[12] - puSrc2->au8[12];
+    puDst->au8[13] = puSrc1->au8[13] - puSrc2->au8[13];
+    puDst->au8[14] = puSrc1->au8[14] - puSrc2->au8[14];
+    puDst->au8[15] = puSrc1->au8[15] - puSrc2->au8[15];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                       PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au8[0]  = puSrc1->au8[0]  - puSrc2->au8[0];
+    puDst->au8[1]  = puSrc1->au8[1]  - puSrc2->au8[1];
+    puDst->au8[2]  = puSrc1->au8[2]  - puSrc2->au8[2];
+    puDst->au8[3]  = puSrc1->au8[3]  - puSrc2->au8[3];
+    puDst->au8[4]  = puSrc1->au8[4]  - puSrc2->au8[4];
+    puDst->au8[5]  = puSrc1->au8[5]  - puSrc2->au8[5];
+    puDst->au8[6]  = puSrc1->au8[6]  - puSrc2->au8[6];
+    puDst->au8[7]  = puSrc1->au8[7]  - puSrc2->au8[7];
+    puDst->au8[8]  = puSrc1->au8[8]  - puSrc2->au8[8];
+    puDst->au8[9]  = puSrc1->au8[9]  - puSrc2->au8[9];
+    puDst->au8[10] = puSrc1->au8[10] - puSrc2->au8[10];
+    puDst->au8[11] = puSrc1->au8[11] - puSrc2->au8[11];
+    puDst->au8[12] = puSrc1->au8[12] - puSrc2->au8[12];
+    puDst->au8[13] = puSrc1->au8[13] - puSrc2->au8[13];
+    puDst->au8[14] = puSrc1->au8[14] - puSrc2->au8[14];
+    puDst->au8[15] = puSrc1->au8[15] - puSrc2->au8[15];
+    puDst->au8[16] = puSrc1->au8[16] - puSrc2->au8[16];
+    puDst->au8[17] = puSrc1->au8[17] - puSrc2->au8[17];
+    puDst->au8[18] = puSrc1->au8[18] - puSrc2->au8[18];
+    puDst->au8[19] = puSrc1->au8[19] - puSrc2->au8[19];
+    puDst->au8[20] = puSrc1->au8[20] - puSrc2->au8[20];
+    puDst->au8[21] = puSrc1->au8[21] - puSrc2->au8[21];
+    puDst->au8[22] = puSrc1->au8[22] - puSrc2->au8[22];
+    puDst->au8[23] = puSrc1->au8[23] - puSrc2->au8[23];
+    puDst->au8[24] = puSrc1->au8[24] - puSrc2->au8[24];
+    puDst->au8[25] = puSrc1->au8[25] - puSrc2->au8[25];
+    puDst->au8[26] = puSrc1->au8[26] - puSrc2->au8[26];
+    puDst->au8[27] = puSrc1->au8[27] - puSrc2->au8[27];
+    puDst->au8[28] = puSrc1->au8[28] - puSrc2->au8[28];
+    puDst->au8[29] = puSrc1->au8[29] - puSrc2->au8[29];
+    puDst->au8[30] = puSrc1->au8[30] - puSrc2->au8[30];
+    puDst->au8[31] = puSrc1->au8[31] - puSrc2->au8[31];
+}
+
+
+/*
+ * PSUBSB / VSUBSB
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubsb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au8[0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[0] - uSrc2.ai8[0]);
+    uDst.au8[1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[1] - uSrc2.ai8[1]);
+    uDst.au8[2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[2] - uSrc2.ai8[2]);
+    uDst.au8[3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[3] - uSrc2.ai8[3]);
+    uDst.au8[4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[4] - uSrc2.ai8[4]);
+    uDst.au8[5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[5] - uSrc2.ai8[5]);
+    uDst.au8[6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[6] - uSrc2.ai8[6]);
+    uDst.au8[7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[7] - uSrc2.ai8[7]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubsb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au8[0]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[0]  - puSrc->ai8[0]);
+    puDst->au8[1]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[1]  - puSrc->ai8[1]);
+    puDst->au8[2]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[2]  - puSrc->ai8[2]);
+    puDst->au8[3]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[3]  - puSrc->ai8[3]);
+    puDst->au8[4]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[4]  - puSrc->ai8[4]);
+    puDst->au8[5]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[5]  - puSrc->ai8[5]);
+    puDst->au8[6]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[6]  - puSrc->ai8[6]);
+    puDst->au8[7]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[7]  - puSrc->ai8[7]);
+    puDst->au8[8]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[8]  - puSrc->ai8[8]);
+    puDst->au8[9]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[9]  - puSrc->ai8[9]);
+    puDst->au8[10] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[10] - puSrc->ai8[10]);
+    puDst->au8[11] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[11] - puSrc->ai8[11]);
+    puDst->au8[12] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[12] - puSrc->ai8[12]);
+    puDst->au8[13] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[13] - puSrc->ai8[13]);
+    puDst->au8[14] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[14] - puSrc->ai8[14]);
+    puDst->au8[15] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.ai8[15] - puSrc->ai8[15]);
+}
+
+#endif
+
+
+/*
+ * PADDSB / VPADDSB
+ */
+#define SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(a_uWord) \
+        ( (uint16_t)(a_uWord) <= (uint16_t)0xff  \
+          ? (uint8_t)(a_uWord) \
+          : (uint8_t)0 )
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubusb_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au8[0] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[0] - uSrc2.au8[0]);
+    uDst.au8[1] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[1] - uSrc2.au8[1]);
+    uDst.au8[2] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[2] - uSrc2.au8[2]);
+    uDst.au8[3] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[3] - uSrc2.au8[3]);
+    uDst.au8[4] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[4] - uSrc2.au8[4]);
+    uDst.au8[5] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[5] - uSrc2.au8[5]);
+    uDst.au8[6] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[6] - uSrc2.au8[6]);
+    uDst.au8[7] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[7] - uSrc2.au8[7]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubusb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au8[0]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[0]  - puSrc->au8[0]);
+    puDst->au8[1]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[1]  - puSrc->au8[1]);
+    puDst->au8[2]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[2]  - puSrc->au8[2]);
+    puDst->au8[3]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[3]  - puSrc->au8[3]);
+    puDst->au8[4]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[4]  - puSrc->au8[4]);
+    puDst->au8[5]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[5]  - puSrc->au8[5]);
+    puDst->au8[6]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[6]  - puSrc->au8[6]);
+    puDst->au8[7]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[7]  - puSrc->au8[7]);
+    puDst->au8[8]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[8]  - puSrc->au8[8]);
+    puDst->au8[9]  = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[9]  - puSrc->au8[9]);
+    puDst->au8[10] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[10] - puSrc->au8[10]);
+    puDst->au8[11] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[11] - puSrc->au8[11]);
+    puDst->au8[12] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[12] - puSrc->au8[12]);
+    puDst->au8[13] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[13] - puSrc->au8[13]);
+    puDst->au8[14] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[14] - puSrc->au8[14]);
+    puDst->au8[15] = SATURATED_UNSIGNED_WORD_TO_UNSIGNED_BYTE_SUB(uSrc1.au8[15] - puSrc->au8[15]);
+}
+
+#endif
+
+
+/*
+ * PSUBW / VPSUBW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au16[0] = uSrc1.au16[0] - uSrc2.au16[0];
+    uDst.au16[1] = uSrc1.au16[1] - uSrc2.au16[1];
+    uDst.au16[2] = uSrc1.au16[2] - uSrc2.au16[2];
+    uDst.au16[3] = uSrc1.au16[3] - uSrc2.au16[3];
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au16[0] = uSrc1.au16[0] - puSrc->au16[0];
+    puDst->au16[1] = uSrc1.au16[1] - puSrc->au16[1];
+    puDst->au16[2] = uSrc1.au16[2] - puSrc->au16[2];
+    puDst->au16[3] = uSrc1.au16[3] - puSrc->au16[3];
+    puDst->au16[4] = uSrc1.au16[4] - puSrc->au16[4];
+    puDst->au16[5] = uSrc1.au16[5] - puSrc->au16[5];
+    puDst->au16[6] = uSrc1.au16[6] - puSrc->au16[6];
+    puDst->au16[7] = uSrc1.au16[7] - puSrc->au16[7];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                       PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au16[0] = puSrc1->au16[0] - puSrc2->au16[0];
+    puDst->au16[1] = puSrc1->au16[1] - puSrc2->au16[1];
+    puDst->au16[2] = puSrc1->au16[2] - puSrc2->au16[2];
+    puDst->au16[3] = puSrc1->au16[3] - puSrc2->au16[3];
+    puDst->au16[4] = puSrc1->au16[4] - puSrc2->au16[4];
+    puDst->au16[5] = puSrc1->au16[5] - puSrc2->au16[5];
+    puDst->au16[6] = puSrc1->au16[6] - puSrc2->au16[6];
+    puDst->au16[7] = puSrc1->au16[7] - puSrc2->au16[7];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                       PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au16[0]  = puSrc1->au16[0]  - puSrc2->au16[0];
+    puDst->au16[1]  = puSrc1->au16[1]  - puSrc2->au16[1];
+    puDst->au16[2]  = puSrc1->au16[2]  - puSrc2->au16[2];
+    puDst->au16[3]  = puSrc1->au16[3]  - puSrc2->au16[3];
+    puDst->au16[4]  = puSrc1->au16[4]  - puSrc2->au16[4];
+    puDst->au16[5]  = puSrc1->au16[5]  - puSrc2->au16[5];
+    puDst->au16[6]  = puSrc1->au16[6]  - puSrc2->au16[6];
+    puDst->au16[7]  = puSrc1->au16[7]  - puSrc2->au16[7];
+    puDst->au16[8]  = puSrc1->au16[8]  - puSrc2->au16[8];
+    puDst->au16[9]  = puSrc1->au16[9]  - puSrc2->au16[9];
+    puDst->au16[10] = puSrc1->au16[10] - puSrc2->au16[10];
+    puDst->au16[11] = puSrc1->au16[11] - puSrc2->au16[11];
+    puDst->au16[12] = puSrc1->au16[12] - puSrc2->au16[12];
+    puDst->au16[13] = puSrc1->au16[13] - puSrc2->au16[13];
+    puDst->au16[14] = puSrc1->au16[14] - puSrc2->au16[14];
+    puDst->au16[15] = puSrc1->au16[15] - puSrc2->au16[15];
+}
+
+
+/*
+ * PSUBSW / VPSUBSW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubsw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] - uSrc2.ai16[0]);
+    uDst.au16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[1] - uSrc2.ai16[1]);
+    uDst.au16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] - uSrc2.ai16[2]);
+    uDst.au16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[3] - uSrc2.ai16[3]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubsw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] - puSrc->ai16[0]);
+    puDst->au16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[1] - puSrc->ai16[1]);
+    puDst->au16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] - puSrc->ai16[2]);
+    puDst->au16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[3] - puSrc->ai16[3]);
+    puDst->au16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[4] - puSrc->ai16[4]);
+    puDst->au16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[5] - puSrc->ai16[5]);
+    puDst->au16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[6] - puSrc->ai16[6]);
+    puDst->au16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[7] - puSrc->ai16[7]);
+}
+
+#endif
+
+
+/*
+ * PSUBUSW / VPSUBUSW
+ */
+#define SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(a_uDword) \
+        ( (uint32_t)(a_uDword) <= (uint16_t)0xffff  \
+          ? (uint16_t)(a_uDword) \
+          : (uint16_t)0 )
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubusw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au16[0] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[0] - uSrc2.au16[0]);
+    uDst.au16[1] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[1] - uSrc2.au16[1]);
+    uDst.au16[2] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[2] - uSrc2.au16[2]);
+    uDst.au16[3] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[3] - uSrc2.au16[3]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubusw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au16[0] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[0] - puSrc->au16[0]);
+    puDst->au16[1] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[1] - puSrc->au16[1]);
+    puDst->au16[2] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[2] - puSrc->au16[2]);
+    puDst->au16[3] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[3] - puSrc->au16[3]);
+    puDst->au16[4] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[4] - puSrc->au16[4]);
+    puDst->au16[5] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[5] - puSrc->au16[5]);
+    puDst->au16[6] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[6] - puSrc->au16[6]);
+    puDst->au16[7] = SATURATED_UNSIGNED_DWORD_TO_UNSIGNED_WORD_SUB(uSrc1.au16[7] - puSrc->au16[7]);
+}
+
+#endif
+
+
+/*
+ * PSUBD / VPSUBD.
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubd_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au32[0] = uSrc1.au32[0] - uSrc2.au32[0];
+    uDst.au32[1] = uSrc1.au32[1] - uSrc2.au32[1];
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au32[0] = uSrc1.au32[0] - puSrc->au32[0];
+    puDst->au32[1] = uSrc1.au32[1] - puSrc->au32[1];
+    puDst->au32[2] = uSrc1.au32[2] - puSrc->au32[2];
+    puDst->au32[3] = uSrc1.au32[3] - puSrc->au32[3];
+}
+
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                       PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au32[0] = puSrc1->au32[0] - puSrc2->au32[0];
+    puDst->au32[1] = puSrc1->au32[1] - puSrc2->au32[1];
+    puDst->au32[2] = puSrc1->au32[2] - puSrc2->au32[2];
+    puDst->au32[3] = puSrc1->au32[3] - puSrc2->au32[3];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                       PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au32[0] = puSrc1->au32[0] - puSrc2->au32[0];
+    puDst->au32[1] = puSrc1->au32[1] - puSrc2->au32[1];
+    puDst->au32[2] = puSrc1->au32[2] - puSrc2->au32[2];
+    puDst->au32[3] = puSrc1->au32[3] - puSrc2->au32[3];
+    puDst->au32[4] = puSrc1->au32[4] - puSrc2->au32[4];
+    puDst->au32[5] = puSrc1->au32[5] - puSrc2->au32[5];
+    puDst->au32[6] = puSrc1->au32[6] - puSrc2->au32[6];
+    puDst->au32[7] = puSrc1->au32[7] - puSrc2->au32[7];
+}
+
+
+/*
+ * PSUBQ / VPSUBQ.
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubq_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    *puDst = *puDst - *puSrc;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psubq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au64[0] = uSrc1.au64[0] - puSrc->au64[0];
+    puDst->au64[1] = uSrc1.au64[1] - puSrc->au64[1];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubq_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                       PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] - puSrc2->au64[0];
+    puDst->au64[1] = puSrc1->au64[1] - puSrc2->au64[1];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsubq_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                       PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RT_NOREF(pExtState);
+    puDst->au64[0] = puSrc1->au64[0] - puSrc2->au64[0];
+    puDst->au64[1] = puSrc1->au64[1] - puSrc2->au64[1];
+    puDst->au64[2] = puSrc1->au64[2] - puSrc2->au64[2];
+    puDst->au64[3] = puSrc1->au64[3] - puSrc2->au64[3];
+}
+
+
+
+/*
+ * PMULLW / VPMULLW / PMULLD / VPMULLD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmullw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.ai16[0] = uSrc1.ai16[0] * uSrc2.ai16[0];
+    uDst.ai16[1] = uSrc1.ai16[1] * uSrc2.ai16[1];
+    uDst.ai16[2] = uSrc1.ai16[2] * uSrc2.ai16[2];
+    uDst.ai16[3] = uSrc1.ai16[3] * uSrc2.ai16[3];
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmullw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->ai16[0] = uSrc1.ai16[0] * puSrc->ai16[0];
+    puDst->ai16[1] = uSrc1.ai16[1] * puSrc->ai16[1];
+    puDst->ai16[2] = uSrc1.ai16[2] * puSrc->ai16[2];
+    puDst->ai16[3] = uSrc1.ai16[3] * puSrc->ai16[3];
+    puDst->ai16[4] = uSrc1.ai16[4] * puSrc->ai16[4];
+    puDst->ai16[5] = uSrc1.ai16[5] * puSrc->ai16[5];
+    puDst->ai16[6] = uSrc1.ai16[6] * puSrc->ai16[6];
+    puDst->ai16[7] = uSrc1.ai16[7] * puSrc->ai16[7];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmulld_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai32[0] = uSrc1.ai32[0] * puSrc->ai32[0];
+    puDst->ai32[1] = uSrc1.ai32[1] * puSrc->ai32[1];
+    puDst->ai32[2] = uSrc1.ai32[2] * puSrc->ai32[2];
+    puDst->ai32[3] = uSrc1.ai32[3] * puSrc->ai32[3];
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmullw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->ai16[0] = puSrc1->ai16[0] * puSrc2->ai16[0];
+    puDst->ai16[1] = puSrc1->ai16[1] * puSrc2->ai16[1];
+    puDst->ai16[2] = puSrc1->ai16[2] * puSrc2->ai16[2];
+    puDst->ai16[3] = puSrc1->ai16[3] * puSrc2->ai16[3];
+    puDst->ai16[4] = puSrc1->ai16[4] * puSrc2->ai16[4];
+    puDst->ai16[5] = puSrc1->ai16[5] * puSrc2->ai16[5];
+    puDst->ai16[6] = puSrc1->ai16[6] * puSrc2->ai16[6];
+    puDst->ai16[7] = puSrc1->ai16[7] * puSrc2->ai16[7];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmullw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->ai16[ 0] = puSrc1->ai16[ 0] * puSrc2->ai16[ 0];
+    puDst->ai16[ 1] = puSrc1->ai16[ 1] * puSrc2->ai16[ 1];
+    puDst->ai16[ 2] = puSrc1->ai16[ 2] * puSrc2->ai16[ 2];
+    puDst->ai16[ 3] = puSrc1->ai16[ 3] * puSrc2->ai16[ 3];
+    puDst->ai16[ 4] = puSrc1->ai16[ 4] * puSrc2->ai16[ 4];
+    puDst->ai16[ 5] = puSrc1->ai16[ 5] * puSrc2->ai16[ 5];
+    puDst->ai16[ 6] = puSrc1->ai16[ 6] * puSrc2->ai16[ 6];
+    puDst->ai16[ 7] = puSrc1->ai16[ 7] * puSrc2->ai16[ 7];
+    puDst->ai16[ 8] = puSrc1->ai16[ 8] * puSrc2->ai16[ 8];
+    puDst->ai16[ 9] = puSrc1->ai16[ 9] * puSrc2->ai16[ 9];
+    puDst->ai16[10] = puSrc1->ai16[10] * puSrc2->ai16[10];
+    puDst->ai16[11] = puSrc1->ai16[11] * puSrc2->ai16[11];
+    puDst->ai16[12] = puSrc1->ai16[12] * puSrc2->ai16[12];
+    puDst->ai16[13] = puSrc1->ai16[13] * puSrc2->ai16[13];
+    puDst->ai16[14] = puSrc1->ai16[14] * puSrc2->ai16[14];
+    puDst->ai16[15] = puSrc1->ai16[15] * puSrc2->ai16[15];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulld_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->ai32[0] = puSrc1->ai32[0] * puSrc2->ai32[0];
+    puDst->ai32[1] = puSrc1->ai32[1] * puSrc2->ai32[1];
+    puDst->ai32[2] = puSrc1->ai32[2] * puSrc2->ai32[2];
+    puDst->ai32[3] = puSrc1->ai32[3] * puSrc2->ai32[3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulld_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->ai32[0] = puSrc1->ai32[0] * puSrc2->ai32[0];
+    puDst->ai32[1] = puSrc1->ai32[1] * puSrc2->ai32[1];
+    puDst->ai32[2] = puSrc1->ai32[2] * puSrc2->ai32[2];
+    puDst->ai32[3] = puSrc1->ai32[3] * puSrc2->ai32[3];
+    puDst->ai32[4] = puSrc1->ai32[4] * puSrc2->ai32[4];
+    puDst->ai32[5] = puSrc1->ai32[5] * puSrc2->ai32[5];
+    puDst->ai32[6] = puSrc1->ai32[6] * puSrc2->ai32[6];
+    puDst->ai32[7] = puSrc1->ai32[7] * puSrc2->ai32[7];
+}
+
+
+/*
+ * PMULHW / VPMULHW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.ai16[0] = RT_HIWORD(uSrc1.ai16[0] * uSrc2.ai16[0]);
+    uDst.ai16[1] = RT_HIWORD(uSrc1.ai16[1] * uSrc2.ai16[1]);
+    uDst.ai16[2] = RT_HIWORD(uSrc1.ai16[2] * uSrc2.ai16[2]);
+    uDst.ai16[3] = RT_HIWORD(uSrc1.ai16[3] * uSrc2.ai16[3]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RT_NOREF(pFpuState);
+    RTUINT128U uSrc1 = *puDst;
+    puDst->ai16[0] = RT_HIWORD(uSrc1.ai16[0] * puSrc->ai16[0]);
+    puDst->ai16[1] = RT_HIWORD(uSrc1.ai16[1] * puSrc->ai16[1]);
+    puDst->ai16[2] = RT_HIWORD(uSrc1.ai16[2] * puSrc->ai16[2]);
+    puDst->ai16[3] = RT_HIWORD(uSrc1.ai16[3] * puSrc->ai16[3]);
+    puDst->ai16[4] = RT_HIWORD(uSrc1.ai16[4] * puSrc->ai16[4]);
+    puDst->ai16[5] = RT_HIWORD(uSrc1.ai16[5] * puSrc->ai16[5]);
+    puDst->ai16[6] = RT_HIWORD(uSrc1.ai16[6] * puSrc->ai16[6]);
+    puDst->ai16[7] = RT_HIWORD(uSrc1.ai16[7] * puSrc->ai16[7]);
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->ai16[0] = RT_HIWORD(puSrc1->ai16[0] * puSrc2->ai16[0]);
+    puDst->ai16[1] = RT_HIWORD(puSrc1->ai16[1] * puSrc2->ai16[1]);
+    puDst->ai16[2] = RT_HIWORD(puSrc1->ai16[2] * puSrc2->ai16[2]);
+    puDst->ai16[3] = RT_HIWORD(puSrc1->ai16[3] * puSrc2->ai16[3]);
+    puDst->ai16[4] = RT_HIWORD(puSrc1->ai16[4] * puSrc2->ai16[4]);
+    puDst->ai16[5] = RT_HIWORD(puSrc1->ai16[5] * puSrc2->ai16[5]);
+    puDst->ai16[6] = RT_HIWORD(puSrc1->ai16[6] * puSrc2->ai16[6]);
+    puDst->ai16[7] = RT_HIWORD(puSrc1->ai16[7] * puSrc2->ai16[7]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->ai16[ 0] = RT_HIWORD(puSrc1->ai16[ 0] * puSrc2->ai16[ 0]);
+    puDst->ai16[ 1] = RT_HIWORD(puSrc1->ai16[ 1] * puSrc2->ai16[ 1]);
+    puDst->ai16[ 2] = RT_HIWORD(puSrc1->ai16[ 2] * puSrc2->ai16[ 2]);
+    puDst->ai16[ 3] = RT_HIWORD(puSrc1->ai16[ 3] * puSrc2->ai16[ 3]);
+    puDst->ai16[ 4] = RT_HIWORD(puSrc1->ai16[ 4] * puSrc2->ai16[ 4]);
+    puDst->ai16[ 5] = RT_HIWORD(puSrc1->ai16[ 5] * puSrc2->ai16[ 5]);
+    puDst->ai16[ 6] = RT_HIWORD(puSrc1->ai16[ 6] * puSrc2->ai16[ 6]);
+    puDst->ai16[ 7] = RT_HIWORD(puSrc1->ai16[ 7] * puSrc2->ai16[ 7]);
+    puDst->ai16[ 8] = RT_HIWORD(puSrc1->ai16[ 8] * puSrc2->ai16[ 8]);
+    puDst->ai16[ 9] = RT_HIWORD(puSrc1->ai16[ 9] * puSrc2->ai16[ 9]);
+    puDst->ai16[10] = RT_HIWORD(puSrc1->ai16[10] * puSrc2->ai16[10]);
+    puDst->ai16[11] = RT_HIWORD(puSrc1->ai16[11] * puSrc2->ai16[11]);
+    puDst->ai16[12] = RT_HIWORD(puSrc1->ai16[12] * puSrc2->ai16[12]);
+    puDst->ai16[13] = RT_HIWORD(puSrc1->ai16[13] * puSrc2->ai16[13]);
+    puDst->ai16[14] = RT_HIWORD(puSrc1->ai16[14] * puSrc2->ai16[14]);
+    puDst->ai16[15] = RT_HIWORD(puSrc1->ai16[15] * puSrc2->ai16[15]);
+}
+
+
+/*
+ * PMULHUW / VPMULHUW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhuw_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uDst.au16[0] = RT_HIWORD(uSrc1.au16[0] * uSrc2.au16[0]);
+    uDst.au16[1] = RT_HIWORD(uSrc1.au16[1] * uSrc2.au16[1]);
+    uDst.au16[2] = RT_HIWORD(uSrc1.au16[2] * uSrc2.au16[2]);
+    uDst.au16[3] = RT_HIWORD(uSrc1.au16[3] * uSrc2.au16[3]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhuw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+    puDst->au16[0] = RT_HIWORD(uSrc1.au16[0] * puSrc->au16[0]);
+    puDst->au16[1] = RT_HIWORD(uSrc1.au16[1] * puSrc->au16[1]);
+    puDst->au16[2] = RT_HIWORD(uSrc1.au16[2] * puSrc->au16[2]);
+    puDst->au16[3] = RT_HIWORD(uSrc1.au16[3] * puSrc->au16[3]);
+    puDst->au16[4] = RT_HIWORD(uSrc1.au16[4] * puSrc->au16[4]);
+    puDst->au16[5] = RT_HIWORD(uSrc1.au16[5] * puSrc->au16[5]);
+    puDst->au16[6] = RT_HIWORD(uSrc1.au16[6] * puSrc->au16[6]);
+    puDst->au16[7] = RT_HIWORD(uSrc1.au16[7] * puSrc->au16[7]);
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhuw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->au16[0] = RT_HIWORD(puSrc1->au16[0] * puSrc2->au16[0]);
+    puDst->au16[1] = RT_HIWORD(puSrc1->au16[1] * puSrc2->au16[1]);
+    puDst->au16[2] = RT_HIWORD(puSrc1->au16[2] * puSrc2->au16[2]);
+    puDst->au16[3] = RT_HIWORD(puSrc1->au16[3] * puSrc2->au16[3]);
+    puDst->au16[4] = RT_HIWORD(puSrc1->au16[4] * puSrc2->au16[4]);
+    puDst->au16[5] = RT_HIWORD(puSrc1->au16[5] * puSrc2->au16[5]);
+    puDst->au16[6] = RT_HIWORD(puSrc1->au16[6] * puSrc2->au16[6]);
+    puDst->au16[7] = RT_HIWORD(puSrc1->au16[7] * puSrc2->au16[7]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhuw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->au16[ 0] = RT_HIWORD(puSrc1->au16[ 0] * puSrc2->au16[ 0]);
+    puDst->au16[ 1] = RT_HIWORD(puSrc1->au16[ 1] * puSrc2->au16[ 1]);
+    puDst->au16[ 2] = RT_HIWORD(puSrc1->au16[ 2] * puSrc2->au16[ 2]);
+    puDst->au16[ 3] = RT_HIWORD(puSrc1->au16[ 3] * puSrc2->au16[ 3]);
+    puDst->au16[ 4] = RT_HIWORD(puSrc1->au16[ 4] * puSrc2->au16[ 4]);
+    puDst->au16[ 5] = RT_HIWORD(puSrc1->au16[ 5] * puSrc2->au16[ 5]);
+    puDst->au16[ 6] = RT_HIWORD(puSrc1->au16[ 6] * puSrc2->au16[ 6]);
+    puDst->au16[ 7] = RT_HIWORD(puSrc1->au16[ 7] * puSrc2->au16[ 7]);
+    puDst->au16[ 8] = RT_HIWORD(puSrc1->au16[ 8] * puSrc2->au16[ 8]);
+    puDst->au16[ 9] = RT_HIWORD(puSrc1->au16[ 9] * puSrc2->au16[ 9]);
+    puDst->au16[10] = RT_HIWORD(puSrc1->au16[10] * puSrc2->au16[10]);
+    puDst->au16[11] = RT_HIWORD(puSrc1->au16[11] * puSrc2->au16[11]);
+    puDst->au16[12] = RT_HIWORD(puSrc1->au16[12] * puSrc2->au16[12]);
+    puDst->au16[13] = RT_HIWORD(puSrc1->au16[13] * puSrc2->au16[13]);
+    puDst->au16[14] = RT_HIWORD(puSrc1->au16[14] * puSrc2->au16[14]);
+    puDst->au16[15] = RT_HIWORD(puSrc1->au16[15] * puSrc2->au16[15]);
+}
+
+
+/*
+ * PSRLW / VPSRLW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrlw_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    if (uSrc2.au64[0] <= 15)
+    {
+        uDst.au16[0] = uSrc1.au16[0] >> uSrc2.au8[0];
+        uDst.au16[1] = uSrc1.au16[1] >> uSrc2.au8[0];
+        uDst.au16[2] = uSrc1.au16[2] >> uSrc2.au8[0];
+        uDst.au16[3] = uSrc1.au16[3] >> uSrc2.au8[0];
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrlw_imm_u64,(uint64_t *puDst, uint8_t uShift))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uDst;
+
+    if (uShift <= 15)
+    {
+        uDst.au16[0] = uSrc1.au16[0] >> uShift;
+        uDst.au16[1] = uSrc1.au16[1] >> uShift;
+        uDst.au16[2] = uSrc1.au16[2] >> uShift;
+        uDst.au16[3] = uSrc1.au16[3] >> uShift;
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrlw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (puSrc->au64[0] <= 15)
+    {
+        puDst->au16[0] = uSrc1.au16[0] >> puSrc->au8[0];
+        puDst->au16[1] = uSrc1.au16[1] >> puSrc->au8[0];
+        puDst->au16[2] = uSrc1.au16[2] >> puSrc->au8[0];
+        puDst->au16[3] = uSrc1.au16[3] >> puSrc->au8[0];
+        puDst->au16[4] = uSrc1.au16[4] >> puSrc->au8[0];
+        puDst->au16[5] = uSrc1.au16[5] >> puSrc->au8[0];
+        puDst->au16[6] = uSrc1.au16[6] >> puSrc->au8[0];
+        puDst->au16[7] = uSrc1.au16[7] >> puSrc->au8[0];
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrlw_imm_u128,(PRTUINT128U puDst, uint8_t uShift))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (uShift <= 15)
+    {
+        puDst->au16[0] = uSrc1.au16[0] >> uShift;
+        puDst->au16[1] = uSrc1.au16[1] >> uShift;
+        puDst->au16[2] = uSrc1.au16[2] >> uShift;
+        puDst->au16[3] = uSrc1.au16[3] >> uShift;
+        puDst->au16[4] = uSrc1.au16[4] >> uShift;
+        puDst->au16[5] = uSrc1.au16[5] >> uShift;
+        puDst->au16[6] = uSrc1.au16[6] >> uShift;
+        puDst->au16[7] = uSrc1.au16[7] >> uShift;
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+#endif
+
+
+/*
+ * PSRAW / VPSRAW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psraw_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    if (uSrc2.au64[0] <= 15)
+    {
+        uDst.ai16[0] = uSrc1.ai16[0] >> uSrc2.au8[0];
+        uDst.ai16[1] = uSrc1.ai16[1] >> uSrc2.au8[0];
+        uDst.ai16[2] = uSrc1.ai16[2] >> uSrc2.au8[0];
+        uDst.ai16[3] = uSrc1.ai16[3] >> uSrc2.au8[0];
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psraw_imm_u64,(uint64_t *puDst, uint8_t uShift))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uDst;
+
+    if (uShift <= 15)
+    {
+        uDst.ai16[0] = uSrc1.ai16[0] >> uShift;
+        uDst.ai16[1] = uSrc1.ai16[1] >> uShift;
+        uDst.ai16[2] = uSrc1.ai16[2] >> uShift;
+        uDst.ai16[3] = uSrc1.ai16[3] >> uShift;
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psraw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (puSrc->au64[0] <= 15)
+    {
+        puDst->ai16[0] = uSrc1.ai16[0] >> puSrc->au8[0];
+        puDst->ai16[1] = uSrc1.ai16[1] >> puSrc->au8[0];
+        puDst->ai16[2] = uSrc1.ai16[2] >> puSrc->au8[0];
+        puDst->ai16[3] = uSrc1.ai16[3] >> puSrc->au8[0];
+        puDst->ai16[4] = uSrc1.ai16[4] >> puSrc->au8[0];
+        puDst->ai16[5] = uSrc1.ai16[5] >> puSrc->au8[0];
+        puDst->ai16[6] = uSrc1.ai16[6] >> puSrc->au8[0];
+        puDst->ai16[7] = uSrc1.ai16[7] >> puSrc->au8[0];
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psraw_imm_u128,(PRTUINT128U puDst, uint8_t uShift))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (uShift <= 15)
+    {
+        puDst->ai16[0] = uSrc1.ai16[0] >> uShift;
+        puDst->ai16[1] = uSrc1.ai16[1] >> uShift;
+        puDst->ai16[2] = uSrc1.ai16[2] >> uShift;
+        puDst->ai16[3] = uSrc1.ai16[3] >> uShift;
+        puDst->ai16[4] = uSrc1.ai16[4] >> uShift;
+        puDst->ai16[5] = uSrc1.ai16[5] >> uShift;
+        puDst->ai16[6] = uSrc1.ai16[6] >> uShift;
+        puDst->ai16[7] = uSrc1.ai16[7] >> uShift;
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+#endif
+
+
+/*
+ * PSLLW / VPSLLW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psllw_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    if (uSrc2.au64[0] <= 15)
+    {
+        uDst.au16[0] = uSrc1.au16[0] << uSrc2.au8[0];
+        uDst.au16[1] = uSrc1.au16[1] << uSrc2.au8[0];
+        uDst.au16[2] = uSrc1.au16[2] << uSrc2.au8[0];
+        uDst.au16[3] = uSrc1.au16[3] << uSrc2.au8[0];
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psllw_imm_u64,(uint64_t *puDst, uint8_t uShift))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uDst;
+
+    if (uShift <= 15)
+    {
+        uDst.au16[0] = uSrc1.au16[0] << uShift;
+        uDst.au16[1] = uSrc1.au16[1] << uShift;
+        uDst.au16[2] = uSrc1.au16[2] << uShift;
+        uDst.au16[3] = uSrc1.au16[3] << uShift;
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psllw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (puSrc->au64[0] <= 15)
+    {
+        puDst->au16[0] = uSrc1.au16[0] << puSrc->au8[0];
+        puDst->au16[1] = uSrc1.au16[1] << puSrc->au8[0];
+        puDst->au16[2] = uSrc1.au16[2] << puSrc->au8[0];
+        puDst->au16[3] = uSrc1.au16[3] << puSrc->au8[0];
+        puDst->au16[4] = uSrc1.au16[4] << puSrc->au8[0];
+        puDst->au16[5] = uSrc1.au16[5] << puSrc->au8[0];
+        puDst->au16[6] = uSrc1.au16[6] << puSrc->au8[0];
+        puDst->au16[7] = uSrc1.au16[7] << puSrc->au8[0];
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psllw_imm_u128,(PRTUINT128U puDst, uint8_t uShift))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (uShift <= 15)
+    {
+        puDst->au16[0] = uSrc1.au16[0] << uShift;
+        puDst->au16[1] = uSrc1.au16[1] << uShift;
+        puDst->au16[2] = uSrc1.au16[2] << uShift;
+        puDst->au16[3] = uSrc1.au16[3] << uShift;
+        puDst->au16[4] = uSrc1.au16[4] << uShift;
+        puDst->au16[5] = uSrc1.au16[5] << uShift;
+        puDst->au16[6] = uSrc1.au16[6] << uShift;
+        puDst->au16[7] = uSrc1.au16[7] << uShift;
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+#endif
+
+
+/*
+ * PSRLD / VPSRLD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrld_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    if (uSrc2.au64[0] <= 31)
+    {
+        uDst.au32[0] = uSrc1.au32[0] >> uSrc2.au8[0];
+        uDst.au32[1] = uSrc1.au32[1] >> uSrc2.au8[0];
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrld_imm_u64,(uint64_t *puDst, uint8_t uShift))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uDst;
+
+    if (uShift <= 31)
+    {
+        uDst.au32[0] = uSrc1.au32[0] >> uShift;
+        uDst.au32[1] = uSrc1.au32[1] >> uShift;
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrld_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (puSrc->au64[0] <= 31)
+    {
+        puDst->au32[0] = uSrc1.au32[0] >> puSrc->au8[0];
+        puDst->au32[1] = uSrc1.au32[1] >> puSrc->au8[0];
+        puDst->au32[2] = uSrc1.au32[2] >> puSrc->au8[0];
+        puDst->au32[3] = uSrc1.au32[3] >> puSrc->au8[0];
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrld_imm_u128,(PRTUINT128U puDst, uint8_t uShift))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (uShift <= 31)
+    {
+        puDst->au32[0] = uSrc1.au32[0] >> uShift;
+        puDst->au32[1] = uSrc1.au32[1] >> uShift;
+        puDst->au32[2] = uSrc1.au32[2] >> uShift;
+        puDst->au32[3] = uSrc1.au32[3] >> uShift;
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+#endif
+
+
+/*
+ * PSRAD / VPSRAD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrad_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    if (uSrc2.au64[0] <= 31)
+    {
+        uDst.ai32[0] = uSrc1.ai32[0] >> uSrc2.au8[0];
+        uDst.ai32[1] = uSrc1.ai32[1] >> uSrc2.au8[0];
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrad_imm_u64,(uint64_t *puDst, uint8_t uShift))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uDst;
+
+    if (uShift <= 31)
+    {
+        uDst.ai32[0] = uSrc1.ai32[0] >> uShift;
+        uDst.ai32[1] = uSrc1.ai32[1] >> uShift;
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrad_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (puSrc->au64[0] <= 31)
+    {
+        puDst->ai32[0] = uSrc1.ai32[0] >> puSrc->au8[0];
+        puDst->ai32[1] = uSrc1.ai32[1] >> puSrc->au8[0];
+        puDst->ai32[2] = uSrc1.ai32[2] >> puSrc->au8[0];
+        puDst->ai32[3] = uSrc1.ai32[3] >> puSrc->au8[0];
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrad_imm_u128,(PRTUINT128U puDst, uint8_t uShift))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (uShift <= 31)
+    {
+        puDst->ai32[0] = uSrc1.ai32[0] >> uShift;
+        puDst->ai32[1] = uSrc1.ai32[1] >> uShift;
+        puDst->ai32[2] = uSrc1.ai32[2] >> uShift;
+        puDst->ai32[3] = uSrc1.ai32[3] >> uShift;
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+#endif
+
+
+/*
+ * PSLLD / VPSLLD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pslld_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    if (uSrc2.au64[0] <= 31)
+    {
+        uDst.au32[0] = uSrc1.au32[0] << uSrc2.au8[0];
+        uDst.au32[1] = uSrc1.au32[1] << uSrc2.au8[0];
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pslld_imm_u64,(uint64_t *puDst, uint8_t uShift))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uDst;
+
+    if (uShift <= 31)
+    {
+        uDst.au32[0] = uSrc1.au32[0] << uShift;
+        uDst.au32[1] = uSrc1.au32[1] << uShift;
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pslld_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (puSrc->au64[0] <= 31)
+    {
+        puDst->au32[0] = uSrc1.au32[0] << puSrc->au8[0];
+        puDst->au32[1] = uSrc1.au32[1] << puSrc->au8[0];
+        puDst->au32[2] = uSrc1.au32[2] << puSrc->au8[0];
+        puDst->au32[3] = uSrc1.au32[3] << puSrc->au8[0];
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pslld_imm_u128,(PRTUINT128U puDst, uint8_t uShift))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (uShift <= 31)
+    {
+        puDst->au32[0] = uSrc1.au32[0] << uShift;
+        puDst->au32[1] = uSrc1.au32[1] << uShift;
+        puDst->au32[2] = uSrc1.au32[2] << uShift;
+        puDst->au32[3] = uSrc1.au32[3] << uShift;
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+#endif
+
+
+/*
+ * PSRLQ / VPSRLQ
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrlq_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    if (uSrc2.au64[0] <= 63)
+    {
+        uDst.au64[0] = uSrc1.au64[0] >> uSrc2.au8[0];
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrlq_imm_u64,(uint64_t *puDst, uint8_t uShift))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uDst;
+
+    if (uShift <= 63)
+    {
+        uDst.au64[0] = uSrc1.au64[0] >> uShift;
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrlq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (puSrc->au64[0] <= 63)
+    {
+        puDst->au64[0] = uSrc1.au64[0] >> puSrc->au8[0];
+        puDst->au64[1] = uSrc1.au64[1] >> puSrc->au8[0];
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrlq_imm_u128,(PRTUINT128U puDst, uint8_t uShift))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (uShift <= 63)
+    {
+        puDst->au64[0] = uSrc1.au64[0] >> uShift;
+        puDst->au64[1] = uSrc1.au64[1] >> uShift;
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+#endif
+
+
+/*
+ * PSLLQ / VPSLLQ
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psllq_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    if (uSrc2.au64[0] <= 63)
+    {
+        uDst.au64[0] = uSrc1.au64[0] << uSrc2.au8[0];
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psllq_imm_u64,(uint64_t *puDst, uint8_t uShift))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uDst;
+
+    if (uShift <= 63)
+    {
+        uDst.au64[0] = uSrc1.au64[0] << uShift;
+    }
+    else
+    {
+        uDst.au64[0] = 0;
+    }
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psllq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (puSrc->au64[0] <= 63)
+    {
+        puDst->au64[0] = uSrc1.au64[0] << puSrc->au8[0];
+        puDst->au64[1] = uSrc1.au64[1] << puSrc->au8[0];
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psllq_imm_u128,(PRTUINT128U puDst, uint8_t uShift))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (uShift <= 63)
+    {
+        puDst->au64[0] = uSrc1.au64[0] << uShift;
+        puDst->au64[1] = uSrc1.au64[1] << uShift;
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+#endif
+
+
+/*
+ * PSRLDQ / VPSRLDQ
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psrldq_imm_u128,(PRTUINT128U puDst, uint8_t uShift))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (uShift < 16)
+    {
+        int i;
+
+        for (i = 0; i < 16 - uShift; ++i)
+            puDst->au8[i] = uSrc1.au8[i + uShift];
+        for (i = 16 - uShift; i < 16; ++i)
+            puDst->au8[i] = 0;
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+#endif
+
+
+/*
+ * PSLLDQ / VPSLLDQ
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pslldq_imm_u128,(PRTUINT128U puDst, uint8_t uShift))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    if (uShift < 16)
+    {
+        int i;
+
+        for (i = 0; i < uShift; ++i)
+            puDst->au8[i] = 0;
+        for (i = uShift; i < 16; ++i)
+            puDst->au8[i] = uSrc1.au8[i - uShift];
+    }
+    else
+    {
+        puDst->au64[0] = 0;
+        puDst->au64[1] = 0;
+    }
+}
+
+#endif
+
+
+/*
+ * PMADDWD / VPMADDWD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaddwd_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    uDst.ai32[0] = (int32_t)uSrc1.ai16[0] * uSrc2.ai16[0] + (int32_t)uSrc1.ai16[1] * uSrc2.ai16[1];
+    uDst.ai32[1] = (int32_t)uSrc1.ai16[2] * uSrc2.ai16[2] + (int32_t)uSrc1.ai16[3] * uSrc2.ai16[3];
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaddwd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai32[0] = (int32_t)uSrc1.ai16[0] * puSrc->ai16[0] + (int32_t)uSrc1.ai16[1] * puSrc->ai16[1];
+    puDst->ai32[1] = (int32_t)uSrc1.ai16[2] * puSrc->ai16[2] + (int32_t)uSrc1.ai16[3] * puSrc->ai16[3];
+    puDst->ai32[2] = (int32_t)uSrc1.ai16[4] * puSrc->ai16[4] + (int32_t)uSrc1.ai16[5] * puSrc->ai16[5];
+    puDst->ai32[3] = (int32_t)uSrc1.ai16[6] * puSrc->ai16[6] + (int32_t)uSrc1.ai16[7] * puSrc->ai16[7];
+    RT_NOREF(pFpuState);
+}
+
+#endif
+
+
+/*
+ * PMAXUB / VPMAXUB / PMAXUW / VPMAXUW / PMAXUD / VPMAXUD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxub_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    uDst.au8[0] = RT_MAX(uSrc1.au8[0], uSrc2.au8[0]);
+    uDst.au8[1] = RT_MAX(uSrc1.au8[1], uSrc2.au8[1]);
+    uDst.au8[2] = RT_MAX(uSrc1.au8[2], uSrc2.au8[2]);
+    uDst.au8[3] = RT_MAX(uSrc1.au8[3], uSrc2.au8[3]);
+    uDst.au8[4] = RT_MAX(uSrc1.au8[4], uSrc2.au8[4]);
+    uDst.au8[5] = RT_MAX(uSrc1.au8[5], uSrc2.au8[5]);
+    uDst.au8[6] = RT_MAX(uSrc1.au8[6], uSrc2.au8[6]);
+    uDst.au8[7] = RT_MAX(uSrc1.au8[7], uSrc2.au8[7]);
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxub_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au8[ 0] = RT_MAX(uSrc1.au8[ 0], puSrc->au8[ 0]);
+    puDst->au8[ 1] = RT_MAX(uSrc1.au8[ 1], puSrc->au8[ 1]);
+    puDst->au8[ 2] = RT_MAX(uSrc1.au8[ 2], puSrc->au8[ 2]);
+    puDst->au8[ 3] = RT_MAX(uSrc1.au8[ 3], puSrc->au8[ 3]);
+    puDst->au8[ 4] = RT_MAX(uSrc1.au8[ 4], puSrc->au8[ 4]);
+    puDst->au8[ 5] = RT_MAX(uSrc1.au8[ 5], puSrc->au8[ 5]);
+    puDst->au8[ 6] = RT_MAX(uSrc1.au8[ 6], puSrc->au8[ 6]);
+    puDst->au8[ 7] = RT_MAX(uSrc1.au8[ 7], puSrc->au8[ 7]);
+    puDst->au8[ 8] = RT_MAX(uSrc1.au8[ 8], puSrc->au8[ 8]);
+    puDst->au8[ 9] = RT_MAX(uSrc1.au8[ 9], puSrc->au8[ 9]);
+    puDst->au8[10] = RT_MAX(uSrc1.au8[10], puSrc->au8[10]);
+    puDst->au8[11] = RT_MAX(uSrc1.au8[11], puSrc->au8[11]);
+    puDst->au8[12] = RT_MAX(uSrc1.au8[12], puSrc->au8[12]);
+    puDst->au8[13] = RT_MAX(uSrc1.au8[13], puSrc->au8[13]);
+    puDst->au8[14] = RT_MAX(uSrc1.au8[14], puSrc->au8[14]);
+    puDst->au8[15] = RT_MAX(uSrc1.au8[15], puSrc->au8[15]);
+    RT_NOREF(pFpuState);
+}
+
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxuw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au16[ 0] = RT_MAX(uSrc1.au16[ 0], puSrc->au16[ 0]);
+    puDst->au16[ 1] = RT_MAX(uSrc1.au16[ 1], puSrc->au16[ 1]);
+    puDst->au16[ 2] = RT_MAX(uSrc1.au16[ 2], puSrc->au16[ 2]);
+    puDst->au16[ 3] = RT_MAX(uSrc1.au16[ 3], puSrc->au16[ 3]);
+    puDst->au16[ 4] = RT_MAX(uSrc1.au16[ 4], puSrc->au16[ 4]);
+    puDst->au16[ 5] = RT_MAX(uSrc1.au16[ 5], puSrc->au16[ 5]);
+    puDst->au16[ 6] = RT_MAX(uSrc1.au16[ 6], puSrc->au16[ 6]);
+    puDst->au16[ 7] = RT_MAX(uSrc1.au16[ 7], puSrc->au16[ 7]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxud_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au32[ 0] = RT_MAX(uSrc1.au32[ 0], puSrc->au32[ 0]);
+    puDst->au32[ 1] = RT_MAX(uSrc1.au32[ 1], puSrc->au32[ 1]);
+    puDst->au32[ 2] = RT_MAX(uSrc1.au32[ 2], puSrc->au32[ 2]);
+    puDst->au32[ 3] = RT_MAX(uSrc1.au32[ 3], puSrc->au32[ 3]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxub_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->au8[ 0] = RT_MAX(puSrc1->au8[ 0], puSrc2->au8[ 0]);
+    puDst->au8[ 1] = RT_MAX(puSrc1->au8[ 1], puSrc2->au8[ 1]);
+    puDst->au8[ 2] = RT_MAX(puSrc1->au8[ 2], puSrc2->au8[ 2]);
+    puDst->au8[ 3] = RT_MAX(puSrc1->au8[ 3], puSrc2->au8[ 3]);
+    puDst->au8[ 4] = RT_MAX(puSrc1->au8[ 4], puSrc2->au8[ 4]);
+    puDst->au8[ 5] = RT_MAX(puSrc1->au8[ 5], puSrc2->au8[ 5]);
+    puDst->au8[ 6] = RT_MAX(puSrc1->au8[ 6], puSrc2->au8[ 6]);
+    puDst->au8[ 7] = RT_MAX(puSrc1->au8[ 7], puSrc2->au8[ 7]);
+    puDst->au8[ 8] = RT_MAX(puSrc1->au8[ 8], puSrc2->au8[ 8]);
+    puDst->au8[ 9] = RT_MAX(puSrc1->au8[ 9], puSrc2->au8[ 9]);
+    puDst->au8[10] = RT_MAX(puSrc1->au8[10], puSrc2->au8[10]);
+    puDst->au8[11] = RT_MAX(puSrc1->au8[11], puSrc2->au8[11]);
+    puDst->au8[12] = RT_MAX(puSrc1->au8[12], puSrc2->au8[12]);
+    puDst->au8[13] = RT_MAX(puSrc1->au8[13], puSrc2->au8[13]);
+    puDst->au8[14] = RT_MAX(puSrc1->au8[14], puSrc2->au8[14]);
+    puDst->au8[15] = RT_MAX(puSrc1->au8[15], puSrc2->au8[15]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxub_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->au8[ 0] = RT_MAX(puSrc1->au8[ 0], puSrc2->au8[ 0]);
+    puDst->au8[ 1] = RT_MAX(puSrc1->au8[ 1], puSrc2->au8[ 1]);
+    puDst->au8[ 2] = RT_MAX(puSrc1->au8[ 2], puSrc2->au8[ 2]);
+    puDst->au8[ 3] = RT_MAX(puSrc1->au8[ 3], puSrc2->au8[ 3]);
+    puDst->au8[ 4] = RT_MAX(puSrc1->au8[ 4], puSrc2->au8[ 4]);
+    puDst->au8[ 5] = RT_MAX(puSrc1->au8[ 5], puSrc2->au8[ 5]);
+    puDst->au8[ 6] = RT_MAX(puSrc1->au8[ 6], puSrc2->au8[ 6]);
+    puDst->au8[ 7] = RT_MAX(puSrc1->au8[ 7], puSrc2->au8[ 7]);
+    puDst->au8[ 8] = RT_MAX(puSrc1->au8[ 8], puSrc2->au8[ 8]);
+    puDst->au8[ 9] = RT_MAX(puSrc1->au8[ 9], puSrc2->au8[ 9]);
+    puDst->au8[10] = RT_MAX(puSrc1->au8[10], puSrc2->au8[10]);
+    puDst->au8[11] = RT_MAX(puSrc1->au8[11], puSrc2->au8[11]);
+    puDst->au8[12] = RT_MAX(puSrc1->au8[12], puSrc2->au8[12]);
+    puDst->au8[13] = RT_MAX(puSrc1->au8[13], puSrc2->au8[13]);
+    puDst->au8[14] = RT_MAX(puSrc1->au8[14], puSrc2->au8[14]);
+    puDst->au8[15] = RT_MAX(puSrc1->au8[15], puSrc2->au8[15]);
+    puDst->au8[16] = RT_MAX(puSrc1->au8[16], puSrc2->au8[16]);
+    puDst->au8[17] = RT_MAX(puSrc1->au8[17], puSrc2->au8[17]);
+    puDst->au8[18] = RT_MAX(puSrc1->au8[18], puSrc2->au8[18]);
+    puDst->au8[19] = RT_MAX(puSrc1->au8[19], puSrc2->au8[19]);
+    puDst->au8[20] = RT_MAX(puSrc1->au8[20], puSrc2->au8[20]);
+    puDst->au8[21] = RT_MAX(puSrc1->au8[21], puSrc2->au8[21]);
+    puDst->au8[22] = RT_MAX(puSrc1->au8[22], puSrc2->au8[22]);
+    puDst->au8[23] = RT_MAX(puSrc1->au8[23], puSrc2->au8[23]);
+    puDst->au8[24] = RT_MAX(puSrc1->au8[24], puSrc2->au8[24]);
+    puDst->au8[25] = RT_MAX(puSrc1->au8[25], puSrc2->au8[25]);
+    puDst->au8[26] = RT_MAX(puSrc1->au8[26], puSrc2->au8[26]);
+    puDst->au8[27] = RT_MAX(puSrc1->au8[27], puSrc2->au8[27]);
+    puDst->au8[28] = RT_MAX(puSrc1->au8[28], puSrc2->au8[28]);
+    puDst->au8[29] = RT_MAX(puSrc1->au8[29], puSrc2->au8[29]);
+    puDst->au8[30] = RT_MAX(puSrc1->au8[30], puSrc2->au8[30]);
+    puDst->au8[31] = RT_MAX(puSrc1->au8[31], puSrc2->au8[31]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxuw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->au16[ 0] = RT_MAX(puSrc1->au16[ 0], puSrc2->au16[ 0]);
+    puDst->au16[ 1] = RT_MAX(puSrc1->au16[ 1], puSrc2->au16[ 1]);
+    puDst->au16[ 2] = RT_MAX(puSrc1->au16[ 2], puSrc2->au16[ 2]);
+    puDst->au16[ 3] = RT_MAX(puSrc1->au16[ 3], puSrc2->au16[ 3]);
+    puDst->au16[ 4] = RT_MAX(puSrc1->au16[ 4], puSrc2->au16[ 4]);
+    puDst->au16[ 5] = RT_MAX(puSrc1->au16[ 5], puSrc2->au16[ 5]);
+    puDst->au16[ 6] = RT_MAX(puSrc1->au16[ 6], puSrc2->au16[ 6]);
+    puDst->au16[ 7] = RT_MAX(puSrc1->au16[ 7], puSrc2->au16[ 7]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxuw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->au16[ 0] = RT_MAX(puSrc1->au16[ 0], puSrc2->au16[ 0]);
+    puDst->au16[ 1] = RT_MAX(puSrc1->au16[ 1], puSrc2->au16[ 1]);
+    puDst->au16[ 2] = RT_MAX(puSrc1->au16[ 2], puSrc2->au16[ 2]);
+    puDst->au16[ 3] = RT_MAX(puSrc1->au16[ 3], puSrc2->au16[ 3]);
+    puDst->au16[ 4] = RT_MAX(puSrc1->au16[ 4], puSrc2->au16[ 4]);
+    puDst->au16[ 5] = RT_MAX(puSrc1->au16[ 5], puSrc2->au16[ 5]);
+    puDst->au16[ 6] = RT_MAX(puSrc1->au16[ 6], puSrc2->au16[ 6]);
+    puDst->au16[ 7] = RT_MAX(puSrc1->au16[ 7], puSrc2->au16[ 7]);
+    puDst->au16[ 8] = RT_MAX(puSrc1->au16[ 8], puSrc2->au16[ 8]);
+    puDst->au16[ 9] = RT_MAX(puSrc1->au16[ 9], puSrc2->au16[ 9]);
+    puDst->au16[10] = RT_MAX(puSrc1->au16[10], puSrc2->au16[10]);
+    puDst->au16[11] = RT_MAX(puSrc1->au16[11], puSrc2->au16[11]);
+    puDst->au16[12] = RT_MAX(puSrc1->au16[12], puSrc2->au16[12]);
+    puDst->au16[13] = RT_MAX(puSrc1->au16[13], puSrc2->au16[13]);
+    puDst->au16[14] = RT_MAX(puSrc1->au16[14], puSrc2->au16[14]);
+    puDst->au16[15] = RT_MAX(puSrc1->au16[15], puSrc2->au16[15]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxud_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->au32[ 0] = RT_MAX(puSrc1->au32[ 0], puSrc2->au32[ 0]);
+    puDst->au32[ 1] = RT_MAX(puSrc1->au32[ 1], puSrc2->au32[ 1]);
+    puDst->au32[ 2] = RT_MAX(puSrc1->au32[ 2], puSrc2->au32[ 2]);
+    puDst->au32[ 3] = RT_MAX(puSrc1->au32[ 3], puSrc2->au32[ 3]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxud_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->au32[ 0] = RT_MAX(puSrc1->au32[ 0], puSrc2->au32[ 0]);
+    puDst->au32[ 1] = RT_MAX(puSrc1->au32[ 1], puSrc2->au32[ 1]);
+    puDst->au32[ 2] = RT_MAX(puSrc1->au32[ 2], puSrc2->au32[ 2]);
+    puDst->au32[ 3] = RT_MAX(puSrc1->au32[ 3], puSrc2->au32[ 3]);
+    puDst->au32[ 4] = RT_MAX(puSrc1->au32[ 4], puSrc2->au32[ 4]);
+    puDst->au32[ 5] = RT_MAX(puSrc1->au32[ 5], puSrc2->au32[ 5]);
+    puDst->au32[ 6] = RT_MAX(puSrc1->au32[ 6], puSrc2->au32[ 6]);
+    puDst->au32[ 7] = RT_MAX(puSrc1->au32[ 7], puSrc2->au32[ 7]);
+    RT_NOREF(pExtState);
+}
+
+
+/*
+ * PMAXSB / VPMAXSB / PMAXSW / VPMAXSW / PMAXSD / VPMAXSD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxsw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    uDst.ai16[0] = RT_MAX(uSrc1.ai16[0], uSrc2.ai16[0]);
+    uDst.ai16[1] = RT_MAX(uSrc1.ai16[1], uSrc2.ai16[1]);
+    uDst.ai16[2] = RT_MAX(uSrc1.ai16[2], uSrc2.ai16[2]);
+    uDst.ai16[3] = RT_MAX(uSrc1.ai16[3], uSrc2.ai16[3]);
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxsw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai16[ 0] = RT_MAX(uSrc1.ai16[ 0], puSrc->ai16[ 0]);
+    puDst->ai16[ 1] = RT_MAX(uSrc1.ai16[ 1], puSrc->ai16[ 1]);
+    puDst->ai16[ 2] = RT_MAX(uSrc1.ai16[ 2], puSrc->ai16[ 2]);
+    puDst->ai16[ 3] = RT_MAX(uSrc1.ai16[ 3], puSrc->ai16[ 3]);
+    puDst->ai16[ 4] = RT_MAX(uSrc1.ai16[ 4], puSrc->ai16[ 4]);
+    puDst->ai16[ 5] = RT_MAX(uSrc1.ai16[ 5], puSrc->ai16[ 5]);
+    puDst->ai16[ 6] = RT_MAX(uSrc1.ai16[ 6], puSrc->ai16[ 6]);
+    puDst->ai16[ 7] = RT_MAX(uSrc1.ai16[ 7], puSrc->ai16[ 7]);
+    RT_NOREF(pFpuState);
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxsb_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai8[ 0] = RT_MAX(uSrc1.ai8[ 0], puSrc->ai8[ 0]);
+    puDst->ai8[ 1] = RT_MAX(uSrc1.ai8[ 1], puSrc->ai8[ 1]);
+    puDst->ai8[ 2] = RT_MAX(uSrc1.ai8[ 2], puSrc->ai8[ 2]);
+    puDst->ai8[ 3] = RT_MAX(uSrc1.ai8[ 3], puSrc->ai8[ 3]);
+    puDst->ai8[ 4] = RT_MAX(uSrc1.ai8[ 4], puSrc->ai8[ 4]);
+    puDst->ai8[ 5] = RT_MAX(uSrc1.ai8[ 5], puSrc->ai8[ 5]);
+    puDst->ai8[ 6] = RT_MAX(uSrc1.ai8[ 6], puSrc->ai8[ 6]);
+    puDst->ai8[ 7] = RT_MAX(uSrc1.ai8[ 7], puSrc->ai8[ 7]);
+    puDst->ai8[ 8] = RT_MAX(uSrc1.ai8[ 8], puSrc->ai8[ 8]);
+    puDst->ai8[ 9] = RT_MAX(uSrc1.ai8[ 9], puSrc->ai8[ 9]);
+    puDst->ai8[10] = RT_MAX(uSrc1.ai8[10], puSrc->ai8[10]);
+    puDst->ai8[11] = RT_MAX(uSrc1.ai8[11], puSrc->ai8[11]);
+    puDst->ai8[12] = RT_MAX(uSrc1.ai8[12], puSrc->ai8[12]);
+    puDst->ai8[13] = RT_MAX(uSrc1.ai8[13], puSrc->ai8[13]);
+    puDst->ai8[14] = RT_MAX(uSrc1.ai8[14], puSrc->ai8[14]);
+    puDst->ai8[15] = RT_MAX(uSrc1.ai8[15], puSrc->ai8[15]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaxsd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai32[ 0] = RT_MAX(uSrc1.ai32[ 0], puSrc->ai32[ 0]);
+    puDst->ai32[ 1] = RT_MAX(uSrc1.ai32[ 1], puSrc->ai32[ 1]);
+    puDst->ai32[ 2] = RT_MAX(uSrc1.ai32[ 2], puSrc->ai32[ 2]);
+    puDst->ai32[ 3] = RT_MAX(uSrc1.ai32[ 3], puSrc->ai32[ 3]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->ai8[ 0] = RT_MAX(puSrc1->ai8[ 0], puSrc2->ai8[ 0]);
+    puDst->ai8[ 1] = RT_MAX(puSrc1->ai8[ 1], puSrc2->ai8[ 1]);
+    puDst->ai8[ 2] = RT_MAX(puSrc1->ai8[ 2], puSrc2->ai8[ 2]);
+    puDst->ai8[ 3] = RT_MAX(puSrc1->ai8[ 3], puSrc2->ai8[ 3]);
+    puDst->ai8[ 4] = RT_MAX(puSrc1->ai8[ 4], puSrc2->ai8[ 4]);
+    puDst->ai8[ 5] = RT_MAX(puSrc1->ai8[ 5], puSrc2->ai8[ 5]);
+    puDst->ai8[ 6] = RT_MAX(puSrc1->ai8[ 6], puSrc2->ai8[ 6]);
+    puDst->ai8[ 7] = RT_MAX(puSrc1->ai8[ 7], puSrc2->ai8[ 7]);
+    puDst->ai8[ 8] = RT_MAX(puSrc1->ai8[ 8], puSrc2->ai8[ 8]);
+    puDst->ai8[ 9] = RT_MAX(puSrc1->ai8[ 9], puSrc2->ai8[ 9]);
+    puDst->ai8[10] = RT_MAX(puSrc1->ai8[10], puSrc2->ai8[10]);
+    puDst->ai8[11] = RT_MAX(puSrc1->ai8[11], puSrc2->ai8[11]);
+    puDst->ai8[12] = RT_MAX(puSrc1->ai8[12], puSrc2->ai8[12]);
+    puDst->ai8[13] = RT_MAX(puSrc1->ai8[13], puSrc2->ai8[13]);
+    puDst->ai8[14] = RT_MAX(puSrc1->ai8[14], puSrc2->ai8[14]);
+    puDst->ai8[15] = RT_MAX(puSrc1->ai8[15], puSrc2->ai8[15]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->ai8[ 0] = RT_MAX(puSrc1->ai8[ 0], puSrc2->ai8[ 0]);
+    puDst->ai8[ 1] = RT_MAX(puSrc1->ai8[ 1], puSrc2->ai8[ 1]);
+    puDst->ai8[ 2] = RT_MAX(puSrc1->ai8[ 2], puSrc2->ai8[ 2]);
+    puDst->ai8[ 3] = RT_MAX(puSrc1->ai8[ 3], puSrc2->ai8[ 3]);
+    puDst->ai8[ 4] = RT_MAX(puSrc1->ai8[ 4], puSrc2->ai8[ 4]);
+    puDst->ai8[ 5] = RT_MAX(puSrc1->ai8[ 5], puSrc2->ai8[ 5]);
+    puDst->ai8[ 6] = RT_MAX(puSrc1->ai8[ 6], puSrc2->ai8[ 6]);
+    puDst->ai8[ 7] = RT_MAX(puSrc1->ai8[ 7], puSrc2->ai8[ 7]);
+    puDst->ai8[ 8] = RT_MAX(puSrc1->ai8[ 8], puSrc2->ai8[ 8]);
+    puDst->ai8[ 9] = RT_MAX(puSrc1->ai8[ 9], puSrc2->ai8[ 9]);
+    puDst->ai8[10] = RT_MAX(puSrc1->ai8[10], puSrc2->ai8[10]);
+    puDst->ai8[11] = RT_MAX(puSrc1->ai8[11], puSrc2->ai8[11]);
+    puDst->ai8[12] = RT_MAX(puSrc1->ai8[12], puSrc2->ai8[12]);
+    puDst->ai8[13] = RT_MAX(puSrc1->ai8[13], puSrc2->ai8[13]);
+    puDst->ai8[14] = RT_MAX(puSrc1->ai8[14], puSrc2->ai8[14]);
+    puDst->ai8[15] = RT_MAX(puSrc1->ai8[15], puSrc2->ai8[15]);
+    puDst->ai8[16] = RT_MAX(puSrc1->ai8[16], puSrc2->ai8[16]);
+    puDst->ai8[17] = RT_MAX(puSrc1->ai8[17], puSrc2->ai8[17]);
+    puDst->ai8[18] = RT_MAX(puSrc1->ai8[18], puSrc2->ai8[18]);
+    puDst->ai8[19] = RT_MAX(puSrc1->ai8[19], puSrc2->ai8[19]);
+    puDst->ai8[20] = RT_MAX(puSrc1->ai8[20], puSrc2->ai8[20]);
+    puDst->ai8[21] = RT_MAX(puSrc1->ai8[21], puSrc2->ai8[21]);
+    puDst->ai8[22] = RT_MAX(puSrc1->ai8[22], puSrc2->ai8[22]);
+    puDst->ai8[23] = RT_MAX(puSrc1->ai8[23], puSrc2->ai8[23]);
+    puDst->ai8[24] = RT_MAX(puSrc1->ai8[24], puSrc2->ai8[24]);
+    puDst->ai8[25] = RT_MAX(puSrc1->ai8[25], puSrc2->ai8[25]);
+    puDst->ai8[26] = RT_MAX(puSrc1->ai8[26], puSrc2->ai8[26]);
+    puDst->ai8[27] = RT_MAX(puSrc1->ai8[27], puSrc2->ai8[27]);
+    puDst->ai8[28] = RT_MAX(puSrc1->ai8[28], puSrc2->ai8[28]);
+    puDst->ai8[29] = RT_MAX(puSrc1->ai8[29], puSrc2->ai8[29]);
+    puDst->ai8[30] = RT_MAX(puSrc1->ai8[30], puSrc2->ai8[30]);
+    puDst->ai8[31] = RT_MAX(puSrc1->ai8[31], puSrc2->ai8[31]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->ai16[ 0] = RT_MAX(puSrc1->ai16[ 0], puSrc2->ai16[ 0]);
+    puDst->ai16[ 1] = RT_MAX(puSrc1->ai16[ 1], puSrc2->ai16[ 1]);
+    puDst->ai16[ 2] = RT_MAX(puSrc1->ai16[ 2], puSrc2->ai16[ 2]);
+    puDst->ai16[ 3] = RT_MAX(puSrc1->ai16[ 3], puSrc2->ai16[ 3]);
+    puDst->ai16[ 4] = RT_MAX(puSrc1->ai16[ 4], puSrc2->ai16[ 4]);
+    puDst->ai16[ 5] = RT_MAX(puSrc1->ai16[ 5], puSrc2->ai16[ 5]);
+    puDst->ai16[ 6] = RT_MAX(puSrc1->ai16[ 6], puSrc2->ai16[ 6]);
+    puDst->ai16[ 7] = RT_MAX(puSrc1->ai16[ 7], puSrc2->ai16[ 7]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->ai16[ 0] = RT_MAX(puSrc1->ai16[ 0], puSrc2->ai16[ 0]);
+    puDst->ai16[ 1] = RT_MAX(puSrc1->ai16[ 1], puSrc2->ai16[ 1]);
+    puDst->ai16[ 2] = RT_MAX(puSrc1->ai16[ 2], puSrc2->ai16[ 2]);
+    puDst->ai16[ 3] = RT_MAX(puSrc1->ai16[ 3], puSrc2->ai16[ 3]);
+    puDst->ai16[ 4] = RT_MAX(puSrc1->ai16[ 4], puSrc2->ai16[ 4]);
+    puDst->ai16[ 5] = RT_MAX(puSrc1->ai16[ 5], puSrc2->ai16[ 5]);
+    puDst->ai16[ 6] = RT_MAX(puSrc1->ai16[ 6], puSrc2->ai16[ 6]);
+    puDst->ai16[ 7] = RT_MAX(puSrc1->ai16[ 7], puSrc2->ai16[ 7]);
+    puDst->ai16[ 8] = RT_MAX(puSrc1->ai16[ 8], puSrc2->ai16[ 8]);
+    puDst->ai16[ 9] = RT_MAX(puSrc1->ai16[ 9], puSrc2->ai16[ 9]);
+    puDst->ai16[10] = RT_MAX(puSrc1->ai16[10], puSrc2->ai16[10]);
+    puDst->ai16[11] = RT_MAX(puSrc1->ai16[11], puSrc2->ai16[11]);
+    puDst->ai16[12] = RT_MAX(puSrc1->ai16[12], puSrc2->ai16[12]);
+    puDst->ai16[13] = RT_MAX(puSrc1->ai16[13], puSrc2->ai16[13]);
+    puDst->ai16[14] = RT_MAX(puSrc1->ai16[14], puSrc2->ai16[14]);
+    puDst->ai16[15] = RT_MAX(puSrc1->ai16[15], puSrc2->ai16[15]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->ai32[ 0] = RT_MAX(puSrc1->ai32[ 0], puSrc2->ai32[ 0]);
+    puDst->ai32[ 1] = RT_MAX(puSrc1->ai32[ 1], puSrc2->ai32[ 1]);
+    puDst->ai32[ 2] = RT_MAX(puSrc1->ai32[ 2], puSrc2->ai32[ 2]);
+    puDst->ai32[ 3] = RT_MAX(puSrc1->ai32[ 3], puSrc2->ai32[ 3]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaxsd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->ai32[ 0] = RT_MAX(puSrc1->ai32[ 0], puSrc2->ai32[ 0]);
+    puDst->ai32[ 1] = RT_MAX(puSrc1->ai32[ 1], puSrc2->ai32[ 1]);
+    puDst->ai32[ 2] = RT_MAX(puSrc1->ai32[ 2], puSrc2->ai32[ 2]);
+    puDst->ai32[ 3] = RT_MAX(puSrc1->ai32[ 3], puSrc2->ai32[ 3]);
+    puDst->ai32[ 4] = RT_MAX(puSrc1->ai32[ 4], puSrc2->ai32[ 4]);
+    puDst->ai32[ 5] = RT_MAX(puSrc1->ai32[ 5], puSrc2->ai32[ 5]);
+    puDst->ai32[ 6] = RT_MAX(puSrc1->ai32[ 6], puSrc2->ai32[ 6]);
+    puDst->ai32[ 7] = RT_MAX(puSrc1->ai32[ 7], puSrc2->ai32[ 7]);
+    RT_NOREF(pExtState);
+}
+
+
+/*
+ * PMINUB / VPMINUB / PMINUW / VPMINUW / PMINUD / VPMINUD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pminub_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    uDst.au8[0] = RT_MIN(uSrc1.au8[0], uSrc2.au8[0]);
+    uDst.au8[1] = RT_MIN(uSrc1.au8[1], uSrc2.au8[1]);
+    uDst.au8[2] = RT_MIN(uSrc1.au8[2], uSrc2.au8[2]);
+    uDst.au8[3] = RT_MIN(uSrc1.au8[3], uSrc2.au8[3]);
+    uDst.au8[4] = RT_MIN(uSrc1.au8[4], uSrc2.au8[4]);
+    uDst.au8[5] = RT_MIN(uSrc1.au8[5], uSrc2.au8[5]);
+    uDst.au8[6] = RT_MIN(uSrc1.au8[6], uSrc2.au8[6]);
+    uDst.au8[7] = RT_MIN(uSrc1.au8[7], uSrc2.au8[7]);
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pminub_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au8[ 0] = RT_MIN(uSrc1.au8[ 0], puSrc->au8[ 0]);
+    puDst->au8[ 1] = RT_MIN(uSrc1.au8[ 1], puSrc->au8[ 1]);
+    puDst->au8[ 2] = RT_MIN(uSrc1.au8[ 2], puSrc->au8[ 2]);
+    puDst->au8[ 3] = RT_MIN(uSrc1.au8[ 3], puSrc->au8[ 3]);
+    puDst->au8[ 4] = RT_MIN(uSrc1.au8[ 4], puSrc->au8[ 4]);
+    puDst->au8[ 5] = RT_MIN(uSrc1.au8[ 5], puSrc->au8[ 5]);
+    puDst->au8[ 6] = RT_MIN(uSrc1.au8[ 6], puSrc->au8[ 6]);
+    puDst->au8[ 7] = RT_MIN(uSrc1.au8[ 7], puSrc->au8[ 7]);
+    puDst->au8[ 8] = RT_MIN(uSrc1.au8[ 8], puSrc->au8[ 8]);
+    puDst->au8[ 9] = RT_MIN(uSrc1.au8[ 9], puSrc->au8[ 9]);
+    puDst->au8[10] = RT_MIN(uSrc1.au8[10], puSrc->au8[10]);
+    puDst->au8[11] = RT_MIN(uSrc1.au8[11], puSrc->au8[11]);
+    puDst->au8[12] = RT_MIN(uSrc1.au8[12], puSrc->au8[12]);
+    puDst->au8[13] = RT_MIN(uSrc1.au8[13], puSrc->au8[13]);
+    puDst->au8[14] = RT_MIN(uSrc1.au8[14], puSrc->au8[14]);
+    puDst->au8[15] = RT_MIN(uSrc1.au8[15], puSrc->au8[15]);
+    RT_NOREF(pFpuState);
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pminuw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au16[ 0] = RT_MIN(uSrc1.au16[ 0], puSrc->au16[ 0]);
+    puDst->au16[ 1] = RT_MIN(uSrc1.au16[ 1], puSrc->au16[ 1]);
+    puDst->au16[ 2] = RT_MIN(uSrc1.au16[ 2], puSrc->au16[ 2]);
+    puDst->au16[ 3] = RT_MIN(uSrc1.au16[ 3], puSrc->au16[ 3]);
+    puDst->au16[ 4] = RT_MIN(uSrc1.au16[ 4], puSrc->au16[ 4]);
+    puDst->au16[ 5] = RT_MIN(uSrc1.au16[ 5], puSrc->au16[ 5]);
+    puDst->au16[ 6] = RT_MIN(uSrc1.au16[ 6], puSrc->au16[ 6]);
+    puDst->au16[ 7] = RT_MIN(uSrc1.au16[ 7], puSrc->au16[ 7]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pminud_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au32[ 0] = RT_MIN(uSrc1.au32[ 0], puSrc->au32[ 0]);
+    puDst->au32[ 1] = RT_MIN(uSrc1.au32[ 1], puSrc->au32[ 1]);
+    puDst->au32[ 2] = RT_MIN(uSrc1.au32[ 2], puSrc->au32[ 2]);
+    puDst->au32[ 3] = RT_MIN(uSrc1.au32[ 3], puSrc->au32[ 3]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminub_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->au8[ 0] = RT_MIN(puSrc1->au8[ 0], puSrc2->au8[ 0]);
+    puDst->au8[ 1] = RT_MIN(puSrc1->au8[ 1], puSrc2->au8[ 1]);
+    puDst->au8[ 2] = RT_MIN(puSrc1->au8[ 2], puSrc2->au8[ 2]);
+    puDst->au8[ 3] = RT_MIN(puSrc1->au8[ 3], puSrc2->au8[ 3]);
+    puDst->au8[ 4] = RT_MIN(puSrc1->au8[ 4], puSrc2->au8[ 4]);
+    puDst->au8[ 5] = RT_MIN(puSrc1->au8[ 5], puSrc2->au8[ 5]);
+    puDst->au8[ 6] = RT_MIN(puSrc1->au8[ 6], puSrc2->au8[ 6]);
+    puDst->au8[ 7] = RT_MIN(puSrc1->au8[ 7], puSrc2->au8[ 7]);
+    puDst->au8[ 8] = RT_MIN(puSrc1->au8[ 8], puSrc2->au8[ 8]);
+    puDst->au8[ 9] = RT_MIN(puSrc1->au8[ 9], puSrc2->au8[ 9]);
+    puDst->au8[10] = RT_MIN(puSrc1->au8[10], puSrc2->au8[10]);
+    puDst->au8[11] = RT_MIN(puSrc1->au8[11], puSrc2->au8[11]);
+    puDst->au8[12] = RT_MIN(puSrc1->au8[12], puSrc2->au8[12]);
+    puDst->au8[13] = RT_MIN(puSrc1->au8[13], puSrc2->au8[13]);
+    puDst->au8[14] = RT_MIN(puSrc1->au8[14], puSrc2->au8[14]);
+    puDst->au8[15] = RT_MIN(puSrc1->au8[15], puSrc2->au8[15]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminub_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->au8[ 0] = RT_MIN(puSrc1->au8[ 0], puSrc2->au8[ 0]);
+    puDst->au8[ 1] = RT_MIN(puSrc1->au8[ 1], puSrc2->au8[ 1]);
+    puDst->au8[ 2] = RT_MIN(puSrc1->au8[ 2], puSrc2->au8[ 2]);
+    puDst->au8[ 3] = RT_MIN(puSrc1->au8[ 3], puSrc2->au8[ 3]);
+    puDst->au8[ 4] = RT_MIN(puSrc1->au8[ 4], puSrc2->au8[ 4]);
+    puDst->au8[ 5] = RT_MIN(puSrc1->au8[ 5], puSrc2->au8[ 5]);
+    puDst->au8[ 6] = RT_MIN(puSrc1->au8[ 6], puSrc2->au8[ 6]);
+    puDst->au8[ 7] = RT_MIN(puSrc1->au8[ 7], puSrc2->au8[ 7]);
+    puDst->au8[ 8] = RT_MIN(puSrc1->au8[ 8], puSrc2->au8[ 8]);
+    puDst->au8[ 9] = RT_MIN(puSrc1->au8[ 9], puSrc2->au8[ 9]);
+    puDst->au8[10] = RT_MIN(puSrc1->au8[10], puSrc2->au8[10]);
+    puDst->au8[11] = RT_MIN(puSrc1->au8[11], puSrc2->au8[11]);
+    puDst->au8[12] = RT_MIN(puSrc1->au8[12], puSrc2->au8[12]);
+    puDst->au8[13] = RT_MIN(puSrc1->au8[13], puSrc2->au8[13]);
+    puDst->au8[14] = RT_MIN(puSrc1->au8[14], puSrc2->au8[14]);
+    puDst->au8[15] = RT_MIN(puSrc1->au8[15], puSrc2->au8[15]);
+    puDst->au8[16] = RT_MIN(puSrc1->au8[16], puSrc2->au8[16]);
+    puDst->au8[17] = RT_MIN(puSrc1->au8[17], puSrc2->au8[17]);
+    puDst->au8[18] = RT_MIN(puSrc1->au8[18], puSrc2->au8[18]);
+    puDst->au8[19] = RT_MIN(puSrc1->au8[19], puSrc2->au8[19]);
+    puDst->au8[20] = RT_MIN(puSrc1->au8[20], puSrc2->au8[20]);
+    puDst->au8[21] = RT_MIN(puSrc1->au8[21], puSrc2->au8[21]);
+    puDst->au8[22] = RT_MIN(puSrc1->au8[22], puSrc2->au8[22]);
+    puDst->au8[23] = RT_MIN(puSrc1->au8[23], puSrc2->au8[23]);
+    puDst->au8[24] = RT_MIN(puSrc1->au8[24], puSrc2->au8[24]);
+    puDst->au8[25] = RT_MIN(puSrc1->au8[25], puSrc2->au8[25]);
+    puDst->au8[26] = RT_MIN(puSrc1->au8[26], puSrc2->au8[26]);
+    puDst->au8[27] = RT_MIN(puSrc1->au8[27], puSrc2->au8[27]);
+    puDst->au8[28] = RT_MIN(puSrc1->au8[28], puSrc2->au8[28]);
+    puDst->au8[29] = RT_MIN(puSrc1->au8[29], puSrc2->au8[29]);
+    puDst->au8[30] = RT_MIN(puSrc1->au8[30], puSrc2->au8[30]);
+    puDst->au8[31] = RT_MIN(puSrc1->au8[31], puSrc2->au8[31]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminuw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->au16[ 0] = RT_MIN(puSrc1->au16[ 0], puSrc2->au16[ 0]);
+    puDst->au16[ 1] = RT_MIN(puSrc1->au16[ 1], puSrc2->au16[ 1]);
+    puDst->au16[ 2] = RT_MIN(puSrc1->au16[ 2], puSrc2->au16[ 2]);
+    puDst->au16[ 3] = RT_MIN(puSrc1->au16[ 3], puSrc2->au16[ 3]);
+    puDst->au16[ 4] = RT_MIN(puSrc1->au16[ 4], puSrc2->au16[ 4]);
+    puDst->au16[ 5] = RT_MIN(puSrc1->au16[ 5], puSrc2->au16[ 5]);
+    puDst->au16[ 6] = RT_MIN(puSrc1->au16[ 6], puSrc2->au16[ 6]);
+    puDst->au16[ 7] = RT_MIN(puSrc1->au16[ 7], puSrc2->au16[ 7]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminuw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->au16[ 0] = RT_MIN(puSrc1->au16[ 0], puSrc2->au16[ 0]);
+    puDst->au16[ 1] = RT_MIN(puSrc1->au16[ 1], puSrc2->au16[ 1]);
+    puDst->au16[ 2] = RT_MIN(puSrc1->au16[ 2], puSrc2->au16[ 2]);
+    puDst->au16[ 3] = RT_MIN(puSrc1->au16[ 3], puSrc2->au16[ 3]);
+    puDst->au16[ 4] = RT_MIN(puSrc1->au16[ 4], puSrc2->au16[ 4]);
+    puDst->au16[ 5] = RT_MIN(puSrc1->au16[ 5], puSrc2->au16[ 5]);
+    puDst->au16[ 6] = RT_MIN(puSrc1->au16[ 6], puSrc2->au16[ 6]);
+    puDst->au16[ 7] = RT_MIN(puSrc1->au16[ 7], puSrc2->au16[ 7]);
+    puDst->au16[ 8] = RT_MIN(puSrc1->au16[ 8], puSrc2->au16[ 8]);
+    puDst->au16[ 9] = RT_MIN(puSrc1->au16[ 9], puSrc2->au16[ 9]);
+    puDst->au16[10] = RT_MIN(puSrc1->au16[10], puSrc2->au16[10]);
+    puDst->au16[11] = RT_MIN(puSrc1->au16[11], puSrc2->au16[11]);
+    puDst->au16[12] = RT_MIN(puSrc1->au16[12], puSrc2->au16[12]);
+    puDst->au16[13] = RT_MIN(puSrc1->au16[13], puSrc2->au16[13]);
+    puDst->au16[14] = RT_MIN(puSrc1->au16[14], puSrc2->au16[14]);
+    puDst->au16[15] = RT_MIN(puSrc1->au16[15], puSrc2->au16[15]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminud_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->au32[ 0] = RT_MIN(puSrc1->au32[ 0], puSrc2->au32[ 0]);
+    puDst->au32[ 1] = RT_MIN(puSrc1->au32[ 1], puSrc2->au32[ 1]);
+    puDst->au32[ 2] = RT_MIN(puSrc1->au32[ 2], puSrc2->au32[ 2]);
+    puDst->au32[ 3] = RT_MIN(puSrc1->au32[ 3], puSrc2->au32[ 3]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminud_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->au32[ 0] = RT_MIN(puSrc1->au32[ 0], puSrc2->au32[ 0]);
+    puDst->au32[ 1] = RT_MIN(puSrc1->au32[ 1], puSrc2->au32[ 1]);
+    puDst->au32[ 2] = RT_MIN(puSrc1->au32[ 2], puSrc2->au32[ 2]);
+    puDst->au32[ 3] = RT_MIN(puSrc1->au32[ 3], puSrc2->au32[ 3]);
+    puDst->au32[ 4] = RT_MIN(puSrc1->au32[ 4], puSrc2->au32[ 4]);
+    puDst->au32[ 5] = RT_MIN(puSrc1->au32[ 5], puSrc2->au32[ 5]);
+    puDst->au32[ 6] = RT_MIN(puSrc1->au32[ 6], puSrc2->au32[ 6]);
+    puDst->au32[ 7] = RT_MIN(puSrc1->au32[ 7], puSrc2->au32[ 7]);
+    RT_NOREF(pExtState);
+}
+
+
+/*
+ * PMINSB / VPMINSB / PMINSW / VPMINSW / PMINSD / VPMINSD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pminsw_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    uDst.ai16[0] = RT_MIN(uSrc1.ai16[0], uSrc2.ai16[0]);
+    uDst.ai16[1] = RT_MIN(uSrc1.ai16[1], uSrc2.ai16[1]);
+    uDst.ai16[2] = RT_MIN(uSrc1.ai16[2], uSrc2.ai16[2]);
+    uDst.ai16[3] = RT_MIN(uSrc1.ai16[3], uSrc2.ai16[3]);
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pminsw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai16[ 0] = RT_MIN(uSrc1.ai16[ 0], puSrc->ai16[ 0]);
+    puDst->ai16[ 1] = RT_MIN(uSrc1.ai16[ 1], puSrc->ai16[ 1]);
+    puDst->ai16[ 2] = RT_MIN(uSrc1.ai16[ 2], puSrc->ai16[ 2]);
+    puDst->ai16[ 3] = RT_MIN(uSrc1.ai16[ 3], puSrc->ai16[ 3]);
+    puDst->ai16[ 4] = RT_MIN(uSrc1.ai16[ 4], puSrc->ai16[ 4]);
+    puDst->ai16[ 5] = RT_MIN(uSrc1.ai16[ 5], puSrc->ai16[ 5]);
+    puDst->ai16[ 6] = RT_MIN(uSrc1.ai16[ 6], puSrc->ai16[ 6]);
+    puDst->ai16[ 7] = RT_MIN(uSrc1.ai16[ 7], puSrc->ai16[ 7]);
+    RT_NOREF(pFpuState);
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pminsb_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai8[ 0] = RT_MIN(uSrc1.ai8[ 0], puSrc->ai8[ 0]);
+    puDst->ai8[ 1] = RT_MIN(uSrc1.ai8[ 1], puSrc->ai8[ 1]);
+    puDst->ai8[ 2] = RT_MIN(uSrc1.ai8[ 2], puSrc->ai8[ 2]);
+    puDst->ai8[ 3] = RT_MIN(uSrc1.ai8[ 3], puSrc->ai8[ 3]);
+    puDst->ai8[ 4] = RT_MIN(uSrc1.ai8[ 4], puSrc->ai8[ 4]);
+    puDst->ai8[ 5] = RT_MIN(uSrc1.ai8[ 5], puSrc->ai8[ 5]);
+    puDst->ai8[ 6] = RT_MIN(uSrc1.ai8[ 6], puSrc->ai8[ 6]);
+    puDst->ai8[ 7] = RT_MIN(uSrc1.ai8[ 7], puSrc->ai8[ 7]);
+    puDst->ai8[ 8] = RT_MIN(uSrc1.ai8[ 8], puSrc->ai8[ 8]);
+    puDst->ai8[ 9] = RT_MIN(uSrc1.ai8[ 9], puSrc->ai8[ 9]);
+    puDst->ai8[10] = RT_MIN(uSrc1.ai8[10], puSrc->ai8[10]);
+    puDst->ai8[11] = RT_MIN(uSrc1.ai8[11], puSrc->ai8[11]);
+    puDst->ai8[12] = RT_MIN(uSrc1.ai8[12], puSrc->ai8[12]);
+    puDst->ai8[13] = RT_MIN(uSrc1.ai8[13], puSrc->ai8[13]);
+    puDst->ai8[14] = RT_MIN(uSrc1.ai8[14], puSrc->ai8[14]);
+    puDst->ai8[15] = RT_MIN(uSrc1.ai8[15], puSrc->ai8[15]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pminsd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai32[ 0] = RT_MIN(uSrc1.ai32[ 0], puSrc->ai32[ 0]);
+    puDst->ai32[ 1] = RT_MIN(uSrc1.ai32[ 1], puSrc->ai32[ 1]);
+    puDst->ai32[ 2] = RT_MIN(uSrc1.ai32[ 2], puSrc->ai32[ 2]);
+    puDst->ai32[ 3] = RT_MIN(uSrc1.ai32[ 3], puSrc->ai32[ 3]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->ai8[ 0] = RT_MIN(puSrc1->ai8[ 0], puSrc2->ai8[ 0]);
+    puDst->ai8[ 1] = RT_MIN(puSrc1->ai8[ 1], puSrc2->ai8[ 1]);
+    puDst->ai8[ 2] = RT_MIN(puSrc1->ai8[ 2], puSrc2->ai8[ 2]);
+    puDst->ai8[ 3] = RT_MIN(puSrc1->ai8[ 3], puSrc2->ai8[ 3]);
+    puDst->ai8[ 4] = RT_MIN(puSrc1->ai8[ 4], puSrc2->ai8[ 4]);
+    puDst->ai8[ 5] = RT_MIN(puSrc1->ai8[ 5], puSrc2->ai8[ 5]);
+    puDst->ai8[ 6] = RT_MIN(puSrc1->ai8[ 6], puSrc2->ai8[ 6]);
+    puDst->ai8[ 7] = RT_MIN(puSrc1->ai8[ 7], puSrc2->ai8[ 7]);
+    puDst->ai8[ 8] = RT_MIN(puSrc1->ai8[ 8], puSrc2->ai8[ 8]);
+    puDst->ai8[ 9] = RT_MIN(puSrc1->ai8[ 9], puSrc2->ai8[ 9]);
+    puDst->ai8[10] = RT_MIN(puSrc1->ai8[10], puSrc2->ai8[10]);
+    puDst->ai8[11] = RT_MIN(puSrc1->ai8[11], puSrc2->ai8[11]);
+    puDst->ai8[12] = RT_MIN(puSrc1->ai8[12], puSrc2->ai8[12]);
+    puDst->ai8[13] = RT_MIN(puSrc1->ai8[13], puSrc2->ai8[13]);
+    puDst->ai8[14] = RT_MIN(puSrc1->ai8[14], puSrc2->ai8[14]);
+    puDst->ai8[15] = RT_MIN(puSrc1->ai8[15], puSrc2->ai8[15]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->ai8[ 0] = RT_MIN(puSrc1->ai8[ 0], puSrc2->ai8[ 0]);
+    puDst->ai8[ 1] = RT_MIN(puSrc1->ai8[ 1], puSrc2->ai8[ 1]);
+    puDst->ai8[ 2] = RT_MIN(puSrc1->ai8[ 2], puSrc2->ai8[ 2]);
+    puDst->ai8[ 3] = RT_MIN(puSrc1->ai8[ 3], puSrc2->ai8[ 3]);
+    puDst->ai8[ 4] = RT_MIN(puSrc1->ai8[ 4], puSrc2->ai8[ 4]);
+    puDst->ai8[ 5] = RT_MIN(puSrc1->ai8[ 5], puSrc2->ai8[ 5]);
+    puDst->ai8[ 6] = RT_MIN(puSrc1->ai8[ 6], puSrc2->ai8[ 6]);
+    puDst->ai8[ 7] = RT_MIN(puSrc1->ai8[ 7], puSrc2->ai8[ 7]);
+    puDst->ai8[ 8] = RT_MIN(puSrc1->ai8[ 8], puSrc2->ai8[ 8]);
+    puDst->ai8[ 9] = RT_MIN(puSrc1->ai8[ 9], puSrc2->ai8[ 9]);
+    puDst->ai8[10] = RT_MIN(puSrc1->ai8[10], puSrc2->ai8[10]);
+    puDst->ai8[11] = RT_MIN(puSrc1->ai8[11], puSrc2->ai8[11]);
+    puDst->ai8[12] = RT_MIN(puSrc1->ai8[12], puSrc2->ai8[12]);
+    puDst->ai8[13] = RT_MIN(puSrc1->ai8[13], puSrc2->ai8[13]);
+    puDst->ai8[14] = RT_MIN(puSrc1->ai8[14], puSrc2->ai8[14]);
+    puDst->ai8[15] = RT_MIN(puSrc1->ai8[15], puSrc2->ai8[15]);
+    puDst->ai8[16] = RT_MIN(puSrc1->ai8[16], puSrc2->ai8[16]);
+    puDst->ai8[17] = RT_MIN(puSrc1->ai8[17], puSrc2->ai8[17]);
+    puDst->ai8[18] = RT_MIN(puSrc1->ai8[18], puSrc2->ai8[18]);
+    puDst->ai8[19] = RT_MIN(puSrc1->ai8[19], puSrc2->ai8[19]);
+    puDst->ai8[20] = RT_MIN(puSrc1->ai8[20], puSrc2->ai8[20]);
+    puDst->ai8[21] = RT_MIN(puSrc1->ai8[21], puSrc2->ai8[21]);
+    puDst->ai8[22] = RT_MIN(puSrc1->ai8[22], puSrc2->ai8[22]);
+    puDst->ai8[23] = RT_MIN(puSrc1->ai8[23], puSrc2->ai8[23]);
+    puDst->ai8[24] = RT_MIN(puSrc1->ai8[24], puSrc2->ai8[24]);
+    puDst->ai8[25] = RT_MIN(puSrc1->ai8[25], puSrc2->ai8[25]);
+    puDst->ai8[26] = RT_MIN(puSrc1->ai8[26], puSrc2->ai8[26]);
+    puDst->ai8[27] = RT_MIN(puSrc1->ai8[27], puSrc2->ai8[27]);
+    puDst->ai8[28] = RT_MIN(puSrc1->ai8[28], puSrc2->ai8[28]);
+    puDst->ai8[29] = RT_MIN(puSrc1->ai8[29], puSrc2->ai8[29]);
+    puDst->ai8[30] = RT_MIN(puSrc1->ai8[30], puSrc2->ai8[30]);
+    puDst->ai8[31] = RT_MIN(puSrc1->ai8[31], puSrc2->ai8[31]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsw_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->ai16[ 0] = RT_MIN(puSrc1->ai16[ 0], puSrc2->ai16[ 0]);
+    puDst->ai16[ 1] = RT_MIN(puSrc1->ai16[ 1], puSrc2->ai16[ 1]);
+    puDst->ai16[ 2] = RT_MIN(puSrc1->ai16[ 2], puSrc2->ai16[ 2]);
+    puDst->ai16[ 3] = RT_MIN(puSrc1->ai16[ 3], puSrc2->ai16[ 3]);
+    puDst->ai16[ 4] = RT_MIN(puSrc1->ai16[ 4], puSrc2->ai16[ 4]);
+    puDst->ai16[ 5] = RT_MIN(puSrc1->ai16[ 5], puSrc2->ai16[ 5]);
+    puDst->ai16[ 6] = RT_MIN(puSrc1->ai16[ 6], puSrc2->ai16[ 6]);
+    puDst->ai16[ 7] = RT_MIN(puSrc1->ai16[ 7], puSrc2->ai16[ 7]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsw_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->ai16[ 0] = RT_MIN(puSrc1->ai16[ 0], puSrc2->ai16[ 0]);
+    puDst->ai16[ 1] = RT_MIN(puSrc1->ai16[ 1], puSrc2->ai16[ 1]);
+    puDst->ai16[ 2] = RT_MIN(puSrc1->ai16[ 2], puSrc2->ai16[ 2]);
+    puDst->ai16[ 3] = RT_MIN(puSrc1->ai16[ 3], puSrc2->ai16[ 3]);
+    puDst->ai16[ 4] = RT_MIN(puSrc1->ai16[ 4], puSrc2->ai16[ 4]);
+    puDst->ai16[ 5] = RT_MIN(puSrc1->ai16[ 5], puSrc2->ai16[ 5]);
+    puDst->ai16[ 6] = RT_MIN(puSrc1->ai16[ 6], puSrc2->ai16[ 6]);
+    puDst->ai16[ 7] = RT_MIN(puSrc1->ai16[ 7], puSrc2->ai16[ 7]);
+    puDst->ai16[ 8] = RT_MIN(puSrc1->ai16[ 8], puSrc2->ai16[ 8]);
+    puDst->ai16[ 9] = RT_MIN(puSrc1->ai16[ 9], puSrc2->ai16[ 9]);
+    puDst->ai16[10] = RT_MIN(puSrc1->ai16[10], puSrc2->ai16[10]);
+    puDst->ai16[11] = RT_MIN(puSrc1->ai16[11], puSrc2->ai16[11]);
+    puDst->ai16[12] = RT_MIN(puSrc1->ai16[12], puSrc2->ai16[12]);
+    puDst->ai16[13] = RT_MIN(puSrc1->ai16[13], puSrc2->ai16[13]);
+    puDst->ai16[14] = RT_MIN(puSrc1->ai16[14], puSrc2->ai16[14]);
+    puDst->ai16[15] = RT_MIN(puSrc1->ai16[15], puSrc2->ai16[15]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsd_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->ai32[ 0] = RT_MIN(puSrc1->ai32[ 0], puSrc2->ai32[ 0]);
+    puDst->ai32[ 1] = RT_MIN(puSrc1->ai32[ 1], puSrc2->ai32[ 1]);
+    puDst->ai32[ 2] = RT_MIN(puSrc1->ai32[ 2], puSrc2->ai32[ 2]);
+    puDst->ai32[ 3] = RT_MIN(puSrc1->ai32[ 3], puSrc2->ai32[ 3]);
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpminsd_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->ai32[ 0] = RT_MIN(puSrc1->ai32[ 0], puSrc2->ai32[ 0]);
+    puDst->ai32[ 1] = RT_MIN(puSrc1->ai32[ 1], puSrc2->ai32[ 1]);
+    puDst->ai32[ 2] = RT_MIN(puSrc1->ai32[ 2], puSrc2->ai32[ 2]);
+    puDst->ai32[ 3] = RT_MIN(puSrc1->ai32[ 3], puSrc2->ai32[ 3]);
+    puDst->ai32[ 4] = RT_MIN(puSrc1->ai32[ 4], puSrc2->ai32[ 4]);
+    puDst->ai32[ 5] = RT_MIN(puSrc1->ai32[ 5], puSrc2->ai32[ 5]);
+    puDst->ai32[ 6] = RT_MIN(puSrc1->ai32[ 6], puSrc2->ai32[ 6]);
+    puDst->ai32[ 7] = RT_MIN(puSrc1->ai32[ 7], puSrc2->ai32[ 7]);
+    RT_NOREF(pExtState);
+}
+
+
+/*
+ * PAVGB / VPAVGB / PAVGW / VPAVGW
+ */
+#define PAVGB_EXEC(a_Src1, a_Src2) ((uint8_t)(((uint16_t)(a_Src1) + (a_Src2) + 1) >> 1))
+#define PAVGW_EXEC(a_Src1, a_Src2) ((uint16_t)(((uint32_t)(a_Src1) + (a_Src2) + 1) >> 1))
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pavgb_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    uDst.au8[0] = PAVGB_EXEC(uSrc1.au8[0], uSrc2.au8[0]);
+    uDst.au8[1] = PAVGB_EXEC(uSrc1.au8[1], uSrc2.au8[1]);
+    uDst.au8[2] = PAVGB_EXEC(uSrc1.au8[2], uSrc2.au8[2]);
+    uDst.au8[3] = PAVGB_EXEC(uSrc1.au8[3], uSrc2.au8[3]);
+    uDst.au8[4] = PAVGB_EXEC(uSrc1.au8[4], uSrc2.au8[4]);
+    uDst.au8[5] = PAVGB_EXEC(uSrc1.au8[5], uSrc2.au8[5]);
+    uDst.au8[6] = PAVGB_EXEC(uSrc1.au8[6], uSrc2.au8[6]);
+    uDst.au8[7] = PAVGB_EXEC(uSrc1.au8[7], uSrc2.au8[7]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pavgb_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au8[ 0] = PAVGB_EXEC(uSrc1.au8[ 0], puSrc->au8[ 0]);
+    puDst->au8[ 1] = PAVGB_EXEC(uSrc1.au8[ 1], puSrc->au8[ 1]);
+    puDst->au8[ 2] = PAVGB_EXEC(uSrc1.au8[ 2], puSrc->au8[ 2]);
+    puDst->au8[ 3] = PAVGB_EXEC(uSrc1.au8[ 3], puSrc->au8[ 3]);
+    puDst->au8[ 4] = PAVGB_EXEC(uSrc1.au8[ 4], puSrc->au8[ 4]);
+    puDst->au8[ 5] = PAVGB_EXEC(uSrc1.au8[ 5], puSrc->au8[ 5]);
+    puDst->au8[ 6] = PAVGB_EXEC(uSrc1.au8[ 6], puSrc->au8[ 6]);
+    puDst->au8[ 7] = PAVGB_EXEC(uSrc1.au8[ 7], puSrc->au8[ 7]);
+    puDst->au8[ 8] = PAVGB_EXEC(uSrc1.au8[ 8], puSrc->au8[ 8]);
+    puDst->au8[ 9] = PAVGB_EXEC(uSrc1.au8[ 9], puSrc->au8[ 9]);
+    puDst->au8[10] = PAVGB_EXEC(uSrc1.au8[10], puSrc->au8[10]);
+    puDst->au8[11] = PAVGB_EXEC(uSrc1.au8[11], puSrc->au8[11]);
+    puDst->au8[12] = PAVGB_EXEC(uSrc1.au8[12], puSrc->au8[12]);
+    puDst->au8[13] = PAVGB_EXEC(uSrc1.au8[13], puSrc->au8[13]);
+    puDst->au8[14] = PAVGB_EXEC(uSrc1.au8[14], puSrc->au8[14]);
+    puDst->au8[15] = PAVGB_EXEC(uSrc1.au8[15], puSrc->au8[15]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pavgw_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    uDst.au16[0] = PAVGW_EXEC(uSrc1.au16[0], uSrc2.au16[0]);
+    uDst.au16[1] = PAVGW_EXEC(uSrc1.au16[1], uSrc2.au16[1]);
+    uDst.au16[2] = PAVGW_EXEC(uSrc1.au16[2], uSrc2.au16[2]);
+    uDst.au16[3] = PAVGW_EXEC(uSrc1.au16[3], uSrc2.au16[3]);
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pavgw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au16[0] = PAVGW_EXEC(uSrc1.au16[0], puSrc->au16[0]);
+    puDst->au16[1] = PAVGW_EXEC(uSrc1.au16[1], puSrc->au16[1]);
+    puDst->au16[2] = PAVGW_EXEC(uSrc1.au16[2], puSrc->au16[2]);
+    puDst->au16[3] = PAVGW_EXEC(uSrc1.au16[3], puSrc->au16[3]);
+    puDst->au16[4] = PAVGW_EXEC(uSrc1.au16[4], puSrc->au16[4]);
+    puDst->au16[5] = PAVGW_EXEC(uSrc1.au16[5], puSrc->au16[5]);
+    puDst->au16[6] = PAVGW_EXEC(uSrc1.au16[6], puSrc->au16[6]);
+    puDst->au16[7] = PAVGW_EXEC(uSrc1.au16[7], puSrc->au16[7]);
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pavgb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au8[ 0] = PAVGB_EXEC(uSrc1.au8[ 0], puSrc->au8[ 0]);
+    puDst->au8[ 1] = PAVGB_EXEC(uSrc1.au8[ 1], puSrc->au8[ 1]);
+    puDst->au8[ 2] = PAVGB_EXEC(uSrc1.au8[ 2], puSrc->au8[ 2]);
+    puDst->au8[ 3] = PAVGB_EXEC(uSrc1.au8[ 3], puSrc->au8[ 3]);
+    puDst->au8[ 4] = PAVGB_EXEC(uSrc1.au8[ 4], puSrc->au8[ 4]);
+    puDst->au8[ 5] = PAVGB_EXEC(uSrc1.au8[ 5], puSrc->au8[ 5]);
+    puDst->au8[ 6] = PAVGB_EXEC(uSrc1.au8[ 6], puSrc->au8[ 6]);
+    puDst->au8[ 7] = PAVGB_EXEC(uSrc1.au8[ 7], puSrc->au8[ 7]);
+    puDst->au8[ 8] = PAVGB_EXEC(uSrc1.au8[ 8], puSrc->au8[ 8]);
+    puDst->au8[ 9] = PAVGB_EXEC(uSrc1.au8[ 9], puSrc->au8[ 9]);
+    puDst->au8[10] = PAVGB_EXEC(uSrc1.au8[10], puSrc->au8[10]);
+    puDst->au8[11] = PAVGB_EXEC(uSrc1.au8[11], puSrc->au8[11]);
+    puDst->au8[12] = PAVGB_EXEC(uSrc1.au8[12], puSrc->au8[12]);
+    puDst->au8[13] = PAVGB_EXEC(uSrc1.au8[13], puSrc->au8[13]);
+    puDst->au8[14] = PAVGB_EXEC(uSrc1.au8[14], puSrc->au8[14]);
+    puDst->au8[15] = PAVGB_EXEC(uSrc1.au8[15], puSrc->au8[15]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pavgw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au8[ 0] = PAVGW_EXEC(uSrc1.au8[ 0], puSrc->au8[ 0]);
+    puDst->au8[ 1] = PAVGW_EXEC(uSrc1.au8[ 1], puSrc->au8[ 1]);
+    puDst->au8[ 2] = PAVGW_EXEC(uSrc1.au8[ 2], puSrc->au8[ 2]);
+    puDst->au8[ 3] = PAVGW_EXEC(uSrc1.au8[ 3], puSrc->au8[ 3]);
+    puDst->au8[ 4] = PAVGW_EXEC(uSrc1.au8[ 4], puSrc->au8[ 4]);
+    puDst->au8[ 5] = PAVGW_EXEC(uSrc1.au8[ 5], puSrc->au8[ 5]);
+    puDst->au8[ 6] = PAVGW_EXEC(uSrc1.au8[ 6], puSrc->au8[ 6]);
+    puDst->au8[ 7] = PAVGW_EXEC(uSrc1.au8[ 7], puSrc->au8[ 7]);
+    puDst->au8[ 8] = PAVGW_EXEC(uSrc1.au8[ 8], puSrc->au8[ 8]);
+    puDst->au8[ 9] = PAVGW_EXEC(uSrc1.au8[ 9], puSrc->au8[ 9]);
+    puDst->au8[10] = PAVGW_EXEC(uSrc1.au8[10], puSrc->au8[10]);
+    puDst->au8[11] = PAVGW_EXEC(uSrc1.au8[11], puSrc->au8[11]);
+    puDst->au8[12] = PAVGW_EXEC(uSrc1.au8[12], puSrc->au8[12]);
+    puDst->au8[13] = PAVGW_EXEC(uSrc1.au8[13], puSrc->au8[13]);
+    puDst->au8[14] = PAVGW_EXEC(uSrc1.au8[14], puSrc->au8[14]);
+    puDst->au8[15] = PAVGW_EXEC(uSrc1.au8[15], puSrc->au8[15]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpavgb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->au8[ 0] = PAVGB_EXEC(puSrc1->au8[ 0], puSrc2->au8[ 0]);
+    puDst->au8[ 1] = PAVGB_EXEC(puSrc1->au8[ 1], puSrc2->au8[ 1]);
+    puDst->au8[ 2] = PAVGB_EXEC(puSrc1->au8[ 2], puSrc2->au8[ 2]);
+    puDst->au8[ 3] = PAVGB_EXEC(puSrc1->au8[ 3], puSrc2->au8[ 3]);
+    puDst->au8[ 4] = PAVGB_EXEC(puSrc1->au8[ 4], puSrc2->au8[ 4]);
+    puDst->au8[ 5] = PAVGB_EXEC(puSrc1->au8[ 5], puSrc2->au8[ 5]);
+    puDst->au8[ 6] = PAVGB_EXEC(puSrc1->au8[ 6], puSrc2->au8[ 6]);
+    puDst->au8[ 7] = PAVGB_EXEC(puSrc1->au8[ 7], puSrc2->au8[ 7]);
+    puDst->au8[ 8] = PAVGB_EXEC(puSrc1->au8[ 8], puSrc2->au8[ 8]);
+    puDst->au8[ 9] = PAVGB_EXEC(puSrc1->au8[ 9], puSrc2->au8[ 9]);
+    puDst->au8[10] = PAVGB_EXEC(puSrc1->au8[10], puSrc2->au8[10]);
+    puDst->au8[11] = PAVGB_EXEC(puSrc1->au8[11], puSrc2->au8[11]);
+    puDst->au8[12] = PAVGB_EXEC(puSrc1->au8[12], puSrc2->au8[12]);
+    puDst->au8[13] = PAVGB_EXEC(puSrc1->au8[13], puSrc2->au8[13]);
+    puDst->au8[14] = PAVGB_EXEC(puSrc1->au8[14], puSrc2->au8[14]);
+    puDst->au8[15] = PAVGB_EXEC(puSrc1->au8[15], puSrc2->au8[15]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpavgb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->au8[ 0] = PAVGB_EXEC(puSrc1->au8[ 0], puSrc2->au8[ 0]);
+    puDst->au8[ 1] = PAVGB_EXEC(puSrc1->au8[ 1], puSrc2->au8[ 1]);
+    puDst->au8[ 2] = PAVGB_EXEC(puSrc1->au8[ 2], puSrc2->au8[ 2]);
+    puDst->au8[ 3] = PAVGB_EXEC(puSrc1->au8[ 3], puSrc2->au8[ 3]);
+    puDst->au8[ 4] = PAVGB_EXEC(puSrc1->au8[ 4], puSrc2->au8[ 4]);
+    puDst->au8[ 5] = PAVGB_EXEC(puSrc1->au8[ 5], puSrc2->au8[ 5]);
+    puDst->au8[ 6] = PAVGB_EXEC(puSrc1->au8[ 6], puSrc2->au8[ 6]);
+    puDst->au8[ 7] = PAVGB_EXEC(puSrc1->au8[ 7], puSrc2->au8[ 7]);
+    puDst->au8[ 8] = PAVGB_EXEC(puSrc1->au8[ 8], puSrc2->au8[ 8]);
+    puDst->au8[ 9] = PAVGB_EXEC(puSrc1->au8[ 9], puSrc2->au8[ 9]);
+    puDst->au8[10] = PAVGB_EXEC(puSrc1->au8[10], puSrc2->au8[10]);
+    puDst->au8[11] = PAVGB_EXEC(puSrc1->au8[11], puSrc2->au8[11]);
+    puDst->au8[12] = PAVGB_EXEC(puSrc1->au8[12], puSrc2->au8[12]);
+    puDst->au8[13] = PAVGB_EXEC(puSrc1->au8[13], puSrc2->au8[13]);
+    puDst->au8[14] = PAVGB_EXEC(puSrc1->au8[14], puSrc2->au8[14]);
+    puDst->au8[15] = PAVGB_EXEC(puSrc1->au8[15], puSrc2->au8[15]);
+    puDst->au8[16] = PAVGB_EXEC(puSrc1->au8[16], puSrc2->au8[16]);
+    puDst->au8[17] = PAVGB_EXEC(puSrc1->au8[17], puSrc2->au8[17]);
+    puDst->au8[18] = PAVGB_EXEC(puSrc1->au8[18], puSrc2->au8[18]);
+    puDst->au8[19] = PAVGB_EXEC(puSrc1->au8[19], puSrc2->au8[19]);
+    puDst->au8[20] = PAVGB_EXEC(puSrc1->au8[20], puSrc2->au8[20]);
+    puDst->au8[21] = PAVGB_EXEC(puSrc1->au8[21], puSrc2->au8[21]);
+    puDst->au8[22] = PAVGB_EXEC(puSrc1->au8[22], puSrc2->au8[22]);
+    puDst->au8[23] = PAVGB_EXEC(puSrc1->au8[23], puSrc2->au8[23]);
+    puDst->au8[24] = PAVGB_EXEC(puSrc1->au8[24], puSrc2->au8[24]);
+    puDst->au8[25] = PAVGB_EXEC(puSrc1->au8[25], puSrc2->au8[25]);
+    puDst->au8[26] = PAVGB_EXEC(puSrc1->au8[26], puSrc2->au8[26]);
+    puDst->au8[27] = PAVGB_EXEC(puSrc1->au8[27], puSrc2->au8[27]);
+    puDst->au8[28] = PAVGB_EXEC(puSrc1->au8[28], puSrc2->au8[28]);
+    puDst->au8[29] = PAVGB_EXEC(puSrc1->au8[29], puSrc2->au8[29]);
+    puDst->au8[30] = PAVGB_EXEC(puSrc1->au8[30], puSrc2->au8[30]);
+    puDst->au8[31] = PAVGB_EXEC(puSrc1->au8[31], puSrc2->au8[31]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpavgw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    puDst->au16[ 0] = PAVGW_EXEC(puSrc1->au16[ 0], puSrc2->au16[ 0]);
+    puDst->au16[ 1] = PAVGW_EXEC(puSrc1->au16[ 1], puSrc2->au16[ 1]);
+    puDst->au16[ 2] = PAVGW_EXEC(puSrc1->au16[ 2], puSrc2->au16[ 2]);
+    puDst->au16[ 3] = PAVGW_EXEC(puSrc1->au16[ 3], puSrc2->au16[ 3]);
+    puDst->au16[ 4] = PAVGW_EXEC(puSrc1->au16[ 4], puSrc2->au16[ 4]);
+    puDst->au16[ 5] = PAVGW_EXEC(puSrc1->au16[ 5], puSrc2->au16[ 5]);
+    puDst->au16[ 6] = PAVGW_EXEC(puSrc1->au16[ 6], puSrc2->au16[ 6]);
+    puDst->au16[ 7] = PAVGW_EXEC(puSrc1->au16[ 7], puSrc2->au16[ 7]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpavgw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    puDst->au16[ 0] = PAVGW_EXEC(puSrc1->au16[ 0], puSrc2->au16[ 0]);
+    puDst->au16[ 1] = PAVGW_EXEC(puSrc1->au16[ 1], puSrc2->au16[ 1]);
+    puDst->au16[ 2] = PAVGW_EXEC(puSrc1->au16[ 2], puSrc2->au16[ 2]);
+    puDst->au16[ 3] = PAVGW_EXEC(puSrc1->au16[ 3], puSrc2->au16[ 3]);
+    puDst->au16[ 4] = PAVGW_EXEC(puSrc1->au16[ 4], puSrc2->au16[ 4]);
+    puDst->au16[ 5] = PAVGW_EXEC(puSrc1->au16[ 5], puSrc2->au16[ 5]);
+    puDst->au16[ 6] = PAVGW_EXEC(puSrc1->au16[ 6], puSrc2->au16[ 6]);
+    puDst->au16[ 7] = PAVGW_EXEC(puSrc1->au16[ 7], puSrc2->au16[ 7]);
+    puDst->au16[ 8] = PAVGW_EXEC(puSrc1->au16[ 8], puSrc2->au16[ 8]);
+    puDst->au16[ 9] = PAVGW_EXEC(puSrc1->au16[ 9], puSrc2->au16[ 9]);
+    puDst->au16[10] = PAVGW_EXEC(puSrc1->au16[10], puSrc2->au16[10]);
+    puDst->au16[11] = PAVGW_EXEC(puSrc1->au16[11], puSrc2->au16[11]);
+    puDst->au16[12] = PAVGW_EXEC(puSrc1->au16[12], puSrc2->au16[12]);
+    puDst->au16[13] = PAVGW_EXEC(puSrc1->au16[13], puSrc2->au16[13]);
+    puDst->au16[14] = PAVGW_EXEC(puSrc1->au16[14], puSrc2->au16[14]);
+    puDst->au16[15] = PAVGW_EXEC(puSrc1->au16[15], puSrc2->au16[15]);
+}
+
+#undef PAVGB_EXEC
+#undef PAVGW_EXEC
+
+
+/*
+ * PMOVMSKB / VPMOVMSKB
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u64,(uint64_t *pu64Dst, uint64_t const *pu64Src))
+{
+    /* The the most signficant bit from each byte and store them in the given general purpose register. */
+    uint64_t const uSrc = *pu64Src;
+    *pu64Dst = ((uSrc >> ( 7-0)) & RT_BIT_64(0))
+             | ((uSrc >> (15-1)) & RT_BIT_64(1))
+             | ((uSrc >> (23-2)) & RT_BIT_64(2))
+             | ((uSrc >> (31-3)) & RT_BIT_64(3))
+             | ((uSrc >> (39-4)) & RT_BIT_64(4))
+             | ((uSrc >> (47-5)) & RT_BIT_64(5))
+             | ((uSrc >> (55-6)) & RT_BIT_64(6))
+             | ((uSrc >> (63-7)) & RT_BIT_64(7));
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u128,(uint64_t *pu64Dst, PCRTUINT128U pu128Src))
+{
+    /* The the most signficant bit from each byte and store them in the given general purpose register. */
+    uint64_t const uSrc0 = pu128Src->QWords.qw0;
+    uint64_t const uSrc1 = pu128Src->QWords.qw1;
+    *pu64Dst = ((uSrc0 >> ( 7-0))        & RT_BIT_64(0))
+             | ((uSrc0 >> (15-1))        & RT_BIT_64(1))
+             | ((uSrc0 >> (23-2))        & RT_BIT_64(2))
+             | ((uSrc0 >> (31-3))        & RT_BIT_64(3))
+             | ((uSrc0 >> (39-4))        & RT_BIT_64(4))
+             | ((uSrc0 >> (47-5))        & RT_BIT_64(5))
+             | ((uSrc0 >> (55-6))        & RT_BIT_64(6))
+             | ((uSrc0 >> (63-7))        & RT_BIT_64(7))
+             | ((uSrc1 << (1 /*7-8*/))   & RT_BIT_64(8))
+             | ((uSrc1 >> (15-9))        & RT_BIT_64(9))
+             | ((uSrc1 >> (23-10))       & RT_BIT_64(10))
+             | ((uSrc1 >> (31-11))       & RT_BIT_64(11))
+             | ((uSrc1 >> (39-12))       & RT_BIT_64(12))
+             | ((uSrc1 >> (47-13))       & RT_BIT_64(13))
+             | ((uSrc1 >> (55-14))       & RT_BIT_64(14))
+             | ((uSrc1 >> (63-15))       & RT_BIT_64(15));
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovmskb_u256_fallback,(uint64_t *pu64Dst, PCRTUINT256U puSrc))
+{
+    /* The the most signficant bit from each byte and store them in the given general purpose register. */
+    uint64_t const uSrc0 = puSrc->QWords.qw0;
+    uint64_t const uSrc1 = puSrc->QWords.qw1;
+    uint64_t const uSrc2 = puSrc->QWords.qw2;
+    uint64_t const uSrc3 = puSrc->QWords.qw3;
+    *pu64Dst = ((uSrc0 >> ( 7-0))            & RT_BIT_64(0))
+             | ((uSrc0 >> (15-1))            & RT_BIT_64(1))
+             | ((uSrc0 >> (23-2))            & RT_BIT_64(2))
+             | ((uSrc0 >> (31-3))            & RT_BIT_64(3))
+             | ((uSrc0 >> (39-4))            & RT_BIT_64(4))
+             | ((uSrc0 >> (47-5))            & RT_BIT_64(5))
+             | ((uSrc0 >> (55-6))            & RT_BIT_64(6))
+             | ((uSrc0 >> (63-7))            & RT_BIT_64(7))
+             | ((uSrc1 << (1 /*7-8*/))       & RT_BIT_64(8))
+             | ((uSrc1 >> (15-9))            & RT_BIT_64(9))
+             | ((uSrc1 >> (23-10))           & RT_BIT_64(10))
+             | ((uSrc1 >> (31-11))           & RT_BIT_64(11))
+             | ((uSrc1 >> (39-12))           & RT_BIT_64(12))
+             | ((uSrc1 >> (47-13))           & RT_BIT_64(13))
+             | ((uSrc1 >> (55-14))           & RT_BIT_64(14))
+             | ((uSrc1 >> (63-15))           & RT_BIT_64(15))
+             | ((uSrc2 << (9 /* 7-16*/))     & RT_BIT_64(16))
+             | ((uSrc2 << (2 /*15-17*/))     & RT_BIT_64(17))
+             | ((uSrc2 >> (23-18))           & RT_BIT_64(18))
+             | ((uSrc2 >> (31-19))           & RT_BIT_64(19))
+             | ((uSrc2 >> (39-20))           & RT_BIT_64(20))
+             | ((uSrc2 >> (47-21))           & RT_BIT_64(21))
+             | ((uSrc2 >> (55-22))           & RT_BIT_64(22))
+             | ((uSrc2 >> (63-23))           & RT_BIT_64(23))
+             | ((uSrc3 << (17 /* 7-24*/))    & RT_BIT_64(24))
+             | ((uSrc3 << (10 /*15-25*/))    & RT_BIT_64(25))
+             | ((uSrc3 << (3  /*23-26*/))    & RT_BIT_64(26))
+             | ((uSrc3 >> (31-27))           & RT_BIT_64(27))
+             | ((uSrc3 >> (39-28))           & RT_BIT_64(28))
+             | ((uSrc3 >> (47-29))           & RT_BIT_64(29))
+             | ((uSrc3 >> (55-30))           & RT_BIT_64(30))
+             | ((uSrc3 >> (63-31))           & RT_BIT_64(31));
+}
+
+
+/*
+ * [V]PSHUFB
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pshufb_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc    = { *puSrc };
+    RTUINT64U const uDstIn  = { *puDst };
+    ASMCompilerBarrier();
+    RTUINT64U       uDstOut = { 0 };
+    for (unsigned iByte = 0; iByte < RT_ELEMENTS(uDstIn.au8); iByte++)
+    {
+        uint8_t idxSrc = uSrc.au8[iByte];
+        if (!(idxSrc & 0x80))
+            uDstOut.au8[iByte] = uDstIn.au8[idxSrc & 7];
+    }
+    *puDst = uDstOut.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pshufb_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc    = *puSrc;
+    RTUINT128U const uDstIn  = *puDst;
+    ASMCompilerBarrier();
+    puDst->au64[0] = 0;
+    puDst->au64[1] = 0;
+    for (unsigned iByte = 0; iByte < RT_ELEMENTS(puDst->au8); iByte++)
+    {
+        uint8_t idxSrc = uSrc.au8[iByte];
+        if (!(idxSrc & 0x80))
+            puDst->au8[iByte] = uDstIn.au8[idxSrc & 15];
+    }
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpshufb_u128_fallback,(PX86XSAVEAREA pExtState, PRTUINT128U puDst,
+                                                        PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc1 = *puSrc1; /* could be same as puDst */
+    RTUINT128U const uSrc2 = *puSrc2; /* could be same as puDst */
+    ASMCompilerBarrier();
+    puDst->au64[0] = 0;
+    puDst->au64[1] = 0;
+    for (unsigned iByte = 0; iByte < 16; iByte++)
+    {
+        uint8_t idxSrc = uSrc2.au8[iByte];
+        if (!(idxSrc & 0x80))
+            puDst->au8[iByte] = uSrc1.au8[(idxSrc & 15)];
+    }
+    RT_NOREF(pExtState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpshufb_u256_fallback,(PX86XSAVEAREA pExtState, PRTUINT256U puDst,
+                                                        PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc1 = *puSrc1; /* could be same as puDst */
+    RTUINT256U const uSrc2 = *puSrc2; /* could be same as puDst */
+    ASMCompilerBarrier();
+    puDst->au64[0] = 0;
+    puDst->au64[1] = 0;
+    puDst->au64[2] = 0;
+    puDst->au64[3] = 0;
+    for (unsigned iByte = 0; iByte < 16; iByte++)
+    {
+        uint8_t idxSrc = uSrc2.au8[iByte];
+        if (!(idxSrc & 0x80))
+            puDst->au8[iByte] = uSrc1.au8[(idxSrc & 15)];
+    }
+    for (unsigned iByte = 16; iByte < RT_ELEMENTS(puDst->au8); iByte++)
+    {
+        uint8_t idxSrc = uSrc2.au8[iByte];
+        if (!(idxSrc & 0x80))
+            puDst->au8[iByte] = uSrc1.au8[(idxSrc & 15) + 16]; /* baka intel */
+    }
+    RT_NOREF(pExtState);
+}
+
+
+/*
+ * PSHUFW, [V]PSHUFHW, [V]PSHUFLW, [V]PSHUFD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pshufw_u64,(uint64_t *puDst, uint64_t const *puSrc, uint8_t bEvil))
+{
+    uint64_t const uSrc = *puSrc;
+    ASMCompilerBarrier();
+    *puDst = RT_MAKE_U64_FROM_U16(uSrc >> (( bEvil       & 3) * 16),
+                                  uSrc >> (((bEvil >> 2) & 3) * 16),
+                                  uSrc >> (((bEvil >> 4) & 3) * 16),
+                                  uSrc >> (((bEvil >> 6) & 3) * 16));
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pshufhw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    puDst->QWords.qw0   = puSrc->QWords.qw0;
+    uint64_t const uSrc = puSrc->QWords.qw1;
+    ASMCompilerBarrier();
+    puDst->QWords.qw1   = RT_MAKE_U64_FROM_U16(uSrc >> (( bEvil       & 3) * 16),
+                                               uSrc >> (((bEvil >> 2) & 3) * 16),
+                                               uSrc >> (((bEvil >> 4) & 3) * 16),
+                                               uSrc >> (((bEvil >> 6) & 3) * 16));
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpshufhw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc, uint8_t bEvil))
+{
+    puDst->QWords.qw0    = puSrc->QWords.qw0;
+    uint64_t const uSrc1 = puSrc->QWords.qw1;
+    puDst->QWords.qw2    = puSrc->QWords.qw2;
+    uint64_t const uSrc3 = puSrc->QWords.qw3;
+    ASMCompilerBarrier();
+    puDst->QWords.qw1    = RT_MAKE_U64_FROM_U16(uSrc1 >> (( bEvil       & 3) * 16),
+                                                uSrc1 >> (((bEvil >> 2) & 3) * 16),
+                                                uSrc1 >> (((bEvil >> 4) & 3) * 16),
+                                                uSrc1 >> (((bEvil >> 6) & 3) * 16));
+    puDst->QWords.qw3    = RT_MAKE_U64_FROM_U16(uSrc3 >> (( bEvil       & 3) * 16),
+                                                uSrc3 >> (((bEvil >> 2) & 3) * 16),
+                                                uSrc3 >> (((bEvil >> 4) & 3) * 16),
+                                                uSrc3 >> (((bEvil >> 6) & 3) * 16));
+}
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_pshuflw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    puDst->QWords.qw1   = puSrc->QWords.qw1;
+    uint64_t const uSrc = puSrc->QWords.qw0;
+    ASMCompilerBarrier();
+    puDst->QWords.qw0   = RT_MAKE_U64_FROM_U16(uSrc >> (( bEvil       & 3) * 16),
+                                               uSrc >> (((bEvil >> 2) & 3) * 16),
+                                               uSrc >> (((bEvil >> 4) & 3) * 16),
+                                               uSrc >> (((bEvil >> 6) & 3) * 16));
+
+}
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpshuflw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc, uint8_t bEvil))
+{
+    puDst->QWords.qw3    = puSrc->QWords.qw3;
+    uint64_t const uSrc2 = puSrc->QWords.qw2;
+    puDst->QWords.qw1    = puSrc->QWords.qw1;
+    uint64_t const uSrc0 = puSrc->QWords.qw0;
+    ASMCompilerBarrier();
+    puDst->QWords.qw0   = RT_MAKE_U64_FROM_U16(uSrc0 >> (( bEvil       & 3) * 16),
+                                               uSrc0 >> (((bEvil >> 2) & 3) * 16),
+                                               uSrc0 >> (((bEvil >> 4) & 3) * 16),
+                                               uSrc0 >> (((bEvil >> 6) & 3) * 16));
+    puDst->QWords.qw2   = RT_MAKE_U64_FROM_U16(uSrc2 >> (( bEvil       & 3) * 16),
+                                               uSrc2 >> (((bEvil >> 2) & 3) * 16),
+                                               uSrc2 >> (((bEvil >> 4) & 3) * 16),
+                                               uSrc2 >> (((bEvil >> 6) & 3) * 16));
+
+}
+
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_pshufd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    RTUINT128U const uSrc = *puSrc;
+    ASMCompilerBarrier();
+    puDst->au32[0] = uSrc.au32[bEvil & 3];
+    puDst->au32[1] = uSrc.au32[(bEvil >> 2) & 3];
+    puDst->au32[2] = uSrc.au32[(bEvil >> 4) & 3];
+    puDst->au32[3] = uSrc.au32[(bEvil >> 6) & 3];
+}
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpshufd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc, uint8_t bEvil))
+{
+    RTUINT256U const uSrc = *puSrc;
+    ASMCompilerBarrier();
+    puDst->au128[0].au32[0] = uSrc.au128[0].au32[bEvil & 3];
+    puDst->au128[0].au32[1] = uSrc.au128[0].au32[(bEvil >> 2) & 3];
+    puDst->au128[0].au32[2] = uSrc.au128[0].au32[(bEvil >> 4) & 3];
+    puDst->au128[0].au32[3] = uSrc.au128[0].au32[(bEvil >> 6) & 3];
+    puDst->au128[1].au32[0] = uSrc.au128[1].au32[bEvil & 3];
+    puDst->au128[1].au32[1] = uSrc.au128[1].au32[(bEvil >> 2) & 3];
+    puDst->au128[1].au32[2] = uSrc.au128[1].au32[(bEvil >> 4) & 3];
+    puDst->au128[1].au32[3] = uSrc.au128[1].au32[(bEvil >> 6) & 3];
+}
+
+
+/*
+ * PUNPCKHBW - high bytes -> words
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhbw_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc2 = { *puSrc };
+    RTUINT64U const uSrc1 = { *puDst };
+    ASMCompilerBarrier();
+    RTUINT64U uDstOut;
+    uDstOut.au8[0] = uSrc1.au8[4];
+    uDstOut.au8[1] = uSrc2.au8[4];
+    uDstOut.au8[2] = uSrc1.au8[5];
+    uDstOut.au8[3] = uSrc2.au8[5];
+    uDstOut.au8[4] = uSrc1.au8[6];
+    uDstOut.au8[5] = uSrc2.au8[6];
+    uDstOut.au8[6] = uSrc1.au8[7];
+    uDstOut.au8[7] = uSrc2.au8[7];
+    *puDst = uDstOut.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhbw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au8[ 0] = uSrc1.au8[ 8];
+    uDstOut.au8[ 1] = uSrc2.au8[ 8];
+    uDstOut.au8[ 2] = uSrc1.au8[ 9];
+    uDstOut.au8[ 3] = uSrc2.au8[ 9];
+    uDstOut.au8[ 4] = uSrc1.au8[10];
+    uDstOut.au8[ 5] = uSrc2.au8[10];
+    uDstOut.au8[ 6] = uSrc1.au8[11];
+    uDstOut.au8[ 7] = uSrc2.au8[11];
+    uDstOut.au8[ 8] = uSrc1.au8[12];
+    uDstOut.au8[ 9] = uSrc2.au8[12];
+    uDstOut.au8[10] = uSrc1.au8[13];
+    uDstOut.au8[11] = uSrc2.au8[13];
+    uDstOut.au8[12] = uSrc1.au8[14];
+    uDstOut.au8[13] = uSrc2.au8[14];
+    uDstOut.au8[14] = uSrc1.au8[15];
+    uDstOut.au8[15] = uSrc2.au8[15];
+    *puDst = uDstOut;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhbw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au8[ 0] = uSrc1.au8[ 8];
+    uDstOut.au8[ 1] = uSrc2.au8[ 8];
+    uDstOut.au8[ 2] = uSrc1.au8[ 9];
+    uDstOut.au8[ 3] = uSrc2.au8[ 9];
+    uDstOut.au8[ 4] = uSrc1.au8[10];
+    uDstOut.au8[ 5] = uSrc2.au8[10];
+    uDstOut.au8[ 6] = uSrc1.au8[11];
+    uDstOut.au8[ 7] = uSrc2.au8[11];
+    uDstOut.au8[ 8] = uSrc1.au8[12];
+    uDstOut.au8[ 9] = uSrc2.au8[12];
+    uDstOut.au8[10] = uSrc1.au8[13];
+    uDstOut.au8[11] = uSrc2.au8[13];
+    uDstOut.au8[12] = uSrc1.au8[14];
+    uDstOut.au8[13] = uSrc2.au8[14];
+    uDstOut.au8[14] = uSrc1.au8[15];
+    uDstOut.au8[15] = uSrc2.au8[15];
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au8[ 0] = uSrc1.au8[ 8];
+    uDstOut.au8[ 1] = uSrc2.au8[ 8];
+    uDstOut.au8[ 2] = uSrc1.au8[ 9];
+    uDstOut.au8[ 3] = uSrc2.au8[ 9];
+    uDstOut.au8[ 4] = uSrc1.au8[10];
+    uDstOut.au8[ 5] = uSrc2.au8[10];
+    uDstOut.au8[ 6] = uSrc1.au8[11];
+    uDstOut.au8[ 7] = uSrc2.au8[11];
+    uDstOut.au8[ 8] = uSrc1.au8[12];
+    uDstOut.au8[ 9] = uSrc2.au8[12];
+    uDstOut.au8[10] = uSrc1.au8[13];
+    uDstOut.au8[11] = uSrc2.au8[13];
+    uDstOut.au8[12] = uSrc1.au8[14];
+    uDstOut.au8[13] = uSrc2.au8[14];
+    uDstOut.au8[14] = uSrc1.au8[15];
+    uDstOut.au8[15] = uSrc2.au8[15];
+    /* As usual, the upper 128-bits are treated like a parallel register to the lower half. */
+    uDstOut.au8[16] = uSrc1.au8[24];
+    uDstOut.au8[17] = uSrc2.au8[24];
+    uDstOut.au8[18] = uSrc1.au8[25];
+    uDstOut.au8[19] = uSrc2.au8[25];
+    uDstOut.au8[20] = uSrc1.au8[26];
+    uDstOut.au8[21] = uSrc2.au8[26];
+    uDstOut.au8[22] = uSrc1.au8[27];
+    uDstOut.au8[23] = uSrc2.au8[27];
+    uDstOut.au8[24] = uSrc1.au8[28];
+    uDstOut.au8[25] = uSrc2.au8[28];
+    uDstOut.au8[26] = uSrc1.au8[29];
+    uDstOut.au8[27] = uSrc2.au8[29];
+    uDstOut.au8[28] = uSrc1.au8[30];
+    uDstOut.au8[29] = uSrc2.au8[30];
+    uDstOut.au8[30] = uSrc1.au8[31];
+    uDstOut.au8[31] = uSrc2.au8[31];
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PUNPCKHBW - high words -> dwords
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhwd_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc2 = { *puSrc };
+    RTUINT64U const uSrc1 = { *puDst };
+    ASMCompilerBarrier();
+    RTUINT64U uDstOut;
+    uDstOut.au16[0] = uSrc1.au16[2];
+    uDstOut.au16[1] = uSrc2.au16[2];
+    uDstOut.au16[2] = uSrc1.au16[3];
+    uDstOut.au16[3] = uSrc2.au16[3];
+    *puDst = uDstOut.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhwd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au16[0] = uSrc1.au16[4];
+    uDstOut.au16[1] = uSrc2.au16[4];
+    uDstOut.au16[2] = uSrc1.au16[5];
+    uDstOut.au16[3] = uSrc2.au16[5];
+    uDstOut.au16[4] = uSrc1.au16[6];
+    uDstOut.au16[5] = uSrc2.au16[6];
+    uDstOut.au16[6] = uSrc1.au16[7];
+    uDstOut.au16[7] = uSrc2.au16[7];
+    *puDst = uDstOut;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhwd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au16[0] = uSrc1.au16[4];
+    uDstOut.au16[1] = uSrc2.au16[4];
+    uDstOut.au16[2] = uSrc1.au16[5];
+    uDstOut.au16[3] = uSrc2.au16[5];
+    uDstOut.au16[4] = uSrc1.au16[6];
+    uDstOut.au16[5] = uSrc2.au16[6];
+    uDstOut.au16[6] = uSrc1.au16[7];
+    uDstOut.au16[7] = uSrc2.au16[7];
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhwd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au16[0]  = uSrc1.au16[4];
+    uDstOut.au16[1]  = uSrc2.au16[4];
+    uDstOut.au16[2]  = uSrc1.au16[5];
+    uDstOut.au16[3]  = uSrc2.au16[5];
+    uDstOut.au16[4]  = uSrc1.au16[6];
+    uDstOut.au16[5]  = uSrc2.au16[6];
+    uDstOut.au16[6]  = uSrc1.au16[7];
+    uDstOut.au16[7]  = uSrc2.au16[7];
+
+    uDstOut.au16[8]  = uSrc1.au16[12];
+    uDstOut.au16[9]  = uSrc2.au16[12];
+    uDstOut.au16[10] = uSrc1.au16[13];
+    uDstOut.au16[11] = uSrc2.au16[13];
+    uDstOut.au16[12] = uSrc1.au16[14];
+    uDstOut.au16[13] = uSrc2.au16[14];
+    uDstOut.au16[14] = uSrc1.au16[15];
+    uDstOut.au16[15] = uSrc2.au16[15];
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PUNPCKHBW - high dwords -> qword(s)
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhdq_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc2 = { *puSrc };
+    RTUINT64U const uSrc1 = { *puDst };
+    ASMCompilerBarrier();
+    RTUINT64U uDstOut;
+    uDstOut.au32[0] = uSrc1.au32[1];
+    uDstOut.au32[1] = uSrc2.au32[1];
+    *puDst = uDstOut.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhdq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au32[0] = uSrc1.au32[2];
+    uDstOut.au32[1] = uSrc2.au32[2];
+    uDstOut.au32[2] = uSrc1.au32[3];
+    uDstOut.au32[3] = uSrc2.au32[3];
+    *puDst = uDstOut;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhdq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au32[0] = uSrc1.au32[2];
+    uDstOut.au32[1] = uSrc2.au32[2];
+    uDstOut.au32[2] = uSrc1.au32[3];
+    uDstOut.au32[3] = uSrc2.au32[3];
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au32[0] = uSrc1.au32[2];
+    uDstOut.au32[1] = uSrc2.au32[2];
+    uDstOut.au32[2] = uSrc1.au32[3];
+    uDstOut.au32[3] = uSrc2.au32[3];
+
+    uDstOut.au32[4] = uSrc1.au32[6];
+    uDstOut.au32[5] = uSrc2.au32[6];
+    uDstOut.au32[6] = uSrc1.au32[7];
+    uDstOut.au32[7] = uSrc2.au32[7];
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PUNPCKHQDQ -> High qwords -> double qword(s).
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhqdq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au64[0] = uSrc1.au64[1];
+    uDstOut.au64[1] = uSrc2.au64[1];
+    *puDst = uDstOut;
+}
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhqdq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au64[0] = uSrc1.au64[1];
+    uDstOut.au64[1] = uSrc2.au64[1];
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckhqdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au64[0] = uSrc1.au64[1];
+    uDstOut.au64[1] = uSrc2.au64[1];
+
+    uDstOut.au64[2] = uSrc1.au64[3];
+    uDstOut.au64[3] = uSrc2.au64[3];
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PUNPCKLBW - low bytes -> words
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklbw_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc2 = { *puSrc };
+    RTUINT64U const uSrc1 = { *puDst };
+    ASMCompilerBarrier();
+    RTUINT64U uDstOut;
+    uDstOut.au8[0] = uSrc1.au8[0];
+    uDstOut.au8[1] = uSrc2.au8[0];
+    uDstOut.au8[2] = uSrc1.au8[1];
+    uDstOut.au8[3] = uSrc2.au8[1];
+    uDstOut.au8[4] = uSrc1.au8[2];
+    uDstOut.au8[5] = uSrc2.au8[2];
+    uDstOut.au8[6] = uSrc1.au8[3];
+    uDstOut.au8[7] = uSrc2.au8[3];
+    *puDst = uDstOut.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklbw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au8[ 0] = uSrc1.au8[0];
+    uDstOut.au8[ 1] = uSrc2.au8[0];
+    uDstOut.au8[ 2] = uSrc1.au8[1];
+    uDstOut.au8[ 3] = uSrc2.au8[1];
+    uDstOut.au8[ 4] = uSrc1.au8[2];
+    uDstOut.au8[ 5] = uSrc2.au8[2];
+    uDstOut.au8[ 6] = uSrc1.au8[3];
+    uDstOut.au8[ 7] = uSrc2.au8[3];
+    uDstOut.au8[ 8] = uSrc1.au8[4];
+    uDstOut.au8[ 9] = uSrc2.au8[4];
+    uDstOut.au8[10] = uSrc1.au8[5];
+    uDstOut.au8[11] = uSrc2.au8[5];
+    uDstOut.au8[12] = uSrc1.au8[6];
+    uDstOut.au8[13] = uSrc2.au8[6];
+    uDstOut.au8[14] = uSrc1.au8[7];
+    uDstOut.au8[15] = uSrc2.au8[7];
+    *puDst = uDstOut;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklbw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au8[ 0] = uSrc1.au8[0];
+    uDstOut.au8[ 1] = uSrc2.au8[0];
+    uDstOut.au8[ 2] = uSrc1.au8[1];
+    uDstOut.au8[ 3] = uSrc2.au8[1];
+    uDstOut.au8[ 4] = uSrc1.au8[2];
+    uDstOut.au8[ 5] = uSrc2.au8[2];
+    uDstOut.au8[ 6] = uSrc1.au8[3];
+    uDstOut.au8[ 7] = uSrc2.au8[3];
+    uDstOut.au8[ 8] = uSrc1.au8[4];
+    uDstOut.au8[ 9] = uSrc2.au8[4];
+    uDstOut.au8[10] = uSrc1.au8[5];
+    uDstOut.au8[11] = uSrc2.au8[5];
+    uDstOut.au8[12] = uSrc1.au8[6];
+    uDstOut.au8[13] = uSrc2.au8[6];
+    uDstOut.au8[14] = uSrc1.au8[7];
+    uDstOut.au8[15] = uSrc2.au8[7];
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au8[ 0] = uSrc1.au8[0];
+    uDstOut.au8[ 1] = uSrc2.au8[0];
+    uDstOut.au8[ 2] = uSrc1.au8[1];
+    uDstOut.au8[ 3] = uSrc2.au8[1];
+    uDstOut.au8[ 4] = uSrc1.au8[2];
+    uDstOut.au8[ 5] = uSrc2.au8[2];
+    uDstOut.au8[ 6] = uSrc1.au8[3];
+    uDstOut.au8[ 7] = uSrc2.au8[3];
+    uDstOut.au8[ 8] = uSrc1.au8[4];
+    uDstOut.au8[ 9] = uSrc2.au8[4];
+    uDstOut.au8[10] = uSrc1.au8[5];
+    uDstOut.au8[11] = uSrc2.au8[5];
+    uDstOut.au8[12] = uSrc1.au8[6];
+    uDstOut.au8[13] = uSrc2.au8[6];
+    uDstOut.au8[14] = uSrc1.au8[7];
+    uDstOut.au8[15] = uSrc2.au8[7];
+    /* As usual, the upper 128-bits are treated like a parallel register to the lower half. */
+    uDstOut.au8[16] = uSrc1.au8[16];
+    uDstOut.au8[17] = uSrc2.au8[16];
+    uDstOut.au8[18] = uSrc1.au8[17];
+    uDstOut.au8[19] = uSrc2.au8[17];
+    uDstOut.au8[20] = uSrc1.au8[18];
+    uDstOut.au8[21] = uSrc2.au8[18];
+    uDstOut.au8[22] = uSrc1.au8[19];
+    uDstOut.au8[23] = uSrc2.au8[19];
+    uDstOut.au8[24] = uSrc1.au8[20];
+    uDstOut.au8[25] = uSrc2.au8[20];
+    uDstOut.au8[26] = uSrc1.au8[21];
+    uDstOut.au8[27] = uSrc2.au8[21];
+    uDstOut.au8[28] = uSrc1.au8[22];
+    uDstOut.au8[29] = uSrc2.au8[22];
+    uDstOut.au8[30] = uSrc1.au8[23];
+    uDstOut.au8[31] = uSrc2.au8[23];
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PUNPCKLBW - low words -> dwords
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklwd_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc2 = { *puSrc };
+    RTUINT64U const uSrc1 = { *puDst };
+    ASMCompilerBarrier();
+    RTUINT64U uDstOut;
+    uDstOut.au16[0] = uSrc1.au16[0];
+    uDstOut.au16[1] = uSrc2.au16[0];
+    uDstOut.au16[2] = uSrc1.au16[1];
+    uDstOut.au16[3] = uSrc2.au16[1];
+    *puDst = uDstOut.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklwd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au16[0] = uSrc1.au16[0];
+    uDstOut.au16[1] = uSrc2.au16[0];
+    uDstOut.au16[2] = uSrc1.au16[1];
+    uDstOut.au16[3] = uSrc2.au16[1];
+    uDstOut.au16[4] = uSrc1.au16[2];
+    uDstOut.au16[5] = uSrc2.au16[2];
+    uDstOut.au16[6] = uSrc1.au16[3];
+    uDstOut.au16[7] = uSrc2.au16[3];
+    *puDst = uDstOut;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklwd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au16[0] = uSrc1.au16[0];
+    uDstOut.au16[1] = uSrc2.au16[0];
+    uDstOut.au16[2] = uSrc1.au16[1];
+    uDstOut.au16[3] = uSrc2.au16[1];
+    uDstOut.au16[4] = uSrc1.au16[2];
+    uDstOut.au16[5] = uSrc2.au16[2];
+    uDstOut.au16[6] = uSrc1.au16[3];
+    uDstOut.au16[7] = uSrc2.au16[3];
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklwd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au16[0]  = uSrc1.au16[0];
+    uDstOut.au16[1]  = uSrc2.au16[0];
+    uDstOut.au16[2]  = uSrc1.au16[1];
+    uDstOut.au16[3]  = uSrc2.au16[1];
+    uDstOut.au16[4]  = uSrc1.au16[2];
+    uDstOut.au16[5]  = uSrc2.au16[2];
+    uDstOut.au16[6]  = uSrc1.au16[3];
+    uDstOut.au16[7]  = uSrc2.au16[3];
+
+    uDstOut.au16[8]  = uSrc1.au16[8];
+    uDstOut.au16[9]  = uSrc2.au16[8];
+    uDstOut.au16[10] = uSrc1.au16[9];
+    uDstOut.au16[11] = uSrc2.au16[9];
+    uDstOut.au16[12] = uSrc1.au16[10];
+    uDstOut.au16[13] = uSrc2.au16[10];
+    uDstOut.au16[14] = uSrc1.au16[11];
+    uDstOut.au16[15] = uSrc2.au16[11];
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PUNPCKLBW - low dwords -> qword(s)
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpckldq_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc2 = { *puSrc };
+    RTUINT64U const uSrc1 = { *puDst };
+    ASMCompilerBarrier();
+    RTUINT64U uDstOut;
+    uDstOut.au32[0] = uSrc1.au32[0];
+    uDstOut.au32[1] = uSrc2.au32[0];
+    *puDst = uDstOut.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpckldq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au32[0] = uSrc1.au32[0];
+    uDstOut.au32[1] = uSrc2.au32[0];
+    uDstOut.au32[2] = uSrc1.au32[1];
+    uDstOut.au32[3] = uSrc2.au32[1];
+    *puDst = uDstOut;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckldq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au32[0] = uSrc1.au32[0];
+    uDstOut.au32[1] = uSrc2.au32[0];
+    uDstOut.au32[2] = uSrc1.au32[1];
+    uDstOut.au32[3] = uSrc2.au32[1];
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpckldq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au32[0] = uSrc1.au32[0];
+    uDstOut.au32[1] = uSrc2.au32[0];
+    uDstOut.au32[2] = uSrc1.au32[1];
+    uDstOut.au32[3] = uSrc2.au32[1];
+
+    uDstOut.au32[4] = uSrc1.au32[4];
+    uDstOut.au32[5] = uSrc2.au32[4];
+    uDstOut.au32[6] = uSrc1.au32[5];
+    uDstOut.au32[7] = uSrc2.au32[5];
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PUNPCKLQDQ -> Low qwords -> double qword(s).
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklqdq_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au64[0] = uSrc1.au64[0];
+    uDstOut.au64[1] = uSrc2.au64[0];
+    *puDst = uDstOut;
+}
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklqdq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au64[0] = uSrc1.au64[0];
+    uDstOut.au64[1] = uSrc2.au64[0];
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpunpcklqdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au64[0] = uSrc1.au64[0];
+    uDstOut.au64[1] = uSrc2.au64[0];
+
+    uDstOut.au64[2] = uSrc1.au64[2];
+    uDstOut.au64[3] = uSrc2.au64[2];
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PACKSSWB - signed words -> signed bytes
+ */
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_packsswb_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc2 = { *puSrc };
+    RTUINT64U const uSrc1 = { *puDst };
+    ASMCompilerBarrier();
+    RTUINT64U uDstOut;
+    uDstOut.au8[0]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[0]);
+    uDstOut.au8[1]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[1]);
+    uDstOut.au8[2]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[2]);
+    uDstOut.au8[3]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[3]);
+    uDstOut.au8[4]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[0]);
+    uDstOut.au8[5]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[1]);
+    uDstOut.au8[6]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[2]);
+    uDstOut.au8[7]  = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[3]);
+    *puDst = uDstOut.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_packsswb_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[0]);
+    uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[1]);
+    uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[2]);
+    uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[3]);
+    uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[4]);
+    uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[5]);
+    uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[6]);
+    uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[7]);
+    uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[0]);
+    uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[1]);
+    uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[2]);
+    uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[3]);
+    uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[4]);
+    uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[5]);
+    uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[6]);
+    uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[7]);
+    *puDst = uDstOut;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpacksswb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[0]);
+    uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[1]);
+    uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[2]);
+    uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[3]);
+    uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[4]);
+    uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[5]);
+    uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[6]);
+    uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[7]);
+    uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[0]);
+    uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[1]);
+    uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[2]);
+    uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[3]);
+    uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[4]);
+    uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[5]);
+    uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[6]);
+    uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[7]);
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpacksswb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[0]);
+    uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[1]);
+    uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[2]);
+    uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[3]);
+    uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[4]);
+    uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[5]);
+    uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[6]);
+    uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[7]);
+    uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[0]);
+    uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[1]);
+    uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[2]);
+    uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[3]);
+    uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[4]);
+    uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[5]);
+    uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[6]);
+    uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[7]);
+
+    uDstOut.au8[16] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[ 8]);
+    uDstOut.au8[17] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[ 9]);
+    uDstOut.au8[18] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[10]);
+    uDstOut.au8[19] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[11]);
+    uDstOut.au8[20] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[12]);
+    uDstOut.au8[21] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[13]);
+    uDstOut.au8[22] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[14]);
+    uDstOut.au8[23] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc1.au16[15]);
+    uDstOut.au8[24] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[ 8]);
+    uDstOut.au8[25] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[ 9]);
+    uDstOut.au8[26] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[10]);
+    uDstOut.au8[27] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[11]);
+    uDstOut.au8[28] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[12]);
+    uDstOut.au8[29] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[13]);
+    uDstOut.au8[30] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[14]);
+    uDstOut.au8[31] = SATURATED_SIGNED_WORD_TO_SIGNED_BYTE(uSrc2.au16[15]);
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PACKUSWB - signed words -> unsigned bytes
+ */
+#define SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(a_iWord) \
+        ( (uint16_t)(a_iWord) <= (uint16_t)0xff  \
+          ? (uint8_t)(a_iWord) \
+          : (uint8_t)0xff * (uint8_t)((((a_iWord) >> 15) & 1) ^ 1) ) /* 0xff = UINT8_MAX; 0x00 == UINT8_MIN; source bit 15 = sign */
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_packuswb_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc2 = { *puSrc };
+    RTUINT64U const uSrc1 = { *puDst };
+    ASMCompilerBarrier();
+    RTUINT64U uDstOut;
+    uDstOut.au8[0]  = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[0]);
+    uDstOut.au8[1]  = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[1]);
+    uDstOut.au8[2]  = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[2]);
+    uDstOut.au8[3]  = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[3]);
+    uDstOut.au8[4]  = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[0]);
+    uDstOut.au8[5]  = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[1]);
+    uDstOut.au8[6]  = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[2]);
+    uDstOut.au8[7]  = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[3]);
+    *puDst = uDstOut.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_packuswb_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[0]);
+    uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[1]);
+    uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[2]);
+    uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[3]);
+    uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[4]);
+    uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[5]);
+    uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[6]);
+    uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[7]);
+    uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[0]);
+    uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[1]);
+    uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[2]);
+    uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[3]);
+    uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[4]);
+    uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[5]);
+    uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[6]);
+    uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[7]);
+    *puDst = uDstOut;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpackuswb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[0]);
+    uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[1]);
+    uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[2]);
+    uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[3]);
+    uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[4]);
+    uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[5]);
+    uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[6]);
+    uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[7]);
+    uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[0]);
+    uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[1]);
+    uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[2]);
+    uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[3]);
+    uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[4]);
+    uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[5]);
+    uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[6]);
+    uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[7]);
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpackuswb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au8[ 0] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[0]);
+    uDstOut.au8[ 1] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[1]);
+    uDstOut.au8[ 2] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[2]);
+    uDstOut.au8[ 3] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[3]);
+    uDstOut.au8[ 4] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[4]);
+    uDstOut.au8[ 5] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[5]);
+    uDstOut.au8[ 6] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[6]);
+    uDstOut.au8[ 7] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[7]);
+    uDstOut.au8[ 8] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[0]);
+    uDstOut.au8[ 9] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[1]);
+    uDstOut.au8[10] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[2]);
+    uDstOut.au8[11] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[3]);
+    uDstOut.au8[12] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[4]);
+    uDstOut.au8[13] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[5]);
+    uDstOut.au8[14] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[6]);
+    uDstOut.au8[15] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[7]);
+
+    uDstOut.au8[16] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[ 8]);
+    uDstOut.au8[17] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[ 9]);
+    uDstOut.au8[18] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[10]);
+    uDstOut.au8[19] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[11]);
+    uDstOut.au8[20] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[12]);
+    uDstOut.au8[21] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[13]);
+    uDstOut.au8[22] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[14]);
+    uDstOut.au8[23] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc1.au16[15]);
+    uDstOut.au8[24] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[ 8]);
+    uDstOut.au8[25] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[ 9]);
+    uDstOut.au8[26] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[10]);
+    uDstOut.au8[27] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[11]);
+    uDstOut.au8[28] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[12]);
+    uDstOut.au8[29] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[13]);
+    uDstOut.au8[30] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[14]);
+    uDstOut.au8[31] = SATURATED_SIGNED_WORD_TO_UNSIGNED_BYTE(uSrc2.au16[15]);
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PACKSSDW - signed dwords -> signed words
+ */
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_packssdw_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc2 = { *puSrc };
+    RTUINT64U const uSrc1 = { *puDst };
+    ASMCompilerBarrier();
+    RTUINT64U uDstOut;
+    uDstOut.au16[0]  = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[0]);
+    uDstOut.au16[1]  = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[1]);
+    uDstOut.au16[2]  = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[0]);
+    uDstOut.au16[3]  = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[1]);
+    *puDst = uDstOut.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_packssdw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[0]);
+    uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[1]);
+    uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[2]);
+    uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[3]);
+    uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[0]);
+    uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[1]);
+    uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[2]);
+    uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[3]);
+    *puDst = uDstOut;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpackssdw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[0]);
+    uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[1]);
+    uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[2]);
+    uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[3]);
+    uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[0]);
+    uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[1]);
+    uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[2]);
+    uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[3]);
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpackssdw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[0]);
+    uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[1]);
+    uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[2]);
+    uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[3]);
+    uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[0]);
+    uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[1]);
+    uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[2]);
+    uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[3]);
+
+    uDstOut.au16[ 8] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[4]);
+    uDstOut.au16[ 9] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[5]);
+    uDstOut.au16[10] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[6]);
+    uDstOut.au16[11] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.au32[7]);
+    uDstOut.au16[12] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[4]);
+    uDstOut.au16[13] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[5]);
+    uDstOut.au16[14] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[6]);
+    uDstOut.au16[15] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.au32[7]);
+    *puDst = uDstOut;
+}
+
+
+/*
+ * PACKUSDW - signed dwords -> unsigned words
+ */
+#define SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(a_iDword) \
+        ( (uint32_t)(a_iDword) <= (uint16_t)0xffff  \
+          ? (uint16_t)(a_iDword) \
+          : (uint16_t)0xffff * (uint16_t)((((a_iDword) >> 31) & 1) ^ 1) ) /* 0xffff = UINT16_MAX; source bit 31 = sign */
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_packusdw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U const uSrc2 = *puSrc;
+    RTUINT128U const uSrc1 = *puDst;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[0]);
+    uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[1]);
+    uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[2]);
+    uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[3]);
+    uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[0]);
+    uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[1]);
+    uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[2]);
+    uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[3]);
+    *puDst = uDstOut;
+}
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpackusdw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U const uSrc2 = *puSrc2;
+    RTUINT128U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT128U uDstOut;
+    uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[0]);
+    uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[1]);
+    uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[2]);
+    uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[3]);
+    uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[0]);
+    uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[1]);
+    uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[2]);
+    uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[3]);
+    *puDst = uDstOut;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpackusdw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U const uSrc2 = *puSrc2;
+    RTUINT256U const uSrc1 = *puSrc1;
+    ASMCompilerBarrier();
+    RTUINT256U uDstOut;
+    uDstOut.au16[ 0] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[0]);
+    uDstOut.au16[ 1] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[1]);
+    uDstOut.au16[ 2] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[2]);
+    uDstOut.au16[ 3] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[3]);
+    uDstOut.au16[ 4] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[0]);
+    uDstOut.au16[ 5] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[1]);
+    uDstOut.au16[ 6] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[2]);
+    uDstOut.au16[ 7] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[3]);
+
+    uDstOut.au16[ 8] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[4]);
+    uDstOut.au16[ 9] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[5]);
+    uDstOut.au16[10] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[6]);
+    uDstOut.au16[11] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc1.au32[7]);
+    uDstOut.au16[12] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[4]);
+    uDstOut.au16[13] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[5]);
+    uDstOut.au16[14] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[6]);
+    uDstOut.au16[15] = SATURATED_SIGNED_DWORD_TO_UNSIGNED_WORD(uSrc2.au32[7]);
+    *puDst = uDstOut;
+}
+
+
+/*
+ * [V]PABSB / [V]PABSW / [V]PABSD
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pabsb_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc    = { *puSrc };
+    RTUINT64U       uDstOut = { 0 };
+
+    uDstOut.au8[0] = RT_ABS(uSrc.ai8[0]);
+    uDstOut.au8[1] = RT_ABS(uSrc.ai8[1]);
+    uDstOut.au8[2] = RT_ABS(uSrc.ai8[2]);
+    uDstOut.au8[3] = RT_ABS(uSrc.ai8[3]);
+    uDstOut.au8[4] = RT_ABS(uSrc.ai8[4]);
+    uDstOut.au8[5] = RT_ABS(uSrc.ai8[5]);
+    uDstOut.au8[6] = RT_ABS(uSrc.ai8[6]);
+    uDstOut.au8[7] = RT_ABS(uSrc.ai8[7]);
+    *puDst = uDstOut.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pabsb_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    puDst->au8[ 0] = RT_ABS(puSrc->ai8[ 0]);
+    puDst->au8[ 1] = RT_ABS(puSrc->ai8[ 1]);
+    puDst->au8[ 2] = RT_ABS(puSrc->ai8[ 2]);
+    puDst->au8[ 3] = RT_ABS(puSrc->ai8[ 3]);
+    puDst->au8[ 4] = RT_ABS(puSrc->ai8[ 4]);
+    puDst->au8[ 5] = RT_ABS(puSrc->ai8[ 5]);
+    puDst->au8[ 6] = RT_ABS(puSrc->ai8[ 6]);
+    puDst->au8[ 7] = RT_ABS(puSrc->ai8[ 7]);
+    puDst->au8[ 8] = RT_ABS(puSrc->ai8[ 8]);
+    puDst->au8[ 9] = RT_ABS(puSrc->ai8[ 9]);
+    puDst->au8[10] = RT_ABS(puSrc->ai8[10]);
+    puDst->au8[11] = RT_ABS(puSrc->ai8[11]);
+    puDst->au8[12] = RT_ABS(puSrc->ai8[12]);
+    puDst->au8[13] = RT_ABS(puSrc->ai8[13]);
+    puDst->au8[14] = RT_ABS(puSrc->ai8[14]);
+    puDst->au8[15] = RT_ABS(puSrc->ai8[15]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pabsw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc    = { *puSrc };
+    RTUINT64U       uDstOut = { 0 };
+
+    uDstOut.au16[0] = RT_ABS(uSrc.ai16[0]);
+    uDstOut.au16[1] = RT_ABS(uSrc.ai16[1]);
+    uDstOut.au16[2] = RT_ABS(uSrc.ai16[2]);
+    uDstOut.au16[3] = RT_ABS(uSrc.ai16[3]);
+    *puDst = uDstOut.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pabsw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    puDst->au16[ 0] = RT_ABS(puSrc->ai16[ 0]);
+    puDst->au16[ 1] = RT_ABS(puSrc->ai16[ 1]);
+    puDst->au16[ 2] = RT_ABS(puSrc->ai16[ 2]);
+    puDst->au16[ 3] = RT_ABS(puSrc->ai16[ 3]);
+    puDst->au16[ 4] = RT_ABS(puSrc->ai16[ 4]);
+    puDst->au16[ 5] = RT_ABS(puSrc->ai16[ 5]);
+    puDst->au16[ 6] = RT_ABS(puSrc->ai16[ 6]);
+    puDst->au16[ 7] = RT_ABS(puSrc->ai16[ 7]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pabsd_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U const uSrc    = { *puSrc };
+    RTUINT64U       uDstOut = { 0 };
+
+    uDstOut.au32[0] = RT_ABS(uSrc.ai32[0]);
+    uDstOut.au32[1] = RT_ABS(uSrc.ai32[1]);
+    *puDst = uDstOut.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pabsd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    puDst->au32[ 0] = RT_ABS(puSrc->ai32[ 0]);
+    puDst->au32[ 1] = RT_ABS(puSrc->ai32[ 1]);
+    puDst->au32[ 2] = RT_ABS(puSrc->ai32[ 2]);
+    puDst->au32[ 3] = RT_ABS(puSrc->ai32[ 3]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    puDst->au8[ 0] = RT_ABS(puSrc->ai8[ 0]);
+    puDst->au8[ 1] = RT_ABS(puSrc->ai8[ 1]);
+    puDst->au8[ 2] = RT_ABS(puSrc->ai8[ 2]);
+    puDst->au8[ 3] = RT_ABS(puSrc->ai8[ 3]);
+    puDst->au8[ 4] = RT_ABS(puSrc->ai8[ 4]);
+    puDst->au8[ 5] = RT_ABS(puSrc->ai8[ 5]);
+    puDst->au8[ 6] = RT_ABS(puSrc->ai8[ 6]);
+    puDst->au8[ 7] = RT_ABS(puSrc->ai8[ 7]);
+    puDst->au8[ 8] = RT_ABS(puSrc->ai8[ 8]);
+    puDst->au8[ 9] = RT_ABS(puSrc->ai8[ 9]);
+    puDst->au8[10] = RT_ABS(puSrc->ai8[10]);
+    puDst->au8[11] = RT_ABS(puSrc->ai8[11]);
+    puDst->au8[12] = RT_ABS(puSrc->ai8[12]);
+    puDst->au8[13] = RT_ABS(puSrc->ai8[13]);
+    puDst->au8[14] = RT_ABS(puSrc->ai8[14]);
+    puDst->au8[15] = RT_ABS(puSrc->ai8[15]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc))
+{
+    puDst->au8[ 0] = RT_ABS(puSrc->ai8[ 0]);
+    puDst->au8[ 1] = RT_ABS(puSrc->ai8[ 1]);
+    puDst->au8[ 2] = RT_ABS(puSrc->ai8[ 2]);
+    puDst->au8[ 3] = RT_ABS(puSrc->ai8[ 3]);
+    puDst->au8[ 4] = RT_ABS(puSrc->ai8[ 4]);
+    puDst->au8[ 5] = RT_ABS(puSrc->ai8[ 5]);
+    puDst->au8[ 6] = RT_ABS(puSrc->ai8[ 6]);
+    puDst->au8[ 7] = RT_ABS(puSrc->ai8[ 7]);
+    puDst->au8[ 8] = RT_ABS(puSrc->ai8[ 8]);
+    puDst->au8[ 9] = RT_ABS(puSrc->ai8[ 9]);
+    puDst->au8[10] = RT_ABS(puSrc->ai8[10]);
+    puDst->au8[11] = RT_ABS(puSrc->ai8[11]);
+    puDst->au8[12] = RT_ABS(puSrc->ai8[12]);
+    puDst->au8[13] = RT_ABS(puSrc->ai8[13]);
+    puDst->au8[14] = RT_ABS(puSrc->ai8[14]);
+    puDst->au8[15] = RT_ABS(puSrc->ai8[15]);
+    puDst->au8[16] = RT_ABS(puSrc->ai8[16]);
+    puDst->au8[17] = RT_ABS(puSrc->ai8[17]);
+    puDst->au8[18] = RT_ABS(puSrc->ai8[18]);
+    puDst->au8[19] = RT_ABS(puSrc->ai8[19]);
+    puDst->au8[20] = RT_ABS(puSrc->ai8[20]);
+    puDst->au8[21] = RT_ABS(puSrc->ai8[21]);
+    puDst->au8[22] = RT_ABS(puSrc->ai8[22]);
+    puDst->au8[23] = RT_ABS(puSrc->ai8[23]);
+    puDst->au8[24] = RT_ABS(puSrc->ai8[24]);
+    puDst->au8[25] = RT_ABS(puSrc->ai8[25]);
+    puDst->au8[26] = RT_ABS(puSrc->ai8[26]);
+    puDst->au8[27] = RT_ABS(puSrc->ai8[27]);
+    puDst->au8[28] = RT_ABS(puSrc->ai8[28]);
+    puDst->au8[29] = RT_ABS(puSrc->ai8[29]);
+    puDst->au8[30] = RT_ABS(puSrc->ai8[30]);
+    puDst->au8[31] = RT_ABS(puSrc->ai8[31]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    puDst->au16[ 0] = RT_ABS(puSrc->ai16[ 0]);
+    puDst->au16[ 1] = RT_ABS(puSrc->ai16[ 1]);
+    puDst->au16[ 2] = RT_ABS(puSrc->ai16[ 2]);
+    puDst->au16[ 3] = RT_ABS(puSrc->ai16[ 3]);
+    puDst->au16[ 4] = RT_ABS(puSrc->ai16[ 4]);
+    puDst->au16[ 5] = RT_ABS(puSrc->ai16[ 5]);
+    puDst->au16[ 6] = RT_ABS(puSrc->ai16[ 6]);
+    puDst->au16[ 7] = RT_ABS(puSrc->ai16[ 7]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc))
+{
+    puDst->au16[ 0] = RT_ABS(puSrc->ai16[ 0]);
+    puDst->au16[ 1] = RT_ABS(puSrc->ai16[ 1]);
+    puDst->au16[ 2] = RT_ABS(puSrc->ai16[ 2]);
+    puDst->au16[ 3] = RT_ABS(puSrc->ai16[ 3]);
+    puDst->au16[ 4] = RT_ABS(puSrc->ai16[ 4]);
+    puDst->au16[ 5] = RT_ABS(puSrc->ai16[ 5]);
+    puDst->au16[ 6] = RT_ABS(puSrc->ai16[ 6]);
+    puDst->au16[ 7] = RT_ABS(puSrc->ai16[ 7]);
+    puDst->au16[ 8] = RT_ABS(puSrc->ai16[ 8]);
+    puDst->au16[ 9] = RT_ABS(puSrc->ai16[ 9]);
+    puDst->au16[10] = RT_ABS(puSrc->ai16[10]);
+    puDst->au16[11] = RT_ABS(puSrc->ai16[11]);
+    puDst->au16[12] = RT_ABS(puSrc->ai16[12]);
+    puDst->au16[13] = RT_ABS(puSrc->ai16[13]);
+    puDst->au16[14] = RT_ABS(puSrc->ai16[14]);
+    puDst->au16[15] = RT_ABS(puSrc->ai16[15]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    puDst->au32[ 0] = RT_ABS(puSrc->ai32[ 0]);
+    puDst->au32[ 1] = RT_ABS(puSrc->ai32[ 1]);
+    puDst->au32[ 2] = RT_ABS(puSrc->ai32[ 2]);
+    puDst->au32[ 3] = RT_ABS(puSrc->ai32[ 3]);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpabsd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc))
+{
+    puDst->au32[ 0] = RT_ABS(puSrc->ai32[ 0]);
+    puDst->au32[ 1] = RT_ABS(puSrc->ai32[ 1]);
+    puDst->au32[ 2] = RT_ABS(puSrc->ai32[ 2]);
+    puDst->au32[ 3] = RT_ABS(puSrc->ai32[ 3]);
+    puDst->au32[ 4] = RT_ABS(puSrc->ai32[ 4]);
+    puDst->au32[ 5] = RT_ABS(puSrc->ai32[ 5]);
+    puDst->au32[ 6] = RT_ABS(puSrc->ai32[ 6]);
+    puDst->au32[ 7] = RT_ABS(puSrc->ai32[ 7]);
+}
+
+
+/*
+ * PSIGNB / VPSIGNB / PSIGNW / VPSIGNW / PSIGND / VPSIGND
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_psignb_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst = { 0 }; /* Shut up MSVC. */
+
+    for (uint32_t i = 0; i < RT_ELEMENTS(uDst.ai8); i++)
+    {
+        if (uSrc2.ai8[i] < 0)
+            uDst.ai8[i] = -uSrc1.ai8[i];
+        else if (uSrc2.ai8[i] == 0)
+            uDst.ai8[i] = 0;
+        else /* uSrc2.ai8[i] > 0 */
+            uDst.ai8[i] = uSrc1.ai8[i];
+    }
+
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psignb_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai8); i++)
+    {
+        if (puSrc->ai8[i] < 0)
+            puDst->ai8[i] = -uSrc1.ai8[i];
+        else if (puSrc->ai8[i] == 0)
+            puDst->ai8[i] = 0;
+        else /* puSrc->ai8[i] > 0 */
+            puDst->ai8[i] = uSrc1.ai8[i];
+    }
+
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psignw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst = { 0 }; /* Shut up MSVC. */
+
+    for (uint32_t i = 0; i < RT_ELEMENTS(uDst.ai16); i++)
+    {
+        if (uSrc2.ai16[i] < 0)
+            uDst.ai16[i] = -uSrc1.ai16[i];
+        else if (uSrc2.ai16[i] == 0)
+            uDst.ai16[i] = 0;
+        else /* uSrc2.ai16[i] > 0 */
+            uDst.ai16[i] = uSrc1.ai16[i];
+    }
+
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psignw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai16); i++)
+    {
+        if (puSrc->ai16[i] < 0)
+            puDst->ai16[i] = -uSrc1.ai16[i];
+        else if (puSrc->ai16[i] == 0)
+            puDst->ai16[i] = 0;
+        else /* puSrc->ai16[i] > 0 */
+            puDst->ai16[i] = uSrc1.ai16[i];
+    }
+
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psignd_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst = { 0 }; /* Shut up MSVC. */
+
+    for (uint32_t i = 0; i < RT_ELEMENTS(uDst.ai32); i++)
+    {
+        if (uSrc2.ai32[i] < 0)
+            uDst.ai32[i] = -uSrc1.ai32[i];
+        else if (uSrc2.ai32[i] == 0)
+            uDst.ai32[i] = 0;
+        else /* uSrc2.ai32[i] > 0 */
+            uDst.ai32[i] = uSrc1.ai32[i];
+    }
+
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psignd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai32); i++)
+    {
+        if (puSrc->ai32[i] < 0)
+            puDst->ai32[i] = -uSrc1.ai32[i];
+        else if (puSrc->ai32[i] == 0)
+            puDst->ai32[i] = 0;
+        else /* puSrc->ai32[i] > 0 */
+            puDst->ai32[i] = uSrc1.ai32[i];
+    }
+
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai8); i++)
+    {
+        if (puSrc2->ai8[i] < 0)
+            puDst->ai8[i] = -puSrc1->ai8[i];
+        else if (puSrc2->ai8[i] == 0)
+            puDst->ai8[i] = 0;
+        else /* puSrc2->ai8[i] > 0 */
+            puDst->ai8[i] = puSrc1->ai8[i];
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai8); i++)
+    {
+        if (puSrc2->ai8[i] < 0)
+            puDst->ai8[i] = -puSrc1->ai8[i];
+        else if (puSrc2->ai8[i] == 0)
+            puDst->ai8[i] = 0;
+        else /* puSrc2->ai8[i] > 0 */
+            puDst->ai8[i] = puSrc1->ai8[i];
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai16); i++)
+    {
+        if (puSrc2->ai16[i] < 0)
+            puDst->ai16[i] = -puSrc1->ai16[i];
+        else if (puSrc2->ai16[i] == 0)
+            puDst->ai16[i] = 0;
+        else /* puSrc2->ai16[i] > 0 */
+            puDst->ai16[i] = puSrc1->ai16[i];
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai16); i++)
+    {
+        if (puSrc2->ai16[i] < 0)
+            puDst->ai16[i] = -puSrc1->ai16[i];
+        else if (puSrc2->ai16[i] == 0)
+            puDst->ai16[i] = 0;
+        else /* puSrc2->ai16[i] > 0 */
+            puDst->ai16[i] = puSrc1->ai16[i];
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai32); i++)
+    {
+        if (puSrc2->ai32[i] < 0)
+            puDst->ai32[i] = -puSrc1->ai32[i];
+        else if (puSrc2->ai32[i] == 0)
+            puDst->ai32[i] = 0;
+        else /* puSrc2->ai32[i] > 0 */
+            puDst->ai32[i] = puSrc1->ai32[i];
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsignd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    for (uint32_t i = 0; i < RT_ELEMENTS(puDst->ai32); i++)
+    {
+        if (puSrc2->ai32[i] < 0)
+            puDst->ai32[i] = -puSrc1->ai32[i];
+        else if (puSrc2->ai32[i] == 0)
+            puDst->ai32[i] = 0;
+        else /* puSrc2->ai32[i] > 0 */
+            puDst->ai32[i] = puSrc1->ai32[i];
+    }
+}
+
+
+/*
+ * PHADDW / VPHADDW / PHADDD / VPHADDD
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_phaddw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst = { 0 }; /* Shut up MSVC. */
+
+    uDst.ai16[0] = uSrc1.ai16[0] + uSrc1.ai16[1];
+    uDst.ai16[1] = uSrc1.ai16[2] + uSrc1.ai16[3];
+    uDst.ai16[2] = uSrc2.ai16[0] + uSrc2.ai16[1];
+    uDst.ai16[3] = uSrc2.ai16[2] + uSrc2.ai16[3];
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_phaddw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai16[0] = uSrc1.ai16[0] + uSrc1.ai16[1];
+    puDst->ai16[1] = uSrc1.ai16[2] + uSrc1.ai16[3];
+    puDst->ai16[2] = uSrc1.ai16[4] + uSrc1.ai16[5];
+    puDst->ai16[3] = uSrc1.ai16[6] + uSrc1.ai16[7];
+
+    puDst->ai16[4] = puSrc->ai16[0] + puSrc->ai16[1];
+    puDst->ai16[5] = puSrc->ai16[2] + puSrc->ai16[3];
+    puDst->ai16[6] = puSrc->ai16[4] + puSrc->ai16[5];
+    puDst->ai16[7] = puSrc->ai16[6] + puSrc->ai16[7];
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_phaddd_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst = { 0 }; /* Shut up MSVC. */
+
+    uDst.ai32[0] = uSrc1.ai32[0] + uSrc1.ai32[1];
+    uDst.ai32[1] = uSrc2.ai32[0] + uSrc2.ai32[1];
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_phaddd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai32[0] = uSrc1.ai32[0] + uSrc1.ai32[1];
+    puDst->ai32[1] = uSrc1.ai32[2] + uSrc1.ai32[3];
+
+    puDst->ai32[2] = puSrc->ai32[0] + puSrc->ai32[1];
+    puDst->ai32[3] = puSrc->ai32[2] + puSrc->ai32[3];
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[0] = puSrc1->ai16[0] + puSrc1->ai16[1];
+    uDst.ai16[1] = puSrc1->ai16[2] + puSrc1->ai16[3];
+    uDst.ai16[2] = puSrc1->ai16[4] + puSrc1->ai16[5];
+    uDst.ai16[3] = puSrc1->ai16[6] + puSrc1->ai16[7];
+
+    uDst.ai16[4] = puSrc2->ai16[0] + puSrc2->ai16[1];
+    uDst.ai16[5] = puSrc2->ai16[2] + puSrc2->ai16[3];
+    uDst.ai16[6] = puSrc2->ai16[4] + puSrc2->ai16[5];
+    uDst.ai16[7] = puSrc2->ai16[6] + puSrc2->ai16[7];
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[ 0] = puSrc1->ai16[ 0] + puSrc1->ai16[ 1];
+    uDst.ai16[ 1] = puSrc1->ai16[ 2] + puSrc1->ai16[ 3];
+    uDst.ai16[ 2] = puSrc1->ai16[ 4] + puSrc1->ai16[ 5];
+    uDst.ai16[ 3] = puSrc1->ai16[ 6] + puSrc1->ai16[ 7];
+    uDst.ai16[ 4] = puSrc2->ai16[ 0] + puSrc2->ai16[ 1];
+    uDst.ai16[ 5] = puSrc2->ai16[ 2] + puSrc2->ai16[ 3];
+    uDst.ai16[ 6] = puSrc2->ai16[ 4] + puSrc2->ai16[ 5];
+    uDst.ai16[ 7] = puSrc2->ai16[ 6] + puSrc2->ai16[ 7];
+
+    uDst.ai16[ 8] = puSrc1->ai16[ 8] + puSrc1->ai16[ 9];
+    uDst.ai16[ 9] = puSrc1->ai16[10] + puSrc1->ai16[11];
+    uDst.ai16[10] = puSrc1->ai16[12] + puSrc1->ai16[13];
+    uDst.ai16[11] = puSrc1->ai16[14] + puSrc1->ai16[15];
+    uDst.ai16[12] = puSrc2->ai16[ 8] + puSrc2->ai16[ 9];
+    uDst.ai16[13] = puSrc2->ai16[10] + puSrc2->ai16[11];
+    uDst.ai16[14] = puSrc2->ai16[12] + puSrc2->ai16[13];
+    uDst.ai16[15] = puSrc2->ai16[14] + puSrc2->ai16[15];
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+    puDst->au64[2] = uDst.au64[2];
+    puDst->au64[3] = uDst.au64[3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai32[0] = puSrc1->ai32[0] + puSrc1->ai32[1];
+    uDst.ai32[1] = puSrc1->ai32[2] + puSrc1->ai32[3];
+
+    uDst.ai32[2] = puSrc2->ai32[0] + puSrc2->ai32[1];
+    uDst.ai32[3] = puSrc2->ai32[2] + puSrc2->ai32[3];
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai32[0] = puSrc1->ai32[ 0] + puSrc1->ai32[ 1];
+    uDst.ai32[1] = puSrc1->ai32[ 2] + puSrc1->ai32[ 3];
+    uDst.ai32[2] = puSrc2->ai32[ 0] + puSrc2->ai32[ 1];
+    uDst.ai32[3] = puSrc2->ai32[ 2] + puSrc2->ai32[ 3];
+
+    uDst.ai32[4] = puSrc1->ai32[ 4] + puSrc1->ai32[ 5];
+    uDst.ai32[5] = puSrc1->ai32[ 6] + puSrc1->ai32[ 7];
+    uDst.ai32[6] = puSrc2->ai32[ 4] + puSrc2->ai32[ 5];
+    uDst.ai32[7] = puSrc2->ai32[ 6] + puSrc2->ai32[ 7];
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+    puDst->au64[2] = uDst.au64[2];
+    puDst->au64[3] = uDst.au64[3];
+}
+
+
+/*
+ * PHSUBW / VPHSUBW / PHSUBD / VPHSUBD
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_phsubw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst = { 0 }; /* Shut up MSVC. */
+
+    uDst.ai16[0] = uSrc1.ai16[0] - uSrc1.ai16[1];
+    uDst.ai16[1] = uSrc1.ai16[2] - uSrc1.ai16[3];
+    uDst.ai16[2] = uSrc2.ai16[0] - uSrc2.ai16[1];
+    uDst.ai16[3] = uSrc2.ai16[2] - uSrc2.ai16[3];
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_phsubw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai16[0] = uSrc1.ai16[0] - uSrc1.ai16[1];
+    puDst->ai16[1] = uSrc1.ai16[2] - uSrc1.ai16[3];
+    puDst->ai16[2] = uSrc1.ai16[4] - uSrc1.ai16[5];
+    puDst->ai16[3] = uSrc1.ai16[6] - uSrc1.ai16[7];
+
+    puDst->ai16[4] = puSrc->ai16[0] - puSrc->ai16[1];
+    puDst->ai16[5] = puSrc->ai16[2] - puSrc->ai16[3];
+    puDst->ai16[6] = puSrc->ai16[4] - puSrc->ai16[5];
+    puDst->ai16[7] = puSrc->ai16[6] - puSrc->ai16[7];
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_phsubd_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst = { 0 }; /* Shut up MSVC. */
+
+    uDst.ai32[0] = uSrc1.ai32[0] - uSrc1.ai32[1];
+    uDst.ai32[1] = uSrc2.ai32[0] - uSrc2.ai32[1];
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_phsubd_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai32[0] = uSrc1.ai32[0] - uSrc1.ai32[1];
+    puDst->ai32[1] = uSrc1.ai32[2] - uSrc1.ai32[3];
+
+    puDst->ai32[2] = puSrc->ai32[0] - puSrc->ai32[1];
+    puDst->ai32[3] = puSrc->ai32[2] - puSrc->ai32[3];
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[0] = puSrc1->ai16[0] - puSrc1->ai16[1];
+    uDst.ai16[1] = puSrc1->ai16[2] - puSrc1->ai16[3];
+    uDst.ai16[2] = puSrc1->ai16[4] - puSrc1->ai16[5];
+    uDst.ai16[3] = puSrc1->ai16[6] - puSrc1->ai16[7];
+
+    uDst.ai16[4] = puSrc2->ai16[0] - puSrc2->ai16[1];
+    uDst.ai16[5] = puSrc2->ai16[2] - puSrc2->ai16[3];
+    uDst.ai16[6] = puSrc2->ai16[4] - puSrc2->ai16[5];
+    uDst.ai16[7] = puSrc2->ai16[6] - puSrc2->ai16[7];
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[ 0] = puSrc1->ai16[ 0] - puSrc1->ai16[ 1];
+    uDst.ai16[ 1] = puSrc1->ai16[ 2] - puSrc1->ai16[ 3];
+    uDst.ai16[ 2] = puSrc1->ai16[ 4] - puSrc1->ai16[ 5];
+    uDst.ai16[ 3] = puSrc1->ai16[ 6] - puSrc1->ai16[ 7];
+    uDst.ai16[ 4] = puSrc2->ai16[ 0] - puSrc2->ai16[ 1];
+    uDst.ai16[ 5] = puSrc2->ai16[ 2] - puSrc2->ai16[ 3];
+    uDst.ai16[ 6] = puSrc2->ai16[ 4] - puSrc2->ai16[ 5];
+    uDst.ai16[ 7] = puSrc2->ai16[ 6] - puSrc2->ai16[ 7];
+
+    uDst.ai16[ 8] = puSrc1->ai16[ 8] - puSrc1->ai16[ 9];
+    uDst.ai16[ 9] = puSrc1->ai16[10] - puSrc1->ai16[11];
+    uDst.ai16[10] = puSrc1->ai16[12] - puSrc1->ai16[13];
+    uDst.ai16[11] = puSrc1->ai16[14] - puSrc1->ai16[15];
+    uDst.ai16[12] = puSrc2->ai16[ 8] - puSrc2->ai16[ 9];
+    uDst.ai16[13] = puSrc2->ai16[10] - puSrc2->ai16[11];
+    uDst.ai16[14] = puSrc2->ai16[12] - puSrc2->ai16[13];
+    uDst.ai16[15] = puSrc2->ai16[14] - puSrc2->ai16[15];
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+    puDst->au64[2] = uDst.au64[2];
+    puDst->au64[3] = uDst.au64[3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai32[0] = puSrc1->ai32[0] - puSrc1->ai32[1];
+    uDst.ai32[1] = puSrc1->ai32[2] - puSrc1->ai32[3];
+
+    uDst.ai32[2] = puSrc2->ai32[0] - puSrc2->ai32[1];
+    uDst.ai32[3] = puSrc2->ai32[2] - puSrc2->ai32[3];
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai32[0] = puSrc1->ai32[ 0] - puSrc1->ai32[ 1];
+    uDst.ai32[1] = puSrc1->ai32[ 2] - puSrc1->ai32[ 3];
+    uDst.ai32[2] = puSrc2->ai32[ 0] - puSrc2->ai32[ 1];
+    uDst.ai32[3] = puSrc2->ai32[ 2] - puSrc2->ai32[ 3];
+
+    uDst.ai32[4] = puSrc1->ai32[ 4] - puSrc1->ai32[ 5];
+    uDst.ai32[5] = puSrc1->ai32[ 6] - puSrc1->ai32[ 7];
+    uDst.ai32[6] = puSrc2->ai32[ 4] - puSrc2->ai32[ 5];
+    uDst.ai32[7] = puSrc2->ai32[ 6] - puSrc2->ai32[ 7];
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+    puDst->au64[2] = uDst.au64[2];
+    puDst->au64[3] = uDst.au64[3];
+}
+
+
+/*
+ * PHADDSW / VPHADDSW
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_phaddsw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst = { 0 }; /* Shut up MSVC. */
+
+    uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] + uSrc1.ai16[1]);
+    uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] + uSrc1.ai16[3]);
+    uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.ai16[0] + uSrc2.ai16[1]);
+    uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.ai16[2] + uSrc2.ai16[3]);
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_phaddsw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] + uSrc1.ai16[1]);
+    puDst->ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] + uSrc1.ai16[3]);
+    puDst->ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[4] + uSrc1.ai16[5]);
+    puDst->ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[6] + uSrc1.ai16[7]);
+
+    puDst->ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[0] + puSrc->ai16[1]);
+    puDst->ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[2] + puSrc->ai16[3]);
+    puDst->ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[4] + puSrc->ai16[5]);
+    puDst->ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[6] + puSrc->ai16[7]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddsw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[0] + puSrc1->ai16[1]);
+    uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[2] + puSrc1->ai16[3]);
+    uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[4] + puSrc1->ai16[5]);
+    uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[6] + puSrc1->ai16[7]);
+
+    uDst.ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[0] + puSrc2->ai16[1]);
+    uDst.ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[2] + puSrc2->ai16[3]);
+    uDst.ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[4] + puSrc2->ai16[5]);
+    uDst.ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[6] + puSrc2->ai16[7]);
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphaddsw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 0] + puSrc1->ai16[ 1]);
+    uDst.ai16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 2] + puSrc1->ai16[ 3]);
+    uDst.ai16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 4] + puSrc1->ai16[ 5]);
+    uDst.ai16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 6] + puSrc1->ai16[ 7]);
+    uDst.ai16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 0] + puSrc2->ai16[ 1]);
+    uDst.ai16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 2] + puSrc2->ai16[ 3]);
+    uDst.ai16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 4] + puSrc2->ai16[ 5]);
+    uDst.ai16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 6] + puSrc2->ai16[ 7]);
+
+    uDst.ai16[ 8] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 8] + puSrc1->ai16[ 9]);
+    uDst.ai16[ 9] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[10] + puSrc1->ai16[11]);
+    uDst.ai16[10] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[12] + puSrc1->ai16[13]);
+    uDst.ai16[11] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[14] + puSrc1->ai16[15]);
+    uDst.ai16[12] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 8] + puSrc2->ai16[ 9]);
+    uDst.ai16[13] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[10] + puSrc2->ai16[11]);
+    uDst.ai16[14] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[12] + puSrc2->ai16[13]);
+    uDst.ai16[15] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[14] + puSrc2->ai16[15]);
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+    puDst->au64[2] = uDst.au64[2];
+    puDst->au64[3] = uDst.au64[3];
+}
+
+
+/*
+ * PHSUBSW / VPHSUBSW
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_phsubsw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst = { 0 }; /* Shut up MSVC. */
+
+    uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] - uSrc1.ai16[1]);
+    uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] - uSrc1.ai16[3]);
+    uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.ai16[0] - uSrc2.ai16[1]);
+    uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc2.ai16[2] - uSrc2.ai16[3]);
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_phsubsw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[0] - uSrc1.ai16[1]);
+    puDst->ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[2] - uSrc1.ai16[3]);
+    puDst->ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[4] - uSrc1.ai16[5]);
+    puDst->ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(uSrc1.ai16[6] - uSrc1.ai16[7]);
+
+    puDst->ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[0] - puSrc->ai16[1]);
+    puDst->ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[2] - puSrc->ai16[3]);
+    puDst->ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[4] - puSrc->ai16[5]);
+    puDst->ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc->ai16[6] - puSrc->ai16[7]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubsw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[0] - puSrc1->ai16[1]);
+    uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[2] - puSrc1->ai16[3]);
+    uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[4] - puSrc1->ai16[5]);
+    uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[6] - puSrc1->ai16[7]);
+
+    uDst.ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[0] - puSrc2->ai16[1]);
+    uDst.ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[2] - puSrc2->ai16[3]);
+    uDst.ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[4] - puSrc2->ai16[5]);
+    uDst.ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[6] - puSrc2->ai16[7]);
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphsubsw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 0] - puSrc1->ai16[ 1]);
+    uDst.ai16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 2] - puSrc1->ai16[ 3]);
+    uDst.ai16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 4] - puSrc1->ai16[ 5]);
+    uDst.ai16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 6] - puSrc1->ai16[ 7]);
+    uDst.ai16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 0] - puSrc2->ai16[ 1]);
+    uDst.ai16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 2] - puSrc2->ai16[ 3]);
+    uDst.ai16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 4] - puSrc2->ai16[ 5]);
+    uDst.ai16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 6] - puSrc2->ai16[ 7]);
+
+    uDst.ai16[ 8] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[ 8] - puSrc1->ai16[ 9]);
+    uDst.ai16[ 9] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[10] - puSrc1->ai16[11]);
+    uDst.ai16[10] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[12] - puSrc1->ai16[13]);
+    uDst.ai16[11] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc1->ai16[14] - puSrc1->ai16[15]);
+    uDst.ai16[12] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[ 8] - puSrc2->ai16[ 9]);
+    uDst.ai16[13] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[10] - puSrc2->ai16[11]);
+    uDst.ai16[14] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[12] - puSrc2->ai16[13]);
+    uDst.ai16[15] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD(puSrc2->ai16[14] - puSrc2->ai16[15]);
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+    puDst->au64[2] = uDst.au64[2];
+    puDst->au64[3] = uDst.au64[3];
+}
+
+
+/*
+ * PMADDUBSW / VPMADDUBSW
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaddubsw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst = { 0 }; /* Shut up MSVC. */
+
+    uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[0] * uSrc2.ai8[0] + (uint16_t)uSrc1.au8[1] * uSrc2.ai8[1]);
+    uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[2] * uSrc2.ai8[2] + (uint16_t)uSrc1.au8[3] * uSrc2.ai8[3]);
+    uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[4] * uSrc2.ai8[4] + (uint16_t)uSrc1.au8[5] * uSrc2.ai8[5]);
+    uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[6] * uSrc2.ai8[6] + (uint16_t)uSrc1.au8[7] * uSrc2.ai8[7]);
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmaddubsw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[ 0] * puSrc->ai8[ 0] + (uint16_t)uSrc1.au8[ 1] * puSrc->ai8[ 1]);
+    puDst->ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[ 2] * puSrc->ai8[ 2] + (uint16_t)uSrc1.au8[ 3] * puSrc->ai8[ 3]);
+    puDst->ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[ 4] * puSrc->ai8[ 4] + (uint16_t)uSrc1.au8[ 5] * puSrc->ai8[ 5]);
+    puDst->ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[ 6] * puSrc->ai8[ 6] + (uint16_t)uSrc1.au8[ 7] * puSrc->ai8[ 7]);
+    puDst->ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[ 8] * puSrc->ai8[ 8] + (uint16_t)uSrc1.au8[ 9] * puSrc->ai8[ 9]);
+    puDst->ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[10] * puSrc->ai8[10] + (uint16_t)uSrc1.au8[11] * puSrc->ai8[11]);
+    puDst->ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[12] * puSrc->ai8[12] + (uint16_t)uSrc1.au8[13] * puSrc->ai8[13]);
+    puDst->ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)uSrc1.au8[14] * puSrc->ai8[14] + (uint16_t)uSrc1.au8[15] * puSrc->ai8[15]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaddubsw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 0] * puSrc2->ai8[ 0] + (uint16_t)puSrc1->au8[ 1] * puSrc2->ai8[ 1]);
+    uDst.ai16[1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 2] * puSrc2->ai8[ 2] + (uint16_t)puSrc1->au8[ 3] * puSrc2->ai8[ 3]);
+    uDst.ai16[2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 4] * puSrc2->ai8[ 4] + (uint16_t)puSrc1->au8[ 5] * puSrc2->ai8[ 5]);
+    uDst.ai16[3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 6] * puSrc2->ai8[ 6] + (uint16_t)puSrc1->au8[ 7] * puSrc2->ai8[ 7]);
+    uDst.ai16[4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 8] * puSrc2->ai8[ 8] + (uint16_t)puSrc1->au8[ 9] * puSrc2->ai8[ 9]);
+    uDst.ai16[5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[10] * puSrc2->ai8[10] + (uint16_t)puSrc1->au8[11] * puSrc2->ai8[11]);
+    uDst.ai16[6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[12] * puSrc2->ai8[12] + (uint16_t)puSrc1->au8[13] * puSrc2->ai8[13]);
+    uDst.ai16[7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[14] * puSrc2->ai8[14] + (uint16_t)puSrc1->au8[15] * puSrc2->ai8[15]);
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmaddubsw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[ 0] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 0] * puSrc2->ai8[ 0] + (uint16_t)puSrc1->au8[ 1] * puSrc2->ai8[ 1]);
+    uDst.ai16[ 1] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 2] * puSrc2->ai8[ 2] + (uint16_t)puSrc1->au8[ 3] * puSrc2->ai8[ 3]);
+    uDst.ai16[ 2] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 4] * puSrc2->ai8[ 4] + (uint16_t)puSrc1->au8[ 5] * puSrc2->ai8[ 5]);
+    uDst.ai16[ 3] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 6] * puSrc2->ai8[ 6] + (uint16_t)puSrc1->au8[ 7] * puSrc2->ai8[ 7]);
+    uDst.ai16[ 4] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[ 8] * puSrc2->ai8[ 8] + (uint16_t)puSrc1->au8[ 9] * puSrc2->ai8[ 9]);
+    uDst.ai16[ 5] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[10] * puSrc2->ai8[10] + (uint16_t)puSrc1->au8[11] * puSrc2->ai8[11]);
+    uDst.ai16[ 6] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[12] * puSrc2->ai8[12] + (uint16_t)puSrc1->au8[13] * puSrc2->ai8[13]);
+    uDst.ai16[ 7] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[14] * puSrc2->ai8[14] + (uint16_t)puSrc1->au8[15] * puSrc2->ai8[15]);
+    uDst.ai16[ 8] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[16] * puSrc2->ai8[16] + (uint16_t)puSrc1->au8[17] * puSrc2->ai8[17]);
+    uDst.ai16[ 9] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[18] * puSrc2->ai8[18] + (uint16_t)puSrc1->au8[19] * puSrc2->ai8[19]);
+    uDst.ai16[10] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[20] * puSrc2->ai8[20] + (uint16_t)puSrc1->au8[21] * puSrc2->ai8[21]);
+    uDst.ai16[11] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[22] * puSrc2->ai8[22] + (uint16_t)puSrc1->au8[23] * puSrc2->ai8[23]);
+    uDst.ai16[12] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[24] * puSrc2->ai8[24] + (uint16_t)puSrc1->au8[25] * puSrc2->ai8[25]);
+    uDst.ai16[13] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[26] * puSrc2->ai8[26] + (uint16_t)puSrc1->au8[27] * puSrc2->ai8[27]);
+    uDst.ai16[14] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[28] * puSrc2->ai8[28] + (uint16_t)puSrc1->au8[29] * puSrc2->ai8[29]);
+    uDst.ai16[15] = SATURATED_SIGNED_DWORD_TO_SIGNED_WORD((uint16_t)puSrc1->au8[30] * puSrc2->ai8[30] + (uint16_t)puSrc1->au8[31] * puSrc2->ai8[31]);
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+    puDst->au64[2] = uDst.au64[2];
+    puDst->au64[3] = uDst.au64[3];
+}
+
+
+/*
+ * PMULHRSW / VPMULHRSW
+ */
+#define DO_PMULHRSW(a_Src1, a_Src2) \
+    (uint16_t)(((((int32_t)(a_Src1) * (a_Src2)) >> 14 ) + 1) >> 1)
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhrsw_u64_fallback,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+
+    uDst.au16[0] = DO_PMULHRSW(uSrc1.ai16[0], uSrc2.ai16[0]);
+    uDst.au16[1] = DO_PMULHRSW(uSrc1.ai16[1], uSrc2.ai16[1]);
+    uDst.au16[2] = DO_PMULHRSW(uSrc1.ai16[2], uSrc2.ai16[2]);
+    uDst.au16[3] = DO_PMULHRSW(uSrc1.ai16[3], uSrc2.ai16[3]);
+    *puDst = uDst.u;
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmulhrsw_u128_fallback,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->ai16[0] = DO_PMULHRSW(uSrc1.ai16[0], puSrc->ai16[0]);
+    puDst->ai16[1] = DO_PMULHRSW(uSrc1.ai16[1], puSrc->ai16[1]);
+    puDst->ai16[2] = DO_PMULHRSW(uSrc1.ai16[2], puSrc->ai16[2]);
+    puDst->ai16[3] = DO_PMULHRSW(uSrc1.ai16[3], puSrc->ai16[3]);
+    puDst->ai16[4] = DO_PMULHRSW(uSrc1.ai16[4], puSrc->ai16[4]);
+    puDst->ai16[5] = DO_PMULHRSW(uSrc1.ai16[5], puSrc->ai16[5]);
+    puDst->ai16[6] = DO_PMULHRSW(uSrc1.ai16[6], puSrc->ai16[6]);
+    puDst->ai16[7] = DO_PMULHRSW(uSrc1.ai16[7], puSrc->ai16[7]);
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhrsw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[0] = DO_PMULHRSW(puSrc1->ai16[0], puSrc2->ai16[0]);
+    uDst.ai16[1] = DO_PMULHRSW(puSrc1->ai16[1], puSrc2->ai16[1]);
+    uDst.ai16[2] = DO_PMULHRSW(puSrc1->ai16[2], puSrc2->ai16[2]);
+    uDst.ai16[3] = DO_PMULHRSW(puSrc1->ai16[3], puSrc2->ai16[3]);
+    uDst.ai16[4] = DO_PMULHRSW(puSrc1->ai16[4], puSrc2->ai16[4]);
+    uDst.ai16[5] = DO_PMULHRSW(puSrc1->ai16[5], puSrc2->ai16[5]);
+    uDst.ai16[6] = DO_PMULHRSW(puSrc1->ai16[6], puSrc2->ai16[6]);
+    uDst.ai16[7] = DO_PMULHRSW(puSrc1->ai16[7], puSrc2->ai16[7]);
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmulhrsw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uDst; /* puDst can be the same as one of the source operands. */
+
+    uDst.ai16[ 0] = DO_PMULHRSW(puSrc1->ai16[ 0], puSrc2->ai16[ 0]);
+    uDst.ai16[ 1] = DO_PMULHRSW(puSrc1->ai16[ 1], puSrc2->ai16[ 1]);
+    uDst.ai16[ 2] = DO_PMULHRSW(puSrc1->ai16[ 2], puSrc2->ai16[ 2]);
+    uDst.ai16[ 3] = DO_PMULHRSW(puSrc1->ai16[ 3], puSrc2->ai16[ 3]);
+    uDst.ai16[ 4] = DO_PMULHRSW(puSrc1->ai16[ 4], puSrc2->ai16[ 4]);
+    uDst.ai16[ 5] = DO_PMULHRSW(puSrc1->ai16[ 5], puSrc2->ai16[ 5]);
+    uDst.ai16[ 6] = DO_PMULHRSW(puSrc1->ai16[ 6], puSrc2->ai16[ 6]);
+    uDst.ai16[ 7] = DO_PMULHRSW(puSrc1->ai16[ 7], puSrc2->ai16[ 7]);
+    uDst.ai16[ 8] = DO_PMULHRSW(puSrc1->ai16[ 8], puSrc2->ai16[ 8]);
+    uDst.ai16[ 9] = DO_PMULHRSW(puSrc1->ai16[ 9], puSrc2->ai16[ 9]);
+    uDst.ai16[10] = DO_PMULHRSW(puSrc1->ai16[10], puSrc2->ai16[10]);
+    uDst.ai16[11] = DO_PMULHRSW(puSrc1->ai16[11], puSrc2->ai16[11]);
+    uDst.ai16[12] = DO_PMULHRSW(puSrc1->ai16[12], puSrc2->ai16[12]);
+    uDst.ai16[13] = DO_PMULHRSW(puSrc1->ai16[13], puSrc2->ai16[13]);
+    uDst.ai16[14] = DO_PMULHRSW(puSrc1->ai16[14], puSrc2->ai16[14]);
+    uDst.ai16[15] = DO_PMULHRSW(puSrc1->ai16[15], puSrc2->ai16[15]);
+
+    puDst->au64[0] = uDst.au64[0];
+    puDst->au64[1] = uDst.au64[1];
+    puDst->au64[2] = uDst.au64[2];
+    puDst->au64[3] = uDst.au64[3];
+}
+
+
+/*
+ * PSADBW / VPSADBW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psadbw_u64,(uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    RTUINT64U uDst;
+    uint16_t uSum = RT_ABS((int16_t)uSrc1.au8[0] - uSrc2.au8[0]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[1] - uSrc2.au8[1]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[2] - uSrc2.au8[2]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[3] - uSrc2.au8[3]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[4] - uSrc2.au8[4]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[5] - uSrc2.au8[5]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[6] - uSrc2.au8[6]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[7] - uSrc2.au8[7]);
+
+    uDst.au64[0] = 0;
+    uDst.au16[0] = uSum;
+    *puDst = uDst.u;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_psadbw_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au64[0] = 0;
+    puDst->au64[1] = 0;
+
+    uint16_t uSum = RT_ABS((int16_t)uSrc1.ai8[0] - puSrc->ai8[0]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[1] - puSrc->au8[1]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[2] - puSrc->au8[2]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[3] - puSrc->au8[3]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[4] - puSrc->au8[4]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[5] - puSrc->au8[5]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[6] - puSrc->au8[6]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[7] - puSrc->au8[7]);
+    puDst->au16[0] = uSum;
+
+    uSum  = RT_ABS((int16_t)uSrc1.au8[ 8] - puSrc->au8[ 8]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[ 9] - puSrc->au8[ 9]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[10] - puSrc->au8[10]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[11] - puSrc->au8[11]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[12] - puSrc->au8[12]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[13] - puSrc->au8[13]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[14] - puSrc->au8[14]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[15] - puSrc->au8[15]);
+    puDst->au16[4] = uSum;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsadbw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uSrc1 = *puSrc1;
+    RTUINT128U uSrc2 = *puSrc2;
+
+    puDst->au64[0] = 0;
+    puDst->au64[1] = 0;
+
+    uint16_t uSum = RT_ABS((int16_t)uSrc1.ai8[0] - uSrc2.ai8[0]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[1] - uSrc2.au8[1]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[2] - uSrc2.au8[2]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[3] - uSrc2.au8[3]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[4] - uSrc2.au8[4]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[5] - uSrc2.au8[5]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[6] - uSrc2.au8[6]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[7] - uSrc2.au8[7]);
+    puDst->au16[0] = uSum;
+
+    uSum  = RT_ABS((int16_t)uSrc1.au8[ 8] - uSrc2.au8[ 8]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[ 9] - uSrc2.au8[ 9]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[10] - uSrc2.au8[10]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[11] - uSrc2.au8[11]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[12] - uSrc2.au8[12]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[13] - uSrc2.au8[13]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[14] - uSrc2.au8[14]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[15] - uSrc2.au8[15]);
+    puDst->au16[4] = uSum;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpsadbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uSrc1 = *puSrc1;
+    RTUINT256U uSrc2 = *puSrc2;
+
+    puDst->au64[0] = 0;
+    puDst->au64[1] = 0;
+    puDst->au64[2] = 0;
+    puDst->au64[3] = 0;
+
+    uint16_t uSum = RT_ABS((int16_t)uSrc1.au8[0] - uSrc2.au8[0]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[1] - uSrc2.au8[1]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[2] - uSrc2.au8[2]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[3] - uSrc2.au8[3]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[4] - uSrc2.au8[4]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[5] - uSrc2.au8[5]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[6] - uSrc2.au8[6]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[7] - uSrc2.au8[7]);
+    puDst->au16[0] = uSum;
+
+    uSum  = RT_ABS((int16_t)uSrc1.au8[ 8] - uSrc2.au8[ 8]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[ 9] - uSrc2.au8[ 9]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[10] - uSrc2.au8[10]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[11] - uSrc2.au8[11]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[12] - uSrc2.au8[12]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[13] - uSrc2.au8[13]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[14] - uSrc2.au8[14]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[15] - uSrc2.au8[15]);
+    puDst->au16[4] = uSum;
+
+    uSum  = RT_ABS((int16_t)uSrc1.au8[16] - uSrc2.au8[16]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[17] - uSrc2.au8[17]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[18] - uSrc2.au8[18]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[19] - uSrc2.au8[19]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[20] - uSrc2.au8[20]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[21] - uSrc2.au8[21]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[22] - uSrc2.au8[22]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[23] - uSrc2.au8[23]);
+    puDst->au16[8] = uSum;
+
+    uSum  = RT_ABS((int16_t)uSrc1.au8[24] - uSrc2.au8[24]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[25] - uSrc2.au8[25]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[26] - uSrc2.au8[26]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[27] - uSrc2.au8[27]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[28] - uSrc2.au8[28]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[29] - uSrc2.au8[29]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[30] - uSrc2.au8[30]);
+    uSum += RT_ABS((int16_t)uSrc1.au8[31] - uSrc2.au8[31]);
+    puDst->au16[12] = uSum;
+}
+
+
+/*
+ * PMULDQ / VPMULDQ
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmuldq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+
+    puDst->au64[0] = (int64_t)uSrc1.ai32[0] * puSrc->ai32[0];
+    puDst->au64[1] = (int64_t)uSrc1.ai32[2] * puSrc->ai32[2];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmuldq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uSrc1 = *puSrc1;
+    RTUINT128U uSrc2 = *puSrc2;
+
+    puDst->au64[0] = (int64_t)uSrc1.ai32[0] * uSrc2.ai32[0];
+    puDst->au64[1] = (int64_t)uSrc1.ai32[2] * uSrc2.ai32[2];
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmuldq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uSrc1 = *puSrc1;
+    RTUINT256U uSrc2 = *puSrc2;
+
+    puDst->au64[0] = (int64_t)uSrc1.ai32[0] * uSrc2.ai32[0];
+    puDst->au64[1] = (int64_t)uSrc1.ai32[2] * uSrc2.ai32[2];
+    puDst->au64[2] = (int64_t)uSrc1.ai32[4] * uSrc2.ai32[4];
+    puDst->au64[3] = (int64_t)uSrc1.ai32[6] * uSrc2.ai32[6];
+}
+
+
+/*
+ * PMULUDQ / VPMULUDQ
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmuludq_u64,(PCX86FXSTATE pFpuState, uint64_t *puDst, uint64_t const *puSrc))
+{
+    RTUINT64U uSrc1 = { *puDst };
+    RTUINT64U uSrc2 = { *puSrc };
+    ASMCompilerBarrier();
+    *puDst = (uint64_t)uSrc1.au32[0] * uSrc2.au32[0];
+    RT_NOREF(pFpuState);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pmuludq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+    RTUINT128U uSrc2 = *puSrc;
+    ASMCompilerBarrier();
+    puDst->au64[0] = (uint64_t)uSrc1.au32[0] * uSrc2.au32[0];
+    puDst->au64[1] = (uint64_t)uSrc1.au32[2] * uSrc2.au32[2];
+    RT_NOREF(pFpuState);
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmuludq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uSrc1 = *puSrc1; /* Could overlap with puDst */
+    RTUINT128U uSrc2 = *puSrc2; /* Could overlap with puDst */
+    ASMCompilerBarrier();
+    puDst->au64[0] = (uint64_t)uSrc1.au32[0] * uSrc2.au32[0];
+    puDst->au64[1] = (uint64_t)uSrc1.au32[2] * uSrc2.au32[2];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmuludq_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uSrc1 = *puSrc1; /* Could overlap with puDst */
+    RTUINT256U uSrc2 = *puSrc2; /* Could overlap with puDst */
+    ASMCompilerBarrier();
+    puDst->au64[0] = (uint64_t)uSrc1.au32[0] * uSrc2.au32[0];
+    puDst->au64[1] = (uint64_t)uSrc1.au32[2] * uSrc2.au32[2];
+    puDst->au64[2] = (uint64_t)uSrc1.au32[4] * uSrc2.au32[4];
+    puDst->au64[3] = (uint64_t)uSrc1.au32[6] * uSrc2.au32[6];
+}
+
+
+/*
+ * UNPCKLPS / VUNPCKLPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_unpcklps_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+    RTUINT128U uSrc2 = *puSrc;
+    ASMCompilerBarrier();
+    puDst->au32[0] = uSrc1.au32[0];
+    puDst->au32[1] = uSrc2.au32[0];
+    puDst->au32[2] = uSrc1.au32[1];
+    puDst->au32[3] = uSrc2.au32[1];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vunpcklps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uSrc1 = *puSrc1; /* Could overlap with puDst */
+    RTUINT128U uSrc2 = *puSrc2; /* Could overlap with puDst */
+    ASMCompilerBarrier();
+    puDst->au32[0] = uSrc1.au32[0];
+    puDst->au32[1] = uSrc2.au32[0];
+    puDst->au32[2] = uSrc1.au32[1];
+    puDst->au32[3] = uSrc2.au32[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vunpcklps_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uSrc1 = *puSrc1; /* Could overlap with puDst */
+    RTUINT256U uSrc2 = *puSrc2; /* Could overlap with puDst */
+    ASMCompilerBarrier();
+    puDst->au32[0] = uSrc1.au32[0];
+    puDst->au32[1] = uSrc2.au32[0];
+    puDst->au32[2] = uSrc1.au32[1];
+    puDst->au32[3] = uSrc2.au32[1];
+
+    puDst->au32[4] = uSrc1.au32[4];
+    puDst->au32[5] = uSrc2.au32[4];
+    puDst->au32[6] = uSrc1.au32[5];
+    puDst->au32[7] = uSrc2.au32[5];
+}
+
+
+/*
+ * UNPCKLPD / VUNPCKLPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_unpcklpd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+    RTUINT128U uSrc2 = *puSrc;
+    ASMCompilerBarrier();
+    puDst->au64[0] = uSrc1.au64[0];
+    puDst->au64[1] = uSrc2.au64[0];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vunpcklpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uSrc1 = *puSrc1; /* Could overlap with puDst */
+    RTUINT128U uSrc2 = *puSrc2; /* Could overlap with puDst */
+    ASMCompilerBarrier();
+    puDst->au64[0] = uSrc1.au64[0];
+    puDst->au64[1] = uSrc2.au64[0];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vunpcklpd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uSrc1 = *puSrc1; /* Could overlap with puDst */
+    RTUINT256U uSrc2 = *puSrc2; /* Could overlap with puDst */
+    ASMCompilerBarrier();
+    puDst->au64[0] = uSrc1.au64[0];
+    puDst->au64[1] = uSrc2.au64[0];
+    puDst->au64[2] = uSrc1.au64[2];
+    puDst->au64[3] = uSrc2.au64[2];
+}
+
+
+/*
+ * UNPCKHPS / VUNPCKHPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_unpckhps_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+    RTUINT128U uSrc2 = *puSrc;
+    ASMCompilerBarrier();
+    puDst->au32[0] = uSrc1.au32[2];
+    puDst->au32[1] = uSrc2.au32[2];
+    puDst->au32[2] = uSrc1.au32[3];
+    puDst->au32[3] = uSrc2.au32[3];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vunpckhps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uSrc1 = *puSrc1; /* Could overlap with puDst */
+    RTUINT128U uSrc2 = *puSrc2; /* Could overlap with puDst */
+    ASMCompilerBarrier();
+    puDst->au32[0] = uSrc1.au32[2];
+    puDst->au32[1] = uSrc2.au32[2];
+    puDst->au32[2] = uSrc1.au32[3];
+    puDst->au32[3] = uSrc2.au32[3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vunpckhps_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uSrc1 = *puSrc1; /* Could overlap with puDst */
+    RTUINT256U uSrc2 = *puSrc2; /* Could overlap with puDst */
+    ASMCompilerBarrier();
+    puDst->au32[0] = uSrc1.au32[2];
+    puDst->au32[1] = uSrc2.au32[2];
+    puDst->au32[2] = uSrc1.au32[3];
+    puDst->au32[3] = uSrc2.au32[3];
+
+    puDst->au32[4] = uSrc1.au32[6];
+    puDst->au32[5] = uSrc2.au32[6];
+    puDst->au32[6] = uSrc1.au32[7];
+    puDst->au32[7] = uSrc2.au32[7];
+}
+
+
+/*
+ * UNPCKHPD / VUNPCKHPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_unpckhpd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puDst;
+    RTUINT128U uSrc2 = *puSrc;
+    ASMCompilerBarrier();
+    puDst->au64[0] = uSrc1.au64[1];
+    puDst->au64[1] = uSrc2.au64[1];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vunpckhpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2))
+{
+    RTUINT128U uSrc1 = *puSrc1; /* Could overlap with puDst */
+    RTUINT128U uSrc2 = *puSrc2; /* Could overlap with puDst */
+    ASMCompilerBarrier();
+    puDst->au64[0] = uSrc1.au64[1];
+    puDst->au64[1] = uSrc2.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vunpckhpd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2))
+{
+    RTUINT256U uSrc1 = *puSrc1; /* Could overlap with puDst */
+    RTUINT256U uSrc2 = *puSrc2; /* Could overlap with puDst */
+    ASMCompilerBarrier();
+    puDst->au64[0] = uSrc1.au64[1];
+    puDst->au64[1] = uSrc2.au64[1];
+    puDst->au64[2] = uSrc1.au64[3];
+    puDst->au64[3] = uSrc2.au64[3];
+}
+
+
+/*
+ * CRC32 (SEE 4.2).
+ */
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_crc32_u8_fallback,(uint32_t *puDst, uint8_t uSrc))
+{
+    *puDst = RTCrc32CProcess(*puDst, &uSrc, sizeof(uSrc));
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_crc32_u16_fallback,(uint32_t *puDst, uint16_t uSrc))
+{
+    *puDst = RTCrc32CProcess(*puDst, &uSrc, sizeof(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_crc32_u32_fallback,(uint32_t *puDst, uint32_t uSrc))
+{
+    *puDst = RTCrc32CProcess(*puDst, &uSrc, sizeof(uSrc));
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_crc32_u64_fallback,(uint32_t *puDst, uint64_t uSrc))
+{
+    *puDst = RTCrc32CProcess(*puDst, &uSrc, sizeof(uSrc));
+}
+
+
+/*
+ * PTEST (SSE 4.1) - special as it output only EFLAGS.
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_ptest_u128,(PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint32_t *pfEFlags))
+{
+    uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+    if (   (puSrc1->au64[0] & puSrc2->au64[0]) == 0
+        && (puSrc1->au64[1] & puSrc2->au64[1]) == 0)
+        fEfl |= X86_EFL_ZF;
+    if (   (~puSrc1->au64[0] & puSrc2->au64[0]) == 0
+        && (~puSrc1->au64[1] & puSrc2->au64[1]) == 0)
+        fEfl |= X86_EFL_CF;
+    *pfEFlags = fEfl;
+}
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vptest_u256_fallback,(PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint32_t *pfEFlags))
+{
+    uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS;
+    if (   (puSrc1->au64[0] & puSrc2->au64[0]) == 0
+        && (puSrc1->au64[1] & puSrc2->au64[1]) == 0
+        && (puSrc1->au64[2] & puSrc2->au64[2]) == 0
+        && (puSrc1->au64[3] & puSrc2->au64[3]) == 0)
+        fEfl |= X86_EFL_ZF;
+    if (   (~puSrc1->au64[0] & puSrc2->au64[0]) == 0
+        && (~puSrc1->au64[1] & puSrc2->au64[1]) == 0
+        && (~puSrc1->au64[2] & puSrc2->au64[2]) == 0
+        && (~puSrc1->au64[3] & puSrc2->au64[3]) == 0)
+        fEfl |= X86_EFL_CF;
+    *pfEFlags = fEfl;
+}
+
+
+/*
+ * PMOVSXBW / VPMOVSXBW
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbw_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc))
+{
+    RTUINT64U uSrc1 = { uSrc };
+    puDst->ai16[0] = uSrc1.ai8[0];
+    puDst->ai16[1] = uSrc1.ai8[1];
+    puDst->ai16[2] = uSrc1.ai8[2];
+    puDst->ai16[3] = uSrc1.ai8[3];
+    puDst->ai16[4] = uSrc1.ai8[4];
+    puDst->ai16[5] = uSrc1.ai8[5];
+    puDst->ai16[6] = uSrc1.ai8[6];
+    puDst->ai16[7] = uSrc1.ai8[7];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->ai16[ 0] = uSrc1.ai8[ 0];
+    puDst->ai16[ 1] = uSrc1.ai8[ 1];
+    puDst->ai16[ 2] = uSrc1.ai8[ 2];
+    puDst->ai16[ 3] = uSrc1.ai8[ 3];
+    puDst->ai16[ 4] = uSrc1.ai8[ 4];
+    puDst->ai16[ 5] = uSrc1.ai8[ 5];
+    puDst->ai16[ 6] = uSrc1.ai8[ 6];
+    puDst->ai16[ 7] = uSrc1.ai8[ 7];
+    puDst->ai16[ 8] = uSrc1.ai8[ 8];
+    puDst->ai16[ 9] = uSrc1.ai8[ 9];
+    puDst->ai16[10] = uSrc1.ai8[10];
+    puDst->ai16[11] = uSrc1.ai8[11];
+    puDst->ai16[12] = uSrc1.ai8[12];
+    puDst->ai16[13] = uSrc1.ai8[13];
+    puDst->ai16[14] = uSrc1.ai8[14];
+    puDst->ai16[15] = uSrc1.ai8[15];
+}
+
+
+/*
+ * PMOVSXBD / VPMOVSXBD
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbd_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc))
+{
+    RTUINT32U uSrc1 = { uSrc };
+    puDst->ai32[0] = uSrc1.ai8[0];
+    puDst->ai32[1] = uSrc1.ai8[1];
+    puDst->ai32[2] = uSrc1.ai8[2];
+    puDst->ai32[3] = uSrc1.ai8[3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->ai32[0] = uSrc1.ai8[0];
+    puDst->ai32[1] = uSrc1.ai8[1];
+    puDst->ai32[2] = uSrc1.ai8[2];
+    puDst->ai32[3] = uSrc1.ai8[3];
+    puDst->ai32[4] = uSrc1.ai8[4];
+    puDst->ai32[5] = uSrc1.ai8[5];
+    puDst->ai32[6] = uSrc1.ai8[6];
+    puDst->ai32[7] = uSrc1.ai8[7];
+}
+
+
+/*
+ * PMOVSXBQ / VPMOVSXBQ
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbq_u128_fallback,(PRTUINT128U puDst, uint16_t uSrc))
+{
+    RTUINT16U uSrc1 = { uSrc };
+    puDst->ai64[0] = uSrc1.ai8[0];
+    puDst->ai64[1] = uSrc1.ai8[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxbq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->ai64[0] = uSrc1.ai8[0];
+    puDst->ai64[1] = uSrc1.ai8[1];
+    puDst->ai64[2] = uSrc1.ai8[2];
+    puDst->ai64[3] = uSrc1.ai8[3];
+}
+
+
+/*
+ * PMOVSXWD / VPMOVSXWD
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwd_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc))
+{
+    RTUINT64U uSrc1 = { uSrc };
+    puDst->ai32[0] = uSrc1.ai16[0];
+    puDst->ai32[1] = uSrc1.ai16[1];
+    puDst->ai32[2] = uSrc1.ai16[2];
+    puDst->ai32[3] = uSrc1.ai16[3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->ai32[0] = uSrc1.ai16[0];
+    puDst->ai32[1] = uSrc1.ai16[1];
+    puDst->ai32[2] = uSrc1.ai16[2];
+    puDst->ai32[3] = uSrc1.ai16[3];
+    puDst->ai32[4] = uSrc1.ai16[4];
+    puDst->ai32[5] = uSrc1.ai16[5];
+    puDst->ai32[6] = uSrc1.ai16[6];
+    puDst->ai32[7] = uSrc1.ai16[7];
+}
+
+
+/*
+ * PMOVSXWQ / VPMOVSXWQ
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwq_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc))
+{
+    RTUINT32U uSrc1 = { uSrc };
+    puDst->ai64[0] = uSrc1.ai16[0];
+    puDst->ai64[1] = uSrc1.ai16[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxwq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->ai64[0] = uSrc1.ai16[0];
+    puDst->ai64[1] = uSrc1.ai16[1];
+    puDst->ai64[2] = uSrc1.ai16[2];
+    puDst->ai64[3] = uSrc1.ai16[3];
+}
+
+
+/*
+ * PMOVSXDQ / VPMOVSXDQ
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxdq_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc))
+{
+    RTUINT64U uSrc1 = { uSrc };
+    puDst->ai64[0] = uSrc1.ai32[0];
+    puDst->ai64[1] = uSrc1.ai32[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovsxdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->ai64[0] = uSrc1.ai32[0];
+    puDst->ai64[1] = uSrc1.ai32[1];
+    puDst->ai64[2] = uSrc1.ai32[2];
+    puDst->ai64[3] = uSrc1.ai32[3];
+}
+
+
+/*
+ * PMOVZXBW / VPMOVZXBW
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbw_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc))
+{
+    RTUINT64U uSrc1 = { uSrc };
+    puDst->au16[0] = uSrc1.au8[0];
+    puDst->au16[1] = uSrc1.au8[1];
+    puDst->au16[2] = uSrc1.au8[2];
+    puDst->au16[3] = uSrc1.au8[3];
+    puDst->au16[4] = uSrc1.au8[4];
+    puDst->au16[5] = uSrc1.au8[5];
+    puDst->au16[6] = uSrc1.au8[6];
+    puDst->au16[7] = uSrc1.au8[7];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbw_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->au16[ 0] = uSrc1.au8[ 0];
+    puDst->au16[ 1] = uSrc1.au8[ 1];
+    puDst->au16[ 2] = uSrc1.au8[ 2];
+    puDst->au16[ 3] = uSrc1.au8[ 3];
+    puDst->au16[ 4] = uSrc1.au8[ 4];
+    puDst->au16[ 5] = uSrc1.au8[ 5];
+    puDst->au16[ 6] = uSrc1.au8[ 6];
+    puDst->au16[ 7] = uSrc1.au8[ 7];
+    puDst->au16[ 8] = uSrc1.au8[ 8];
+    puDst->au16[ 9] = uSrc1.au8[ 9];
+    puDst->au16[10] = uSrc1.au8[10];
+    puDst->au16[11] = uSrc1.au8[11];
+    puDst->au16[12] = uSrc1.au8[12];
+    puDst->au16[13] = uSrc1.au8[13];
+    puDst->au16[14] = uSrc1.au8[14];
+    puDst->au16[15] = uSrc1.au8[15];
+}
+
+
+/*
+ * PMOVZXBD / VPMOVZXBD
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbd_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc))
+{
+    RTUINT32U uSrc1 = { uSrc };
+    puDst->au32[0] = uSrc1.au8[0];
+    puDst->au32[1] = uSrc1.au8[1];
+    puDst->au32[2] = uSrc1.au8[2];
+    puDst->au32[3] = uSrc1.au8[3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->au32[0] = uSrc1.au8[0];
+    puDst->au32[1] = uSrc1.au8[1];
+    puDst->au32[2] = uSrc1.au8[2];
+    puDst->au32[3] = uSrc1.au8[3];
+    puDst->au32[4] = uSrc1.au8[4];
+    puDst->au32[5] = uSrc1.au8[5];
+    puDst->au32[6] = uSrc1.au8[6];
+    puDst->au32[7] = uSrc1.au8[7];
+}
+
+
+/*
+ * PMOVZXBQ / VPMOVZXBQ
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbq_u128_fallback,(PRTUINT128U puDst, uint16_t uSrc))
+{
+    RTUINT16U uSrc1 = { uSrc };
+    puDst->au64[0] = uSrc1.au8[0];
+    puDst->au64[1] = uSrc1.au8[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxbq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->au64[0] = uSrc1.au8[0];
+    puDst->au64[1] = uSrc1.au8[1];
+    puDst->au64[2] = uSrc1.au8[2];
+    puDst->au64[3] = uSrc1.au8[3];
+}
+
+
+/*
+ * PMOVZXWD / VPMOVZXWD
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwd_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc))
+{
+    RTUINT64U uSrc1 = { uSrc };
+    puDst->au32[0] = uSrc1.au16[0];
+    puDst->au32[1] = uSrc1.au16[1];
+    puDst->au32[2] = uSrc1.au16[2];
+    puDst->au32[3] = uSrc1.au16[3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwd_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->au32[0] = uSrc1.au16[0];
+    puDst->au32[1] = uSrc1.au16[1];
+    puDst->au32[2] = uSrc1.au16[2];
+    puDst->au32[3] = uSrc1.au16[3];
+    puDst->au32[4] = uSrc1.au16[4];
+    puDst->au32[5] = uSrc1.au16[5];
+    puDst->au32[6] = uSrc1.au16[6];
+    puDst->au32[7] = uSrc1.au16[7];
+}
+
+
+/*
+ * PMOVZXWQ / VPMOVZXWQ
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwq_u128_fallback,(PRTUINT128U puDst, uint32_t uSrc))
+{
+    RTUINT32U uSrc1 = { uSrc };
+    puDst->au64[0] = uSrc1.au16[0];
+    puDst->au64[1] = uSrc1.au16[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxwq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->au64[0] = uSrc1.au16[0];
+    puDst->au64[1] = uSrc1.au16[1];
+    puDst->au64[2] = uSrc1.au16[2];
+    puDst->au64[3] = uSrc1.au16[3];
+}
+
+
+/*
+ * PMOVZXDQ / VPMOVZXDQ
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxdq_u128_fallback,(PRTUINT128U puDst, uint64_t uSrc))
+{
+    RTUINT64U uSrc1 = { uSrc };
+    puDst->au64[0] = uSrc1.au32[0];
+    puDst->au64[1] = uSrc1.au32[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpmovzxdq_u256_fallback,(PRTUINT256U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U uSrc1 = *puSrc; /* puDst could overlap */
+    puDst->au64[0] = uSrc1.au32[0];
+    puDst->au64[1] = uSrc1.au32[1];
+    puDst->au64[2] = uSrc1.au32[2];
+    puDst->au64[3] = uSrc1.au32[3];
+}
+
+/**
+ * Converts from the packed IPRT 32-bit (single precision) floating point format to
+ * the SoftFloat 32-bit floating point format (float32_t).
+ *
+ * This is only a structure format conversion, nothing else.
+ */
+DECLINLINE(float32_t) iemFpSoftF32FromIprt(PCRTFLOAT32U pr32Val)
+{
+    float32_t Tmp;
+    Tmp.v = pr32Val->u;
+    return Tmp;
+}
+
+
+/**
+ * Converts from SoftFloat 32-bit floating point format (float32_t)
+ * to the packed IPRT 32-bit floating point (RTFLOAT32U) format.
+ *
+ * This is only a structure format conversion, nothing else.
+ */
+DECLINLINE(PRTFLOAT32U) iemFpSoftF32ToIprt(PRTFLOAT32U pr32Dst, float32_t const r32XSrc)
+{
+    pr32Dst->u = r32XSrc.v;
+    return pr32Dst;
+}
+
+
+/**
+ * Converts from the packed IPRT 64-bit (single precision) floating point format to
+ * the SoftFloat 64-bit floating point format (float64_t).
+ *
+ * This is only a structure format conversion, nothing else.
+ */
+DECLINLINE(float64_t) iemFpSoftF64FromIprt(PCRTFLOAT64U pr64Val)
+{
+    float64_t Tmp;
+    Tmp.v = pr64Val->u;
+    return Tmp;
+}
+
+
+/**
+ * Converts from SoftFloat 64-bit floating point format (float64_t)
+ * to the packed IPRT 64-bit floating point (RTFLOAT64U) format.
+ *
+ * This is only a structure format conversion, nothing else.
+ */
+DECLINLINE(PRTFLOAT64U) iemFpSoftF64ToIprt(PRTFLOAT64U pr64Dst, float64_t const r64XSrc)
+{
+    pr64Dst->u = r64XSrc.v;
+    return pr64Dst;
+}
+
+
+/** Initializer for the SoftFloat state structure. */
+# define IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(a_Mxcsr) \
+    { \
+        softfloat_tininess_afterRounding, \
+          ((a_Mxcsr) & X86_MXCSR_RC_MASK) == X86_MXCSR_RC_NEAREST ? (uint8_t)softfloat_round_near_even \
+        : ((a_Mxcsr) & X86_MXCSR_RC_MASK) == X86_MXCSR_RC_UP      ? (uint8_t)softfloat_round_max \
+        : ((a_Mxcsr) & X86_MXCSR_RC_MASK) == X86_MXCSR_RC_DOWN    ? (uint8_t)softfloat_round_min \
+        :                                                           (uint8_t)softfloat_round_minMag, \
+        0, \
+        (uint8_t)(((a_Mxcsr) & X86_MXCSR_XCPT_MASK) >> X86_MXCSR_XCPT_MASK_SHIFT), /* Matches X86_FSW_?E */\
+        32 /* Rounding precision, not relevant for SIMD. */ \
+    }
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+/**
+ * Helper for transfering exception to MXCSR and setting the result value
+ * accordingly.
+ *
+ * @returns Updated MXCSR.
+ * @param   pSoftState      The SoftFloat state following the operation.
+ * @param   r32Result       The result of the SoftFloat operation.
+ * @param   pr32Result      Where to store the result for IEM.
+ * @param   fMxcsr          The original MXCSR value.
+ */
+DECLINLINE(uint32_t) iemSseSoftStateAndR32ToMxcsrAndIprtResult(softfloat_state_t const *pSoftState, float32_t r32Result,
+                                                               PRTFLOAT32U pr32Result, uint32_t fMxcsr)
+{
+    iemFpSoftF32ToIprt(pr32Result, r32Result);
+
+    uint8_t fXcpt = pSoftState->exceptionFlags;
+    if (   (fMxcsr & X86_MXCSR_FZ)
+        && RTFLOAT32U_IS_SUBNORMAL(pr32Result))
+    {
+        /* Underflow masked and flush to zero is set. */
+        pr32Result->s.uFraction = 0;
+        pr32Result->s.uExponent = 0;
+        fXcpt |= X86_MXCSR_UE | X86_MXCSR_PE;
+    }
+
+    /* If DAZ is set \#DE is never set. */
+    if (   fMxcsr & X86_MXCSR_DAZ
+        || (   (fXcpt & X86_MXCSR_DE) /* Softfloat sets DE for sub-normal values. */
+            && (RTFLOAT32U_IS_SUBNORMAL(pr32Result))))
+        fXcpt &= ~X86_MXCSR_DE;
+
+    return fMxcsr | (fXcpt & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+/**
+ * Helper for transfering exception to MXCSR and setting the result value
+ * accordingly - ignores Flush-to-Zero.
+ *
+ * @returns Updated MXCSR.
+ * @param   pSoftState      The SoftFloat state following the operation.
+ * @param   r32Result       The result of the SoftFloat operation.
+ * @param   pr32Result      Where to store the result for IEM.
+ * @param   fMxcsr          The original MXCSR value.
+ */
+DECLINLINE(uint32_t) iemSseSoftStateAndR32ToMxcsrAndIprtResultNoFz(softfloat_state_t const *pSoftState, float32_t r32Result,
+                                                                   PRTFLOAT32U pr32Result, uint32_t fMxcsr)
+{
+    iemFpSoftF32ToIprt(pr32Result, r32Result);
+
+    uint8_t fXcpt = pSoftState->exceptionFlags;
+    /* If DAZ is set \#DE is never set. */
+    if (   fMxcsr & X86_MXCSR_DAZ
+        || (   (fXcpt & X86_MXCSR_DE) /* Softfloat sets DE for sub-normal values. */
+            && (RTFLOAT32U_IS_SUBNORMAL(pr32Result))))
+        fXcpt &= ~X86_MXCSR_DE;
+
+    return fMxcsr | (fXcpt & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+/**
+ * Helper for transfering exception to MXCSR and setting the result value
+ * accordingly.
+ *
+ * @returns Updated MXCSR.
+ * @param   pSoftState      The SoftFloat state following the operation.
+ * @param   r64Result       The result of the SoftFloat operation.
+ * @param   pr64Result      Where to store the result for IEM.
+ * @param   fMxcsr          The original MXCSR value.
+ */
+DECLINLINE(uint32_t) iemSseSoftStateAndR64ToMxcsrAndIprtResult(softfloat_state_t const *pSoftState, float64_t r64Result,
+                                                               PRTFLOAT64U pr64Result, uint32_t fMxcsr)
+{
+    iemFpSoftF64ToIprt(pr64Result, r64Result);
+    uint8_t fXcpt = pSoftState->exceptionFlags;
+    if (   (fMxcsr & X86_MXCSR_FZ)
+        && RTFLOAT64U_IS_SUBNORMAL(pr64Result))
+    {
+        /* Underflow masked and flush to zero is set. */
+        iemFpSoftF64ToIprt(pr64Result, r64Result);
+        pr64Result->s.uFractionHigh = 0;
+        pr64Result->s.uFractionLow  = 0;
+        pr64Result->s.uExponent     = 0;
+        fXcpt |= X86_MXCSR_UE | X86_MXCSR_PE;
+    }
+
+    /* If DAZ is set \#DE is never set. */
+    if (   fMxcsr & X86_MXCSR_DAZ
+        || (   (fXcpt & X86_MXCSR_DE) /* Softfloat sets DE for sub-normal values. */
+            && (RTFLOAT64U_IS_SUBNORMAL(pr64Result))))
+        fXcpt &= ~X86_MXCSR_DE;
+
+    return fMxcsr | (fXcpt & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+/**
+ * Helper for transfering exception to MXCSR and setting the result value
+ * accordingly - ignores Flush-to-Zero.
+ *
+ * @returns Updated MXCSR.
+ * @param   pSoftState      The SoftFloat state following the operation.
+ * @param   r64Result       The result of the SoftFloat operation.
+ * @param   pr64Result      Where to store the result for IEM.
+ * @param   fMxcsr          The original MXCSR value.
+ */
+DECLINLINE(uint32_t) iemSseSoftStateAndR64ToMxcsrAndIprtResultNoFz(softfloat_state_t const *pSoftState, float64_t r64Result,
+                                                                   PRTFLOAT64U pr64Result, uint32_t fMxcsr)
+{
+    iemFpSoftF64ToIprt(pr64Result, r64Result);
+
+    uint8_t fXcpt = pSoftState->exceptionFlags;
+    /* If DAZ is set \#DE is never set. */
+    if (   fMxcsr & X86_MXCSR_DAZ
+        || (   (fXcpt & X86_MXCSR_DE) /* Softfloat sets DE for sub-normal values. */
+            && (RTFLOAT64U_IS_SUBNORMAL(pr64Result))))
+        fXcpt &= ~X86_MXCSR_DE;
+
+    return fMxcsr | (fXcpt & X86_MXCSR_XCPT_FLAGS);
+}
+
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+
+/**
+ * Sets the given single precision floating point input value to the given output taking the Denormals-as-zero flag
+ * in MXCSR into account.
+ *
+ * @returns The output MXCSR De-normal flag if the input is a de-normal and the DAZ flag is not set.
+ * @param   pr32Val         Where to store the result.
+ * @param   fMxcsr          The input MXCSR value.
+ * @param   pr32Src         The value to use.
+ */
+DECLINLINE(uint32_t) iemSsePrepareValueR32(PRTFLOAT32U pr32Val, uint32_t fMxcsr, PCRTFLOAT32U pr32Src)
+{
+    if (RTFLOAT32U_IS_SUBNORMAL(pr32Src))
+    {
+        if (fMxcsr & X86_MXCSR_DAZ)
+        {
+            /* De-normals are changed to 0. */
+            pr32Val->s.fSign     = pr32Src->s.fSign;
+            pr32Val->s.uFraction = 0;
+            pr32Val->s.uExponent = 0;
+            return 0;
+        }
+
+        *pr32Val = *pr32Src;
+        return X86_MXCSR_DE;
+    }
+
+    *pr32Val = *pr32Src;
+    return 0;
+}
+
+
+/**
+ * Sets the given double precision floating point input value to the given output taking the Denormals-as-zero flag
+ * in MXCSR into account.
+ *
+ * @returns The output MXCSR De-normal flag if the input is a de-normal and the DAZ flag is not set.
+ * @param   pr64Val         Where to store the result.
+ * @param   fMxcsr          The input MXCSR value.
+ * @param   pr64Src         The value to use.
+ */
+DECLINLINE(uint32_t) iemSsePrepareValueR64(PRTFLOAT64U pr64Val, uint32_t fMxcsr, PCRTFLOAT64U pr64Src)
+{
+    if (RTFLOAT64U_IS_SUBNORMAL(pr64Src))
+    {
+        if (fMxcsr & X86_MXCSR_DAZ)
+        {
+            /* De-normals are changed to 0. */
+            pr64Val->s64.fSign     = pr64Src->s.fSign;
+            pr64Val->s64.uFraction = 0;
+            pr64Val->s64.uExponent = 0;
+            return 0;
+        }
+
+        *pr64Val = *pr64Src;
+        return X86_MXCSR_DE;
+    }
+
+    *pr64Val = *pr64Src;
+    return 0;
+}
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+
+/**
+ * Validates the given input operands returning whether the operation can continue or whether one
+ * of the source operands contains a NaN value, setting the output accordingly.
+ *
+ * @returns Flag whether the operation can continue (false) or whether a NaN value was detected in one of the operands (true).
+ * @param   pr32Res         Where to store the result in case the operation can't continue.
+ * @param   pr32Val1        The first input operand.
+ * @param   pr32Val2        The second input operand.
+ * @param   pfMxcsr         Where to return the modified MXCSR state when false is returned.
+ */
+DECLINLINE(bool) iemSseBinaryValIsNaNR32(PRTFLOAT32U pr32Res, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2, uint32_t *pfMxcsr)
+{
+    uint8_t const cQNan = RTFLOAT32U_IS_QUIET_NAN(pr32Val1)      + RTFLOAT32U_IS_QUIET_NAN(pr32Val2);
+    uint8_t const cSNan = RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val1) + RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val2);
+    if (cSNan + cQNan == 2)
+    {
+        /* Both values are either SNan or QNan, first operand is placed into the result and converted to a QNan. */
+        *pr32Res = *pr32Val1;
+        pr32Res->s.uFraction |= RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1);
+        *pfMxcsr |= (cSNan ? X86_MXCSR_IE : 0);
+        return true;
+    }
+    if (cSNan)
+    {
+        /* One operand is an SNan and placed into the result, converting it to a QNan. */
+        *pr32Res = RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val1) ? *pr32Val1 : *pr32Val2;
+        pr32Res->s.uFraction |= RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1);
+        *pfMxcsr |= X86_MXCSR_IE;
+        return true;
+    }
+    if (cQNan)
+    {
+        /* The QNan operand is placed into the result. */
+        *pr32Res = RTFLOAT32U_IS_QUIET_NAN(pr32Val1) ? *pr32Val1 : *pr32Val2;
+        return true;
+    }
+
+    Assert(!cQNan && !cSNan);
+    return false;
+}
+
+
+/**
+ * Validates the given double precision input operands returning whether the operation can continue or whether one
+ * of the source operands contains a NaN value, setting the output accordingly.
+ *
+ * @returns Flag whether the operation can continue (false) or whether a NaN value was detected in one of the operands (true).
+ * @param   pr64Res         Where to store the result in case the operation can't continue.
+ * @param   pr64Val1        The first input operand.
+ * @param   pr64Val2        The second input operand.
+ * @param   pfMxcsr         Where to return the modified MXCSR state when false is returned.
+ */
+DECLINLINE(bool) iemSseBinaryValIsNaNR64(PRTFLOAT64U pr64Res, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2, uint32_t *pfMxcsr)
+{
+    uint8_t const cQNan = RTFLOAT64U_IS_QUIET_NAN(pr64Val1)      + RTFLOAT64U_IS_QUIET_NAN(pr64Val2);
+    uint8_t const cSNan = RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val1) + RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val2);
+    if (cSNan + cQNan == 2)
+    {
+        /* Both values are either SNan or QNan, first operand is placed into the result and converted to a QNan. */
+        *pr64Res = *pr64Val1;
+        pr64Res->s64.uFraction |= RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1);
+        *pfMxcsr |= (cSNan ? X86_MXCSR_IE : 0);
+        return true;
+    }
+    if (cSNan)
+    {
+        /* One operand is an SNan and placed into the result, converting it to a QNan. */
+        *pr64Res = RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val1) ? *pr64Val1 : *pr64Val2;
+        pr64Res->s64.uFraction |= RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1);
+        *pfMxcsr |= X86_MXCSR_IE;
+        return true;
+    }
+    if (cQNan)
+    {
+        /* The QNan operand is placed into the result. */
+        *pr64Res = RTFLOAT64U_IS_QUIET_NAN(pr64Val1) ? *pr64Val1 : *pr64Val2;
+        return true;
+    }
+
+    Assert(!cQNan && !cSNan);
+    return false;
+}
+
+
+/**
+ * Validates the given single input operand returning whether the operation can continue or whether
+ * contains a NaN value, setting the output accordingly.
+ *
+ * @returns Flag whether the operation can continue (false) or whether a NaN value was detected in the operand (true).
+ * @param   pr32Res         Where to store the result in case the operation can't continue.
+ * @param   pr32Val         The input operand.
+ * @param   pfMxcsr         Where to return the modified MXCSR state when false is returned.
+ */
+DECLINLINE(bool) iemSseUnaryValIsNaNR32(PRTFLOAT32U pr32Res, PCRTFLOAT32U pr32Val, uint32_t *pfMxcsr)
+{
+    if (RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val))
+    {
+        /* One operand is an SNan and placed into the result, converting it to a QNan. */
+        *pr32Res = *pr32Val;
+        pr32Res->s.uFraction |= RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1);
+        *pfMxcsr |= X86_MXCSR_IE;
+        return true;
+    }
+    if (RTFLOAT32U_IS_QUIET_NAN(pr32Val))
+    {
+        /* The QNan operand is placed into the result. */
+        *pr32Res = *pr32Val;
+        return true;
+    }
+
+    return false;
+}
+
+
+/**
+ * Validates the given double input operand returning whether the operation can continue or whether
+ * contains a NaN value, setting the output accordingly.
+ *
+ * @returns Flag whether the operation can continue (false) or whether a NaN value was detected in the operand (true).
+ * @param   pr64Res         Where to store the result in case the operation can't continue.
+ * @param   pr64Val         The input operand.
+ * @param   pfMxcsr         Where to return the modified MXCSR state when false is returned.
+ */
+DECLINLINE(bool) iemSseUnaryValIsNaNR64(PRTFLOAT64U pr64Res, PCRTFLOAT64U pr64Val, uint32_t *pfMxcsr)
+{
+    if (RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val))
+    {
+        /* One operand is an SNan and placed into the result, converting it to a QNan. */
+        *pr64Res = *pr64Val;
+        pr64Res->s64.uFraction |= RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1);
+        *pfMxcsr |= X86_MXCSR_IE;
+        return true;
+    }
+    if (RTFLOAT64U_IS_QUIET_NAN(pr64Val))
+    {
+        /* The QNan operand is placed into the result. */
+        *pr64Res = *pr64Val;
+        return true;
+    }
+
+    return false;
+}
+
+#endif /* IEM_WITHOUT_ASSEMBLY */
+
+/**
+ * ADDPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_addps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2)
+{
+    if (iemSseBinaryValIsNaNR32(pr32Res, pr32Val1, pr32Val2, &fMxcsr))
+        return fMxcsr;
+
+    RTFLOAT32U r32Src1, r32Src2;
+    fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1);
+    fMxcsr |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2);
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float32_t r32Result = f32_add(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_addps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_addps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * ADDSS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_addss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2))
+{
+    pResult->MXCSR = iemAImpl_addps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2);
+    pResult->uResult.ar32[1] = puSrc1->ar32[1];
+    pResult->uResult.ar32[2] = puSrc1->ar32[2];
+    pResult->uResult.ar32[3] = puSrc1->ar32[3];
+}
+#endif
+
+
+/**
+ * ADDPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_addpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2)
+{
+    if (iemSseBinaryValIsNaNR64(pr64Res, pr64Val1, pr64Val2, &fMxcsr))
+        return fMxcsr;
+
+    RTFLOAT64U r64Src1, r64Src2;
+    fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1);
+    fMxcsr |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2);
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float64_t r64Result = f64_add(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_addpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]);
+}
+#endif
+
+
+/**
+ * ADDSD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_addsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2))
+{
+    pResult->MXCSR = iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2);
+    pResult->uResult.ar64[1] = puSrc1->ar64[1];
+}
+#endif
+
+
+/**
+ * MULPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_mulps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2)
+{
+    if (iemSseBinaryValIsNaNR32(pr32Res, pr32Val1, pr32Val2, &fMxcsr))
+        return fMxcsr;
+
+    RTFLOAT32U r32Src1, r32Src2;
+    fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1);
+    fMxcsr |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2);
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float32_t r32Result = f32_mul(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_mulps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_mulps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_mulps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_mulps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_mulps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * MULSS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_mulss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2))
+{
+    pResult->MXCSR = iemAImpl_mulps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2);
+    pResult->uResult.ar32[1] = puSrc1->ar32[1];
+    pResult->uResult.ar32[2] = puSrc1->ar32[2];
+    pResult->uResult.ar32[3] = puSrc1->ar32[3];
+}
+#endif
+
+
+/**
+ * MULPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_mulpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2)
+{
+    if (iemSseBinaryValIsNaNR64(pr64Res, pr64Val1, pr64Val2, &fMxcsr))
+        return fMxcsr;
+
+    RTFLOAT64U r64Src1, r64Src2;
+    fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1);
+    fMxcsr |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2);
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float64_t r64Result = f64_mul(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_mulpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_mulpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_mulpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]);
+}
+#endif
+
+
+/**
+ * MULSD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_mulsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2))
+{
+    pResult->MXCSR = iemAImpl_mulpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2);
+    pResult->uResult.ar64[1] = puSrc1->ar64[1];
+}
+#endif
+
+
+/**
+ * SUBPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_subps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2)
+{
+    if (iemSseBinaryValIsNaNR32(pr32Res, pr32Val1, pr32Val2, &fMxcsr))
+        return fMxcsr;
+
+    RTFLOAT32U r32Src1, r32Src2;
+    fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1);
+    fMxcsr |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2);
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float32_t r32Result = f32_sub(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_subps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_subps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * SUBSS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_subss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2))
+{
+    pResult->MXCSR = iemAImpl_subps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2);
+    pResult->uResult.ar32[1] = puSrc1->ar32[1];
+    pResult->uResult.ar32[2] = puSrc1->ar32[2];
+    pResult->uResult.ar32[3] = puSrc1->ar32[3];
+}
+#endif
+
+
+/**
+ * SUBPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_subpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2)
+{
+    if (iemSseBinaryValIsNaNR64(pr64Res, pr64Val1, pr64Val2, &fMxcsr))
+        return fMxcsr;
+
+    RTFLOAT64U r64Src1, r64Src2;
+    fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1);
+    fMxcsr |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2);
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float64_t r64Result = f64_sub(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_subpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]);
+}
+#endif
+
+
+/**
+ * SUBSD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_subsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2))
+{
+    pResult->MXCSR = iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2);
+    pResult->uResult.ar64[1] = puSrc1->ar64[1];
+}
+#endif
+
+
+/**
+ * MINPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_minps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2)
+{
+    if (RTFLOAT32U_IS_NAN(pr32Val1) || RTFLOAT32U_IS_NAN(pr32Val2))
+    {
+        /* The DAZ flag gets honored but the DE flag will not get set because \#IE has higher priority. */
+        iemSsePrepareValueR32(pr32Res, fMxcsr, pr32Val2);
+        return fMxcsr | X86_MXCSR_IE;
+    }
+
+    RTFLOAT32U r32Src1, r32Src2;
+    fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1);
+    fMxcsr |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2);
+    if (RTFLOAT32U_IS_ZERO(&r32Src1) && RTFLOAT32U_IS_ZERO(&r32Src2))
+    {
+        *pr32Res = r32Src2;
+        return fMxcsr;
+    }
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    bool fLe = f32_le(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResultNoFz(&SoftState,
+                                                           fLe
+                                                         ? iemFpSoftF32FromIprt(&r32Src1)
+                                                         : iemFpSoftF32FromIprt(&r32Src2),
+                                                         pr32Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_minps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_minps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_minps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_minps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_minps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * MINSS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_minss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2))
+{
+    pResult->MXCSR = iemAImpl_minps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2);
+    pResult->uResult.ar32[1] = puSrc1->ar32[1];
+    pResult->uResult.ar32[2] = puSrc1->ar32[2];
+    pResult->uResult.ar32[3] = puSrc1->ar32[3];
+}
+#endif
+
+
+/**
+ * MINPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_minpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2)
+{
+    if (RTFLOAT64U_IS_NAN(pr64Val1) || RTFLOAT64U_IS_NAN(pr64Val2))
+    {
+        /* The DAZ flag gets honored but the DE flag will not get set because \#IE has higher priority. */
+        iemSsePrepareValueR64(pr64Res, fMxcsr, pr64Val2);
+        return fMxcsr | X86_MXCSR_IE;
+    }
+
+    RTFLOAT64U r64Src1, r64Src2;
+    fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1);
+    fMxcsr |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2);
+    if (RTFLOAT64U_IS_ZERO(&r64Src1) && RTFLOAT64U_IS_ZERO(&r64Src2))
+    {
+        *pr64Res = r64Src2;
+        return fMxcsr;
+    }
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    bool fLe = f64_le(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResultNoFz(&SoftState,
+                                                           fLe
+                                                         ? iemFpSoftF64FromIprt(&r64Src1)
+                                                         : iemFpSoftF64FromIprt(&r64Src2),
+                                                         pr64Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_minpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_minpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_minpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]);
+}
+#endif
+
+
+/**
+ * MINSD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_minsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2))
+{
+    pResult->MXCSR = iemAImpl_minpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2);
+    pResult->uResult.ar64[1] = puSrc1->ar64[1];
+}
+#endif
+
+
+/**
+ * DIVPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_divps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2)
+{
+    if (iemSseBinaryValIsNaNR32(pr32Res, pr32Val1, pr32Val2, &fMxcsr))
+        return fMxcsr;
+
+    RTFLOAT32U r32Src1, r32Src2;
+    uint32_t fDe  = iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1);
+             fDe |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2);
+    if (RTFLOAT32U_IS_ZERO(&r32Src2))
+    {
+        if (   RTFLOAT32U_IS_ZERO(&r32Src1)
+            || RTFLOAT32U_IS_QUIET_NAN(&r32Src1))
+        {
+            *pr32Res = g_ar32QNaN[1];
+            return fMxcsr | X86_MXCSR_IE;
+        }
+        else if (RTFLOAT32U_IS_INF(&r32Src1))
+        {
+            *pr32Res = g_ar32Infinity[r32Src1.s.fSign != r32Src2.s.fSign];
+            return fMxcsr;
+        }
+        else
+        {
+            *pr32Res = g_ar32Infinity[r32Src1.s.fSign != r32Src2.s.fSign];
+            return fMxcsr | X86_MXCSR_ZE;
+        }
+    }
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float32_t r32Result = f32_div(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr | fDe);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_divps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_divps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_divps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_divps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_divps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * DIVSS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_divss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2))
+{
+    pResult->MXCSR = iemAImpl_divps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2);
+    pResult->uResult.ar32[1] = puSrc1->ar32[1];
+    pResult->uResult.ar32[2] = puSrc1->ar32[2];
+    pResult->uResult.ar32[3] = puSrc1->ar32[3];
+}
+#endif
+
+
+/**
+ * DIVPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_divpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2)
+{
+    if (iemSseBinaryValIsNaNR64(pr64Res, pr64Val1, pr64Val2, &fMxcsr))
+        return fMxcsr;
+
+    RTFLOAT64U r64Src1, r64Src2;
+    uint32_t fDe  = iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1);
+             fDe |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2);
+    if (RTFLOAT64U_IS_ZERO(&r64Src2))
+    {
+        if (   RTFLOAT64U_IS_ZERO(&r64Src1)
+            || RTFLOAT64U_IS_QUIET_NAN(&r64Src1))
+        {
+            *pr64Res = g_ar64QNaN[1];
+            return fMxcsr | X86_MXCSR_IE;
+        }
+        else if (RTFLOAT64U_IS_INF(&r64Src1))
+        {
+            *pr64Res = g_ar64Infinity[r64Src1.s.fSign != r64Src2.s.fSign];
+            return fMxcsr;
+        }
+        else
+        {
+            *pr64Res = g_ar64Infinity[r64Src1.s.fSign != r64Src2.s.fSign];
+            return fMxcsr | X86_MXCSR_ZE;
+        }
+    }
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float64_t r64Result = f64_div(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr | fDe);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_divpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_divpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_divpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]);
+}
+#endif
+
+
+/**
+ * DIVSD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_divsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2))
+{
+    pResult->MXCSR = iemAImpl_divpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2);
+    pResult->uResult.ar64[1] = puSrc1->ar64[1];
+}
+#endif
+
+
+/**
+ * MAXPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_maxps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1, PCRTFLOAT32U pr32Val2)
+{
+    if (RTFLOAT32U_IS_NAN(pr32Val1) || RTFLOAT32U_IS_NAN(pr32Val2))
+    {
+        /* The DAZ flag gets honored but the DE flag will not get set because \#IE has higher priority. */
+        iemSsePrepareValueR32(pr32Res, fMxcsr, pr32Val2);
+        return fMxcsr | X86_MXCSR_IE;
+    }
+
+    RTFLOAT32U r32Src1, r32Src2;
+    fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1);
+    fMxcsr |= iemSsePrepareValueR32(&r32Src2, fMxcsr, pr32Val2);
+    if (RTFLOAT32U_IS_ZERO(&r32Src1) && RTFLOAT32U_IS_ZERO(&r32Src2))
+    {
+        *pr32Res = r32Src2;
+        return fMxcsr;
+    }
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    bool fLe = f32_le(iemFpSoftF32FromIprt(&r32Src1), iemFpSoftF32FromIprt(&r32Src2), &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResultNoFz(&SoftState,
+                                                           fLe
+                                                         ? iemFpSoftF32FromIprt(&r32Src2)
+                                                         : iemFpSoftF32FromIprt(&r32Src1),
+                                                         pr32Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_maxps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_maxps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_maxps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_maxps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_maxps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * MAXSS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_maxss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2))
+{
+    pResult->MXCSR = iemAImpl_maxps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], pr32Src2);
+    pResult->uResult.ar32[1] = puSrc1->ar32[1];
+    pResult->uResult.ar32[2] = puSrc1->ar32[2];
+    pResult->uResult.ar32[3] = puSrc1->ar32[3];
+}
+#endif
+
+
+/**
+ * MAXPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_maxpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1, PCRTFLOAT64U pr64Val2)
+{
+    if (RTFLOAT64U_IS_NAN(pr64Val1) || RTFLOAT64U_IS_NAN(pr64Val2))
+    {
+        /* The DAZ flag gets honored but the DE flag will not get set because \#IE has higher priority. */
+        iemSsePrepareValueR64(pr64Res, fMxcsr, pr64Val2);
+        return fMxcsr | X86_MXCSR_IE;
+    }
+
+    RTFLOAT64U r64Src1, r64Src2;
+    fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1);
+    fMxcsr |= iemSsePrepareValueR64(&r64Src2, fMxcsr, pr64Val2);
+    if (RTFLOAT64U_IS_ZERO(&r64Src1) && RTFLOAT64U_IS_ZERO(&r64Src2))
+    {
+        *pr64Res = r64Src2;
+        return fMxcsr;
+    }
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    bool fLe = f64_le(iemFpSoftF64FromIprt(&r64Src1), iemFpSoftF64FromIprt(&r64Src2), &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResultNoFz(&SoftState,
+                                                           fLe
+                                                         ? iemFpSoftF64FromIprt(&r64Src2)
+                                                         : iemFpSoftF64FromIprt(&r64Src1),
+                                                         pr64Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_maxpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_maxpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_maxpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]);
+}
+#endif
+
+
+/**
+ * MAXSD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_maxsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2))
+{
+    pResult->MXCSR = iemAImpl_maxpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], pr64Src2);
+    pResult->uResult.ar64[1] = puSrc1->ar64[1];
+}
+#endif
+
+
+/**
+ * CVTSS2SD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtss2sd_u128_r32_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1)
+{
+    RTFLOAT32U r32Src1;
+    fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1);
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float64_t r64Result = f32_to_f64(iemFpSoftF32FromIprt(&r32Src1), &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtss2sd_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2))
+{
+    pResult->MXCSR = iemAImpl_cvtss2sd_u128_r32_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, pr32Src2);
+    pResult->uResult.ar64[1] = puSrc1->ar64[1];
+}
+#endif
+
+
+/**
+ * CVTSD2SS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtsd2ss_u128_r64_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1)
+{
+    RTFLOAT64U r64Src1;
+    fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1);
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float32_t r32Result = f64_to_f32(iemFpSoftF64FromIprt(&r64Src1), &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsd2ss_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2))
+{
+    pResult->MXCSR = iemAImpl_cvtsd2ss_u128_r64_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, pr64Src2);
+    pResult->uResult.ar32[1] = puSrc1->ar32[1];
+    pResult->uResult.ar32[2] = puSrc1->ar32[2];
+    pResult->uResult.ar32[3] = puSrc1->ar32[3];
+}
+#endif
+
+
+/**
+ * HADDPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_haddps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_addps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc1->ar32[1]);
+    pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc1->ar32[3]);
+    pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc2->ar32[0], &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc2->ar32[2], &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * HADDPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_haddpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc1->ar64[1]);
+    pResult->MXCSR |= iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc2->ar64[0], &puSrc2->ar64[1]);
+}
+#endif
+
+
+/**
+ * HSUBPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_hsubps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_subps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc1->ar32[1]);
+    pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc1->ar32[3]);
+    pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc2->ar32[0], &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc2->ar32[2], &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * HSUBPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_hsubpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    pResult->MXCSR  = iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc1->ar64[1]);
+    pResult->MXCSR |= iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc2->ar64[0], &puSrc2->ar64[1]);
+}
+#endif
+
+
+/**
+ * SQRTPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_sqrtps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val)
+{
+    if (iemSseUnaryValIsNaNR32(pr32Res, pr32Val, &fMxcsr))
+        return fMxcsr;
+
+    RTFLOAT32U r32Src;
+    uint32_t fDe = iemSsePrepareValueR32(&r32Src, fMxcsr, pr32Val);
+    if (RTFLOAT32U_IS_ZERO(&r32Src))
+    {
+        *pr32Res = r32Src;
+        return fMxcsr;
+    }
+    else if (r32Src.s.fSign)
+    {
+        *pr32Res = g_ar32QNaN[1];
+        return fMxcsr | X86_MXCSR_IE;
+    }
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float32_t r32Result = f32_sqrt(iemFpSoftF32FromIprt(&r32Src), &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr | fDe);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sqrtps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_sqrtps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_sqrtps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_sqrtps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_sqrtps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * SQRTSS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_sqrtss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2))
+{
+    pResult->MXCSR = iemAImpl_sqrtps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, pr32Src2);
+    pResult->uResult.ar32[1] = puSrc1->ar32[1];
+    pResult->uResult.ar32[2] = puSrc1->ar32[2];
+    pResult->uResult.ar32[3] = puSrc1->ar32[3];
+}
+#endif
+
+
+/**
+ * SQRTPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_sqrtpd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val)
+{
+    if (iemSseUnaryValIsNaNR64(pr64Res, pr64Val, &fMxcsr))
+        return fMxcsr;
+
+    RTFLOAT64U r64Src;
+    uint32_t fDe = iemSsePrepareValueR64(&r64Src, fMxcsr, pr64Val);
+    if (RTFLOAT64U_IS_ZERO(&r64Src))
+    {
+        *pr64Res = r64Src;
+        return fMxcsr;
+    }
+    else if (r64Src.s.fSign)
+    {
+        *pr64Res = g_ar64QNaN[1];
+        return fMxcsr | X86_MXCSR_IE;
+    }
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float64_t r64Result = f64_sqrt(iemFpSoftF64FromIprt(&r64Src), &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr | fDe);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_sqrtpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_sqrtpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_sqrtpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc2->ar64[1]);
+}
+#endif
+
+
+/**
+ * SQRTSD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_sqrtsd_u128_r64,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT64U pr64Src2))
+{
+    pResult->MXCSR = iemAImpl_sqrtpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, pr64Src2);
+    pResult->uResult.ar64[1] = puSrc1->ar64[1];
+}
+#endif
+
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+/**
+ * RSQRTPS
+ */
+static uint32_t iemAImpl_rsqrt_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val)
+{
+    RT_NOREF(pr32Res);
+    RT_NOREF(pr32Val);
+    AssertReleaseFailed();
+    return fMxcsr;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_rsqrtps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_rsqrt_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_rsqrt_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_rsqrt_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_rsqrt_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc2->ar32[3]);
+}
+
+
+/**
+ * RSQRTSS
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_rsqrtss_u128_r32,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCRTFLOAT32U pr32Src2))
+{
+    pResult->MXCSR = iemAImpl_rsqrt_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, pr32Src2);
+    pResult->uResult.ar32[1] = puSrc1->ar32[1];
+    pResult->uResult.ar32[2] = puSrc1->ar32[2];
+    pResult->uResult.ar32[3] = puSrc1->ar32[3];
+}
+#endif
+
+
+/**
+ * ADDSUBPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_addsubps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_subps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc1->ar32[0], &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc1->ar32[1], &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_subps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, &puSrc1->ar32[2], &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_addps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, &puSrc1->ar32[3], &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * ADDSUBPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_addsubpd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_subpd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc1->ar64[0], &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_addpd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc1->ar64[1], &puSrc2->ar64[1]);
+}
+#endif
+
+
+/**
+ * CVTPD2PS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtpd2ps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Val1)
+{
+    RTFLOAT64U r64Src1;
+    fMxcsr |= iemSsePrepareValueR64(&r64Src1, fMxcsr, pr64Val1);
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float32_t r32Result = f64_to_f32(iemFpSoftF64FromIprt(&r64Src1), &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtpd2ps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_cvtpd2ps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_cvtpd2ps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, &puSrc2->ar64[1]);
+    pResult->uResult.au32[2] = 0;
+    pResult->uResult.au32[3] = 0;
+}
+#endif
+
+
+/**
+ * CVTPS2PD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtps2pd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Val1)
+{
+    RTFLOAT32U r32Src1;
+    fMxcsr |= iemSsePrepareValueR32(&r32Src1, fMxcsr, pr32Val1);
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float64_t r64Result = f32_to_f64(iemFpSoftF32FromIprt(&r32Src1), &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtps2pd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_cvtps2pd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_cvtps2pd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, &puSrc2->ar32[1]);
+}
+#endif
+
+
+/**
+ * CVTDQ2PS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtdq2ps_u128_worker(PRTFLOAT32U pr32Res, uint32_t fMxcsr, int32_t i32Val)
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float32_t r32Result = i32_to_f32(i32Val, &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Result, pr32Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtdq2ps_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_cvtdq2ps_u128_worker(&pResult->uResult.ar32[0], pFpuState->MXCSR, puSrc2->ai32[0]);
+    pResult->MXCSR |= iemAImpl_cvtdq2ps_u128_worker(&pResult->uResult.ar32[1], pFpuState->MXCSR, puSrc2->ai32[1]);
+    pResult->MXCSR |= iemAImpl_cvtdq2ps_u128_worker(&pResult->uResult.ar32[2], pFpuState->MXCSR, puSrc2->ai32[2]);
+    pResult->MXCSR |= iemAImpl_cvtdq2ps_u128_worker(&pResult->uResult.ar32[3], pFpuState->MXCSR, puSrc2->ai32[3]);
+}
+#endif
+
+
+/**
+ * CVTPS2DQ
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtps2dq_u128_worker(int32_t *pi32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Src)
+{
+    RTFLOAT32U r32Src;
+    iemSsePrepareValueR32(&r32Src, fMxcsr, pr32Src); /* De-normal seems to be ignored. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    *pi32Res = f32_to_i32(iemFpSoftF32FromIprt(&r32Src), SoftState.roundingMode, true /*exact*/, &SoftState);
+    return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtps2dq_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_cvtps2dq_u128_worker(&pResult->uResult.ai32[0], pFpuState->MXCSR, &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_cvtps2dq_u128_worker(&pResult->uResult.ai32[1], pFpuState->MXCSR, &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_cvtps2dq_u128_worker(&pResult->uResult.ai32[2], pFpuState->MXCSR, &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_cvtps2dq_u128_worker(&pResult->uResult.ai32[3], pFpuState->MXCSR, &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * CVTTPS2DQ
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvttps2dq_u128_worker(int32_t *pi32Res, uint32_t fMxcsr, PCRTFLOAT32U pr32Src)
+{
+    RTFLOAT32U r32Src;
+    iemSsePrepareValueR32(&r32Src, fMxcsr, pr32Src); /* De-normal seems to be ignored. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    SoftState.roundingMode = softfloat_round_minMag;
+    *pi32Res = f32_to_i32_r_minMag(iemFpSoftF32FromIprt(&r32Src), true /*exact*/, &SoftState);
+    return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvttps2dq_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_cvttps2dq_u128_worker(&pResult->uResult.ai32[0], pFpuState->MXCSR, &puSrc2->ar32[0]);
+    pResult->MXCSR |= iemAImpl_cvttps2dq_u128_worker(&pResult->uResult.ai32[1], pFpuState->MXCSR, &puSrc2->ar32[1]);
+    pResult->MXCSR |= iemAImpl_cvttps2dq_u128_worker(&pResult->uResult.ai32[2], pFpuState->MXCSR, &puSrc2->ar32[2]);
+    pResult->MXCSR |= iemAImpl_cvttps2dq_u128_worker(&pResult->uResult.ai32[3], pFpuState->MXCSR, &puSrc2->ar32[3]);
+}
+#endif
+
+
+/**
+ * CVTTPD2DQ
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvttpd2dq_u128_worker(int32_t *pi32Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Src)
+{
+    RTFLOAT64U r64Src;
+    iemSsePrepareValueR64(&r64Src, fMxcsr, pr64Src); /* De-normal seems to be ignored. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    SoftState.roundingMode = softfloat_round_minMag;
+    *pi32Res = f64_to_i32(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, true /*exact*/, &SoftState);
+    return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvttpd2dq_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_cvttpd2dq_u128_worker(&pResult->uResult.ai32[0], pFpuState->MXCSR, &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_cvttpd2dq_u128_worker(&pResult->uResult.ai32[1], pFpuState->MXCSR, &puSrc2->ar64[1]);
+    pResult->uResult.au64[1] = 0;
+}
+#endif
+
+
+/**
+ * CVTDQ2PD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtdq2pd_u128_worker(PRTFLOAT64U pr64Res, uint32_t fMxcsr, int32_t i32Val)
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float64_t r64Result = i32_to_f64(i32Val, &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Result, pr64Res, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtdq2pd_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_cvtdq2pd_u128_worker(&pResult->uResult.ar64[0], pFpuState->MXCSR, puSrc2->ai32[0]);
+    pResult->MXCSR |= iemAImpl_cvtdq2pd_u128_worker(&pResult->uResult.ar64[1], pFpuState->MXCSR, puSrc2->ai32[1]);
+}
+#endif
+
+
+/**
+ * CVTPD2DQ
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtpd2dq_u128_worker(int32_t *pi32Res, uint32_t fMxcsr, PCRTFLOAT64U pr64Src)
+{
+    RTFLOAT64U r64Src;
+    iemSsePrepareValueR64(&r64Src, fMxcsr, pr64Src); /* De-normal seems to be ignored. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    *pi32Res = f64_to_i32(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, true /*exact*/, &SoftState);
+    return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtpd2dq_u128,(PX86FXSTATE pFpuState, PIEMSSERESULT pResult, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    RT_NOREF(puSrc1);
+
+    pResult->MXCSR  = iemAImpl_cvtpd2dq_u128_worker(&pResult->uResult.ai32[0], pFpuState->MXCSR, &puSrc2->ar64[0]);
+    pResult->MXCSR |= iemAImpl_cvtpd2dq_u128_worker(&pResult->uResult.ai32[1], pFpuState->MXCSR, &puSrc2->ar64[1]);
+    pResult->uResult.au64[1] = 0;
+}
+#endif
+
+
+/**
+ * [V]SHUFPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_shufps_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    RTUINT128U const uSrc1 = *puDst;
+    RTUINT128U const uSrc2 = *puSrc;
+    ASMCompilerBarrier();
+    puDst->au32[0] = uSrc1.au32[bEvil & 0x3];
+    puDst->au32[1] = uSrc1.au32[(bEvil >> 2) & 0x3];
+    puDst->au32[2] = uSrc2.au32[(bEvil >> 4) & 0x3];
+    puDst->au32[3] = uSrc2.au32[(bEvil >> 6) & 0x3];
+}
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vshufps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil))
+{
+    RTUINT128U const uSrc1 = *puSrc1;
+    RTUINT128U const uSrc2 = *puSrc2;
+    ASMCompilerBarrier();
+    puDst->au32[0] = uSrc1.au32[bEvil & 0x3];
+    puDst->au32[1] = uSrc1.au32[(bEvil >> 2) & 0x3];
+    puDst->au32[2] = uSrc2.au32[(bEvil >> 4) & 0x3];
+    puDst->au32[3] = uSrc2.au32[(bEvil >> 6) & 0x3];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vshufps_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil))
+{
+    RTUINT256U const uSrc1 = *puSrc1;
+    RTUINT256U const uSrc2 = *puSrc2;
+    ASMCompilerBarrier();
+    puDst->au32[0] = uSrc1.au32[bEvil & 0x3];
+    puDst->au32[1] = uSrc1.au32[(bEvil >> 2) & 0x3];
+    puDst->au32[2] = uSrc2.au32[(bEvil >> 4) & 0x3];
+    puDst->au32[3] = uSrc2.au32[(bEvil >> 6) & 0x3];
+
+    puDst->au32[4] = uSrc1.au32[4 + (bEvil & 0x3)];
+    puDst->au32[5] = uSrc1.au32[4 + ((bEvil >> 2) & 0x3)];
+    puDst->au32[6] = uSrc2.au32[4 + ((bEvil >> 4) & 0x3)];
+    puDst->au32[7] = uSrc2.au32[4 + ((bEvil >> 6) & 0x3)];
+}
+
+
+/**
+ * [V]SHUFPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_shufpd_u128,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    RTUINT128U const uSrc1 = *puDst;
+    RTUINT128U const uSrc2 = *puSrc;
+    ASMCompilerBarrier();
+    puDst->au64[0] = (bEvil & RT_BIT(0)) ? uSrc1.au64[1] : uSrc1.au64[0];
+    puDst->au64[1] = (bEvil & RT_BIT(1)) ? uSrc2.au64[1] : uSrc2.au64[0];
+}
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vshufpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil))
+{
+    RTUINT128U const uSrc1 = *puSrc1;
+    RTUINT128U const uSrc2 = *puSrc2;
+    ASMCompilerBarrier();
+    puDst->au64[0] = (bEvil & RT_BIT(0)) ? uSrc1.au64[1] : uSrc1.au64[0];
+    puDst->au64[1] = (bEvil & RT_BIT(1)) ? uSrc2.au64[1] : uSrc2.au64[0];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vshufpd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil))
+{
+    RTUINT256U const uSrc1 = *puSrc1;
+    RTUINT256U const uSrc2 = *puSrc2;
+    ASMCompilerBarrier();
+    puDst->au64[0] = (bEvil & RT_BIT(0)) ? uSrc1.au64[1] : uSrc1.au64[0];
+    puDst->au64[1] = (bEvil & RT_BIT(1)) ? uSrc2.au64[1] : uSrc2.au64[0];
+    puDst->au64[2] = (bEvil & RT_BIT(2)) ? uSrc1.au64[3] : uSrc1.au64[2];
+    puDst->au64[3] = (bEvil & RT_BIT(3)) ? uSrc2.au64[3] : uSrc2.au64[2];
+}
+
+
+/*
+ * PHMINPOSUW / VPHMINPOSUW
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_phminposuw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    uint16_t u16Min = puSrc->au16[0];
+    uint8_t  idxMin = 0;
+
+    for (uint8_t i = 1; i < RT_ELEMENTS(puSrc->au16); i++)
+        if (puSrc->au16[i] < u16Min)
+        {
+            u16Min = puSrc->au16[i];
+            idxMin = i;
+        }
+
+    puDst->au64[0] = 0;
+    puDst->au64[1] = 0;
+    puDst->au16[0] = u16Min;
+    puDst->au16[1] = idxMin;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vphminposuw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    iemAImpl_phminposuw_u128_fallback(puDst, puSrc);
+}
+
+
+/*
+ * [V]PBLENDVB
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_pblendvb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, PCRTUINT128U puMask))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au8); i++)
+        if (puMask->au8[i] & RT_BIT(7))
+            puDst->au8[i] = puSrc->au8[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpblendvb_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, PCRTUINT128U puMask))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au8); i++)
+            puDst->au8[i] = puMask->au8[i] & RT_BIT(7) ? puSrc2->au8[i] : puSrc1->au8[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpblendvb_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, PCRTUINT256U puMask))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au8); i++)
+            puDst->au8[i] = puMask->au8[i] & RT_BIT(7) ? puSrc2->au8[i] : puSrc1->au8[i];
+}
+
+
+/*
+ * [V]BLENDVPS
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_blendvps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, PCRTUINT128U puMask))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++)
+        if (puMask->au32[i] & RT_BIT_32(31))
+            puDst->au32[i] = puSrc->au32[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vblendvps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, PCRTUINT128U puMask))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++)
+            puDst->au32[i] = (puMask->au32[i] & RT_BIT_32(31)) ? puSrc2->au32[i] : puSrc1->au32[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vblendvps_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, PCRTUINT256U puMask))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++)
+            puDst->au32[i] = (puMask->au32[i] & RT_BIT_32(31)) ? puSrc2->au32[i] : puSrc1->au32[i];
+}
+
+
+/*
+ * [V]BLENDVPD
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_blendvpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, PCRTUINT128U puMask))
+{
+    if (puMask->au64[0] & RT_BIT_64(63)) puDst->au64[0] = puSrc->au64[0];
+    if (puMask->au64[1] & RT_BIT_64(63)) puDst->au64[1] = puSrc->au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vblendvpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, PCRTUINT128U puMask))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au64); i++)
+            puDst->au64[i] = (puMask->au64[i] & RT_BIT_64(63)) ? puSrc2->au64[i] : puSrc1->au64[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vblendvpd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, PCRTUINT256U puMask))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au64); i++)
+            puDst->au64[i] = (puMask->au64[i] & RT_BIT_64(63)) ? puSrc2->au64[i] : puSrc1->au64[i];
+}
+
+
+/**
+ * [V]PALIGNR
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_palignr_u64_fallback,(uint64_t *pu64Dst, uint64_t u64Src2, uint8_t bEvil))
+{
+    uint64_t const u64Src1 = *pu64Dst;
+    ASMCompilerBarrier();
+
+    if (bEvil >= 16)
+        *pu64Dst = 0;
+    else if (bEvil >= 8)
+        *pu64Dst = u64Src1 >> ((bEvil - 8) * 8);
+    else
+    {
+        uint8_t cShift = bEvil * 8;
+        *pu64Dst =   ((u64Src1 & (RT_BIT_64(cShift) - 1)) << ((8 - bEvil) * 8))
+                   | (u64Src2 >> cShift);
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_palignr_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    RTUINT128U const uSrc1 = *puDst;
+    RTUINT128U const uSrc2 = *puSrc;
+    ASMCompilerBarrier();
+
+    puDst->au64[0] = 0;
+    puDst->au64[1] = 0;
+    if (bEvil >= 32)
+    { /* Everything stays 0. */ }
+    else if (bEvil >= 16)
+    {
+        bEvil -= 16;
+        for (uint8_t i = bEvil; i < RT_ELEMENTS(puDst->au8); i++)
+            puDst->au8[i - bEvil] = uSrc1.au8[i];
+    }
+    else
+    {
+        for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au8) - bEvil; i++)
+            puDst->au8[i] = uSrc2.au8[i + bEvil];
+        for (uint8_t i = 0; i < bEvil; i++)
+            puDst->au8[i + RT_ELEMENTS(puDst->au8) - bEvil] = uSrc1.au8[i];
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpalignr_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil))
+{
+    RTUINT128U const uSrc1 = *puSrc1; /* Might overlap with destination. */
+    RTUINT128U const uSrc2 = *puSrc2;
+    ASMCompilerBarrier();
+
+    puDst->au64[0] = 0;
+    puDst->au64[1] = 0;
+    if (bEvil >= 32)
+    { /* Everything stays 0. */ }
+    else if (bEvil >= 16)
+    {
+        bEvil -= 16;
+        for (uint8_t i = bEvil; i < RT_ELEMENTS(puDst->au8); i++)
+            puDst->au8[i - bEvil] = uSrc1.au8[i];
+    }
+    else
+    {
+        for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au8) - bEvil; i++)
+            puDst->au8[i] = uSrc2.au8[i + bEvil];
+        for (uint8_t i = 0; i < bEvil; i++)
+            puDst->au8[i + RT_ELEMENTS(puDst->au8) - bEvil] = uSrc1.au8[i];
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpalignr_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil))
+{
+    RTUINT256U const uSrc1 = *puSrc1; /* Might overlap with destination. */
+    RTUINT256U const uSrc2 = *puSrc2;
+    ASMCompilerBarrier();
+
+    iemAImpl_vpalignr_u128_fallback(&puDst->au128[0], &uSrc1.au128[0], &uSrc2.au128[0], bEvil);
+    iemAImpl_vpalignr_u128_fallback(&puDst->au128[1], &uSrc1.au128[1], &uSrc2.au128[1], bEvil);
+}
+
+
+/**
+ * [V]PBLENDW
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_pblendw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au16); i++)
+        if (bEvil & RT_BIT(i))
+            puDst->au16[i] = puSrc->au16[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpblendw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au16); i++)
+        if (bEvil & RT_BIT(i))
+            puDst->au16[i] = puSrc2->au16[i];
+        else
+            puDst->au16[i] = puSrc1->au16[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpblendw_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < 8; i++)
+        if (bEvil & RT_BIT(i))
+        {
+            puDst->au16[    i] = puSrc2->au16[    i];
+            puDst->au16[8 + i] = puSrc2->au16[8 + i];
+        }
+        else
+        {
+            puDst->au16[    i] = puSrc1->au16[    i];
+            puDst->au16[8 + i] = puSrc1->au16[8 + i];
+        }
+}
+
+
+/**
+ * [V]BLENDPS
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_blendps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++)
+        if (bEvil & RT_BIT(i))
+            puDst->au32[i] = puSrc->au32[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vblendps_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++)
+        if (bEvil & RT_BIT(i))
+            puDst->au32[i] = puSrc2->au32[i];
+        else
+            puDst->au32[i] = puSrc1->au32[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vblendps_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au32); i++)
+        if (bEvil & RT_BIT(i))
+            puDst->au32[i] = puSrc2->au32[i];
+        else
+            puDst->au32[i] = puSrc1->au32[i];
+}
+
+
+/**
+ * [V]BLENDPD
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_blendpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au64); i++)
+        if (bEvil & RT_BIT(i))
+            puDst->au64[i] = puSrc->au64[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vblendpd_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au64); i++)
+        if (bEvil & RT_BIT(i))
+            puDst->au64[i] = puSrc2->au64[i];
+        else
+            puDst->au64[i] = puSrc1->au64[i];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vblendpd_u256_fallback,(PRTUINT256U puDst, PCRTUINT256U puSrc1, PCRTUINT256U puSrc2, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->au64); i++)
+        if (bEvil & RT_BIT(i))
+            puDst->au64[i] = puSrc2->au64[i];
+        else
+            puDst->au64[i] = puSrc1->au64[i];
+}
+
+
+/**
+ * AES tables and helper routines. Tables from Intel AES-NI whitepaper.
+ */
+
+static uint8_t iemAImpl_aes_sbox[] = {
+    0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76,
+    0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0,
+    0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
+    0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75,
+    0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84,
+    0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
+    0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8,
+    0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2,
+    0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
+    0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb,
+    0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79,
+    0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
+    0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a,
+    0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e,
+    0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
+    0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16
+};
+
+/* The InvS-Box lookup table. */
+static uint8_t iemAImpl_aes_inv_sbox[] = {
+    0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
+    0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
+    0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
+    0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
+    0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
+    0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
+    0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
+    0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
+    0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
+    0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
+    0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
+    0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
+    0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
+    0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
+    0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
+    0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
+};
+
+/* The ShiftRows lookup table. */
+static uint8_t iemAImpl_aes_shift_rows_tbl[] = {
+     0,  5, 10, 15,  4,  9, 14,  3,  8, 13,  2,  7, 12,  1,  6, 11
+};
+
+/* The InvShiftRows lookup table. */
+static uint8_t iemAImpl_aes_inv_shift_rows_tbl[] = {
+     0, 13, 10,  7,  4,  1, 14, 11,  8,  5,  2, 15, 12,  9,  6,  3
+};
+
+static inline RTUINT128U iemAImpl_aes_sub_bytes(PCRTUINT128U puSrc, uint8_t abSubst[256])
+{
+    RTUINT128U  uVal;
+    int         i;
+
+    for (i = 0; i < 16; ++i)
+        uVal.au8[i] = abSubst[puSrc->au8[i]];
+
+    return uVal;
+}
+
+static inline uint8_t iemAImpl_aes_xtime(uint8_t u)
+{
+    return (u << 1) ^ (((u >> 7) & 1) * 27);
+}
+
+static RTUINT128U iemAImpl_aes_mix_col(PCRTUINT128U puSrc)
+{
+    RTUINT128U  uVal;
+    int         i;
+    uint8_t     tmp;
+
+    for (i = 0; i < 16; i += 4) {
+        tmp = puSrc->au8[i+0] ^ puSrc->au8[i+1] ^ puSrc->au8[i+2] ^ puSrc->au8[i+3];
+        uVal.au8[i+0] = puSrc->au8[i+0] ^ tmp ^ iemAImpl_aes_xtime(puSrc->au8[i+0] ^ puSrc->au8[i+1]);
+        uVal.au8[i+1] = puSrc->au8[i+1] ^ tmp ^ iemAImpl_aes_xtime(puSrc->au8[i+1] ^ puSrc->au8[i+2]);
+        uVal.au8[i+2] = puSrc->au8[i+2] ^ tmp ^ iemAImpl_aes_xtime(puSrc->au8[i+2] ^ puSrc->au8[i+3]);
+        uVal.au8[i+3] = puSrc->au8[i+3] ^ tmp ^ iemAImpl_aes_xtime(puSrc->au8[i+3] ^ puSrc->au8[i+0]);
+    }
+
+    return uVal;
+}
+
+static inline RTUINT128U iemAImpl_aes_shift_rows(PCRTUINT128U puSrc, uint8_t abShift[16])
+{
+    RTUINT128U  uVal;
+    int         i;
+
+    for (i = 0; i < 16; ++i)
+        uVal.au8[i] = puSrc->au8[abShift[i]];
+
+    return uVal;
+}
+
+static uint8_t iemAImpl_aes_clmul(uint8_t a, uint8_t b)
+{
+    uint8_t     val;
+
+    val  = ((b >> 0) & 1) * a;
+    val ^= ((b >> 1) & 1) * iemAImpl_aes_xtime(a);
+    val ^= ((b >> 2) & 1) * iemAImpl_aes_xtime(iemAImpl_aes_xtime(a));
+    val ^= ((b >> 3) & 1) * iemAImpl_aes_xtime(iemAImpl_aes_xtime(iemAImpl_aes_xtime(a)));
+    val ^= ((b >> 4) & 1) * iemAImpl_aes_xtime(iemAImpl_aes_xtime(iemAImpl_aes_xtime(iemAImpl_aes_xtime(a))));
+
+    return val;
+}
+
+static RTUINT128U iemAImpl_aes_inv_mix_col(PCRTUINT128U puSrc)
+{
+    RTUINT128U  uVal;
+    int         i;
+
+    for (i = 0; i < 16; i += 4) {
+        uVal.au8[i+0] = iemAImpl_aes_clmul(puSrc->au8[i+0], 0x0e) ^ iemAImpl_aes_clmul(puSrc->au8[i+1], 0x0b)^ iemAImpl_aes_clmul(puSrc->au8[i+2], 0x0d) ^ iemAImpl_aes_clmul(puSrc->au8[i+3], 0x09);
+        uVal.au8[i+1] = iemAImpl_aes_clmul(puSrc->au8[i+0], 0x09) ^ iemAImpl_aes_clmul(puSrc->au8[i+1], 0x0e)^ iemAImpl_aes_clmul(puSrc->au8[i+2], 0x0b) ^ iemAImpl_aes_clmul(puSrc->au8[i+3], 0x0d);
+        uVal.au8[i+2] = iemAImpl_aes_clmul(puSrc->au8[i+0], 0x0d) ^ iemAImpl_aes_clmul(puSrc->au8[i+1], 0x09)^ iemAImpl_aes_clmul(puSrc->au8[i+2], 0x0e) ^ iemAImpl_aes_clmul(puSrc->au8[i+3], 0x0b);
+        uVal.au8[i+3] = iemAImpl_aes_clmul(puSrc->au8[i+0], 0x0b) ^ iemAImpl_aes_clmul(puSrc->au8[i+1], 0x0d)^ iemAImpl_aes_clmul(puSrc->au8[i+2], 0x09) ^ iemAImpl_aes_clmul(puSrc->au8[i+3], 0x0e);
+    }
+
+    return uVal;
+}
+
+static inline uint32_t iemAImpl_aes_sub_word(uint32_t w)
+{
+    RTUINT32U    uTmp;
+
+    uTmp.au32[0] = w;
+    uTmp.au8[0] = iemAImpl_aes_sbox[uTmp.au8[0]];
+    uTmp.au8[1] = iemAImpl_aes_sbox[uTmp.au8[1]];
+    uTmp.au8[2] = iemAImpl_aes_sbox[uTmp.au8[2]];
+    uTmp.au8[3] = iemAImpl_aes_sbox[uTmp.au8[3]];
+
+    return uTmp.au32[0];
+}
+
+static inline uint32_t iemAImpl_aes_rot_word(uint32_t w)
+{
+    return (w << 24) | (w >> 8);
+}
+
+/**
+ * [V]AESKEYGENASSIST
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_aeskeygenassist_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bImm))
+{
+    RTUINT128U  uTmp;
+    uint32_t    uRCon = bImm;   /* Round constant. */
+
+    uTmp.au32[0] = iemAImpl_aes_sub_word(puSrc->au32[1]);   /* puSrc = KeyGen. */
+    uTmp.au32[1] = iemAImpl_aes_rot_word(iemAImpl_aes_sub_word(puSrc->au32[1])) ^ uRCon;
+    uTmp.au32[2] = iemAImpl_aes_sub_word(puSrc->au32[3]);
+    uTmp.au32[3] = iemAImpl_aes_rot_word(iemAImpl_aes_sub_word(puSrc->au32[3])) ^ uRCon;
+
+    *puDst = uTmp;
+}
+
+
+/**
+ * [V]AESIMC
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_aesimc_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    *puDst = iemAImpl_aes_inv_mix_col(puSrc);   /* Src = Key. */
+}
+
+
+/**
+ * [V]AESENC
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_aesenc_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U  uTmp;
+
+    uTmp = iemAImpl_aes_shift_rows(puDst, iemAImpl_aes_shift_rows_tbl); /* Dst = state. */
+    uTmp = iemAImpl_aes_sub_bytes(&uTmp, iemAImpl_aes_sbox);
+    uTmp = iemAImpl_aes_mix_col(&uTmp);
+    uTmp.au64[0] ^= puSrc->au64[0];  /* Src = Round Key. */
+    uTmp.au64[1] ^= puSrc->au64[1];
+
+    *puDst = uTmp;
+}
+
+
+/**
+ * [V]AESENCLAST
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_aesenclast_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U  uTmp;
+
+    uTmp = iemAImpl_aes_shift_rows(puDst, iemAImpl_aes_shift_rows_tbl); /* Dst = state. */
+    uTmp = iemAImpl_aes_sub_bytes(&uTmp, iemAImpl_aes_sbox);
+    uTmp.au64[0] ^= puSrc->au64[0];  /* Src = Round Key. */
+    uTmp.au64[1] ^= puSrc->au64[1];
+
+    *puDst = uTmp;
+}
+
+
+/**
+ * [V]AESDEC
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_aesdec_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U  uTmp;
+
+    uTmp = iemAImpl_aes_shift_rows(puDst, iemAImpl_aes_inv_shift_rows_tbl); /* Dst = state. */
+    uTmp = iemAImpl_aes_sub_bytes(&uTmp, iemAImpl_aes_inv_sbox);
+    uTmp = iemAImpl_aes_inv_mix_col(&uTmp);
+    uTmp.au64[0] ^= puSrc->au64[0];  /* Src = Round Key. */
+    uTmp.au64[1] ^= puSrc->au64[1];
+
+    *puDst = uTmp;
+}
+
+
+/**
+ * [V]AESDECLAST
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_aesdeclast_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc))
+{
+    RTUINT128U  uTmp;
+
+    uTmp = iemAImpl_aes_shift_rows(puDst, iemAImpl_aes_inv_shift_rows_tbl); /* Dst = state. */
+    uTmp = iemAImpl_aes_sub_bytes(&uTmp, iemAImpl_aes_inv_sbox);
+    uTmp.au64[0] ^= puSrc->au64[0];  /* Src = Round Key. */
+    uTmp.au64[1] ^= puSrc->au64[1];
+
+    *puDst = uTmp;
+}
+
+
+/**
+ * [V]PCMPISTRI
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpistri_u128_fallback,(uint32_t *pu32Ecx, uint32_t *pEFlags, PCIEMPCMPISTRISRC pSrc, uint8_t bEvil))
+{
+    RT_NOREF(pu32Ecx, pEFlags, pSrc, bEvil);
+    AssertReleaseFailed();
+}
+
+
+/*
+ * [V]PCLMULQDQ
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_pclmulqdq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    iemAImpl_vpclmulqdq_u128_fallback(puDst, puDst, puSrc, bEvil);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpclmulqdq_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc1, PCRTUINT128U puSrc2, uint8_t bEvil))
+{
+    uint64_t uSrc1 = puSrc1->au64[bEvil & 0x1];
+    uint64_t uSrc2 = puSrc2->au64[(bEvil >> 4) & 0x1];
+
+    puDst->au64[0] = 0;
+    puDst->au64[1] = 0;
+
+    /*
+     * See https://en.wikipedia.org/wiki/Carry-less_product#Example (as of 2022-09-08) for the algorithm.
+     * Do the first round outside the loop to avoid ASAN complaining about shift exponent being too large (64)
+     * and squeeze out some optimizations.
+     */
+    if (uSrc1 & 0x1)
+        puDst->au64[0] = uSrc2;
+
+    uSrc1 >>= 1;
+
+    uint8_t iDigit = 1;
+    while (uSrc1)
+    {
+        if (uSrc1 & 0x1)
+        {
+            puDst->au64[0] ^= (uSrc2 << iDigit);
+            puDst->au64[1] ^= uSrc2 >> (64 - iDigit);
+        }
+
+        uSrc1 >>= 1;
+        iDigit++;
+    }
+}
+
+
+/**
+ * [V]PINSRW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_pinsrw_u64,(uint64_t *pu64Dst, uint16_t u16Src, uint8_t bEvil))
+{
+    uint8_t cShift = (bEvil & 0x3) * 16;
+    *pu64Dst = (*pu64Dst & ~(UINT64_C(0xffff) << cShift)) | ((uint64_t)u16Src << cShift);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pinsrw_u128,(PRTUINT128U puDst, uint16_t u16Src, uint8_t bEvil))
+{
+    puDst->au16[bEvil & 0x7] = u16Src;
+}
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpinsrw_u128_fallback,(PRTUINT128U puDst, PCRTUINT128U puSrc, uint16_t u16Src, uint8_t bEvil))
+{
+    *puDst = *puSrc;
+    puDst->au16[bEvil & 0x7] = u16Src;
+}
+
+
+/**
+ * [V]PEXTRW
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_pextrw_u64,(uint16_t *pu16Dst, uint64_t u64Src, uint8_t bEvil))
+{
+    *pu16Dst = (uint16_t)(u64Src >> ((bEvil & 0x3) * 16));
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_pextrw_u128,(uint16_t *pu16Dst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    *pu16Dst = puSrc->au16[bEvil & 0x7];
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vpextrw_u128_fallback,(uint16_t *pu16Dst, PCRTUINT128U puSrc, uint8_t bEvil))
+{
+    *pu16Dst = puSrc->au16[bEvil & 0x7];
+}
+
+
+/**
+ * [V]MOVMSKPS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_movmskps_u128,(uint8_t *pu8Dst, PCRTUINT128U puSrc))
+{
+    *pu8Dst  =  puSrc->au32[0] >> 31;
+    *pu8Dst |= (puSrc->au32[1] >> 31) << 1;
+    *pu8Dst |= (puSrc->au32[2] >> 31) << 2;
+    *pu8Dst |= (puSrc->au32[3] >> 31) << 3;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovmskps_u128_fallback,(uint8_t *pu8Dst, PCRTUINT128U puSrc))
+{
+    *pu8Dst  =  puSrc->au32[0] >> 31;
+    *pu8Dst |= (puSrc->au32[1] >> 31) << 1;
+    *pu8Dst |= (puSrc->au32[2] >> 31) << 2;
+    *pu8Dst |= (puSrc->au32[3] >> 31) << 3;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovmskps_u256_fallback,(uint8_t *pu8Dst, PCRTUINT256U puSrc))
+{
+    *pu8Dst  =  puSrc->au32[0] >> 31;
+    *pu8Dst |= (puSrc->au32[1] >> 31) << 1;
+    *pu8Dst |= (puSrc->au32[2] >> 31) << 2;
+    *pu8Dst |= (puSrc->au32[3] >> 31) << 3;
+    *pu8Dst |= (puSrc->au32[4] >> 31) << 4;
+    *pu8Dst |= (puSrc->au32[5] >> 31) << 5;
+    *pu8Dst |= (puSrc->au32[6] >> 31) << 6;
+    *pu8Dst |= (puSrc->au32[7] >> 31) << 7;
+}
+
+
+/**
+ * [V]MOVMSKPD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_movmskpd_u128,(uint8_t *pu8Dst, PCRTUINT128U puSrc))
+{
+    *pu8Dst  =  puSrc->au64[0] >> 63;
+    *pu8Dst |= (puSrc->au64[1] >> 63) << 1;
+}
+
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovmskpd_u128_fallback,(uint8_t *pu8Dst, PCRTUINT128U puSrc))
+{
+    *pu8Dst  =  puSrc->au64[0] >> 63;
+    *pu8Dst |= (puSrc->au64[1] >> 63) << 1;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vmovmskpd_u256_fallback,(uint8_t *pu8Dst, PCRTUINT256U puSrc))
+{
+    *pu8Dst  =  puSrc->au64[0] >> 63;
+    *pu8Dst |= (puSrc->au64[1] >> 63) << 1;
+    *pu8Dst |= (puSrc->au64[2] >> 63) << 2;
+    *pu8Dst |= (puSrc->au64[3] >> 63) << 3;
+}
+
+
+/**
+ * CVTTSD2SI
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvttsd2si_i32_r64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int32_t *pi32Dst, const uint64_t *pu64Src))
+{
+    RTFLOAT64U r64Src;
+
+    r64Src.u = *pu64Src;
+    iemSsePrepareValueR64(&r64Src, pFpuState->MXCSR, &r64Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    *pi32Dst = f64_to_i32_r_minMag(iemFpSoftF64FromIprt(&r64Src), true /*exact*/, &SoftState);
+    *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvttsd2si_i64_r64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int64_t *pi64Dst, const uint64_t *pu64Src))
+{
+    RTFLOAT64U r64Src;
+
+    r64Src.u = *pu64Src;
+    iemSsePrepareValueR64(&r64Src, pFpuState->MXCSR, &r64Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    *pi64Dst = f64_to_i64_r_minMag(iemFpSoftF64FromIprt(&r64Src), true /*exact*/, &SoftState);
+    *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+#endif
+
+
+/**
+ * CVTSD2SI
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsd2si_i32_r64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int32_t *pi32Dst, const uint64_t *pu64Src))
+{
+    RTFLOAT64U r64Src;
+
+    r64Src.u = *pu64Src;
+    iemSsePrepareValueR64(&r64Src, pFpuState->MXCSR, &r64Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    *pi32Dst = f64_to_i32(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, true /*exact*/, &SoftState);
+    *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsd2si_i64_r64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int64_t *pi64Dst, const uint64_t *pu64Src))
+{
+    RTFLOAT64U r64Src;
+
+    r64Src.u = *pu64Src;
+    iemSsePrepareValueR64(&r64Src, pFpuState->MXCSR, &r64Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    *pi64Dst = f64_to_i64(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, true /*exact*/, &SoftState);
+    *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+#endif
+
+
+/**
+ * CVTTSS2SI
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvttss2si_i32_r32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int32_t *pi32Dst, const uint32_t *pu32Src))
+{
+    RTFLOAT32U r32Src;
+
+    r32Src.u = *pu32Src;
+    iemSsePrepareValueR32(&r32Src, pFpuState->MXCSR, &r32Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    *pi32Dst = f32_to_i32_r_minMag(iemFpSoftF32FromIprt(&r32Src), true /*exact*/, &SoftState);
+    *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvttss2si_i64_r32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int64_t *pi64Dst, const uint32_t *pu32Src))
+{
+    RTFLOAT32U r32Src;
+
+    r32Src.u = *pu32Src;
+    iemSsePrepareValueR32(&r32Src, pFpuState->MXCSR, &r32Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    *pi64Dst = f32_to_i64_r_minMag(iemFpSoftF32FromIprt(&r32Src), true /*exact*/, &SoftState);
+    *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+#endif
+
+
+/**
+ * CVTSS2SI
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtss2si_i32_r32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int32_t *pi32Dst, const uint32_t *pu32Src))
+{
+    RTFLOAT32U r32Src;
+
+    r32Src.u = *pu32Src;
+    iemSsePrepareValueR32(&r32Src, pFpuState->MXCSR, &r32Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    *pi32Dst = f32_to_i32(iemFpSoftF32FromIprt(&r32Src), SoftState.roundingMode, true /*exact*/, &SoftState);
+    *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtss2si_i64_r32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, int64_t *pi64Dst, const uint32_t *pu32Src))
+{
+    RTFLOAT32U r32Src;
+
+    r32Src.u = *pu32Src;
+    iemSsePrepareValueR32(&r32Src, pFpuState->MXCSR, &r32Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    *pi64Dst = f32_to_i64(iemFpSoftF32FromIprt(&r32Src), SoftState.roundingMode, true /*exact*/, &SoftState);
+    *pfMxcsr = pFpuState->MXCSR | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+#endif
+
+
+/**
+ * CVTSI2SD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsi2sd_r64_i32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, PRTFLOAT64U pr64Dst, const int32_t *pi32Src))
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    float64_t r64Res = i32_to_f64(*pi32Src, &SoftState);
+    *pfMxcsr = iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Res, pr64Dst, pFpuState->MXCSR);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsi2sd_r64_i64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, PRTFLOAT64U pr64Dst, const int64_t *pi64Src))
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    float64_t r64Res = i64_to_f64(*pi64Src, &SoftState);
+    *pfMxcsr = iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Res, pr64Dst, pFpuState->MXCSR);
+}
+#endif
+
+
+/**
+ * CVTSI2SS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsi2ss_r32_i32,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, PRTFLOAT32U pr32Dst, const int32_t *pi32Src))
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    float32_t r32Res = i32_to_f32(*pi32Src, &SoftState);
+    *pfMxcsr = iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Res, pr32Dst, pFpuState->MXCSR);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtsi2ss_r32_i64,(PCX86FXSTATE pFpuState, uint32_t *pfMxcsr, PRTFLOAT32U pr32Dst, const int64_t *pi64Src))
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(pFpuState->MXCSR);
+    float32_t r32Res = i64_to_f32(*pi64Src, &SoftState);
+    *pfMxcsr = iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Res, pr32Dst, pFpuState->MXCSR);
+}
+#endif
+
+
+/**
+ * [V]UCOMISS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_ucomiss_u128,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    uint32_t fEFlagsNew = *pfEFlags & ~X86_EFL_STATUS_BITS;
+
+    if (RTFLOAT32U_IS_SIGNALLING_NAN(&puSrc1->ar32[0]) || RTFLOAT32U_IS_SIGNALLING_NAN(&puSrc2->ar32[0]))
+    {
+        *pfMxcsr   |= X86_MXCSR_IE;
+        fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */
+    }
+    else if (RTFLOAT32U_IS_QUIET_NAN(&puSrc1->ar32[0]) || RTFLOAT32U_IS_QUIET_NAN(&puSrc2->ar32[0]))
+    {
+        /* ucomiss doesn't raise \#IE for quiet NaNs. */
+        fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */
+    }
+    else
+    {
+        softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr);
+
+        RTFLOAT32U r32Src1, r32Src2;
+        uint32_t fDe  = iemSsePrepareValueR32(&r32Src1, *pfMxcsr, &puSrc1->ar32[0]);
+                 fDe |= iemSsePrepareValueR32(&r32Src2, *pfMxcsr, &puSrc2->ar32[0]);
+
+        float32_t f32Src1 = iemFpSoftF32FromIprt(&r32Src1);
+        float32_t f32Src2 = iemFpSoftF32FromIprt(&r32Src2);
+        if (f32_eq(f32Src1, f32Src2, &SoftState))
+            fEFlagsNew |= X86_EFL_ZF; /* EQUAL        100 */
+        else if (f32_lt(f32Src1, f32Src2, &SoftState))
+            fEFlagsNew |= X86_EFL_CF; /* LESS_THAN    001 */
+        /* else:                         GREATER_THAN 000 */
+
+        *pfMxcsr |= fDe;
+    }
+
+    *pfEFlags = fEFlagsNew;
+}
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vucomiss_u128_fallback,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    iemAImpl_ucomiss_u128(pfMxcsr, pfEFlags, puSrc1, puSrc2);
+}
+
+
+/**
+ * [V]UCOMISD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_ucomisd_u128,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    uint32_t fEFlagsNew = *pfEFlags & ~X86_EFL_STATUS_BITS;
+
+    if (RTFLOAT64U_IS_SIGNALLING_NAN(&puSrc1->ar64[0]) || RTFLOAT64U_IS_SIGNALLING_NAN(&puSrc2->ar64[0]))
+    {
+        *pfMxcsr   |= X86_MXCSR_IE;
+        fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */
+    }
+    else if (RTFLOAT64U_IS_QUIET_NAN(&puSrc1->ar64[0]) || RTFLOAT64U_IS_QUIET_NAN(&puSrc2->ar64[0]))
+    {
+        /* ucomiss doesn't raise \#IE for quiet NaNs. */
+        fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */
+    }
+    else
+    {
+        softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr);
+
+        RTFLOAT64U r64Src1, r64Src2;
+        uint32_t fDe  = iemSsePrepareValueR64(&r64Src1, *pfMxcsr, &puSrc1->ar64[0]);
+                 fDe |= iemSsePrepareValueR64(&r64Src2, *pfMxcsr, &puSrc2->ar64[0]);
+
+        float64_t f64Src1 = iemFpSoftF64FromIprt(&r64Src1);
+        float64_t f64Src2 = iemFpSoftF64FromIprt(&r64Src2);
+        if (f64_eq(f64Src1, f64Src2, &SoftState))
+            fEFlagsNew |= X86_EFL_ZF; /* EQUAL        100 */
+        else if (f64_lt(f64Src1, f64Src2, &SoftState))
+            fEFlagsNew |= X86_EFL_CF; /* LESS_THAN    001 */
+        /* else:                         GREATER_THAN 000 */
+
+        *pfMxcsr |= fDe;
+    }
+
+    *pfEFlags = fEFlagsNew;
+}
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vucomisd_u128_fallback,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    iemAImpl_ucomisd_u128(pfMxcsr, pfEFlags, puSrc1, puSrc2);
+}
+
+
+/**
+ * [V]COMISS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_comiss_u128,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    uint32_t fEFlagsNew = *pfEFlags & ~X86_EFL_STATUS_BITS;
+
+    if (   RTFLOAT32U_IS_SIGNALLING_NAN(&puSrc1->ar32[0]) || RTFLOAT32U_IS_SIGNALLING_NAN(&puSrc2->ar32[0])
+        || RTFLOAT32U_IS_QUIET_NAN(&puSrc1->ar32[0]) || RTFLOAT32U_IS_QUIET_NAN(&puSrc2->ar32[0]))
+    {
+        *pfMxcsr   |= X86_MXCSR_IE;
+        fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */
+    }
+    else
+    {
+        softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr);
+
+        RTFLOAT32U r32Src1, r32Src2;
+        uint32_t fDe  = iemSsePrepareValueR32(&r32Src1, *pfMxcsr, &puSrc1->ar32[0]);
+                 fDe |= iemSsePrepareValueR32(&r32Src2, *pfMxcsr, &puSrc2->ar32[0]);
+
+        float32_t f32Src1 = iemFpSoftF32FromIprt(&r32Src1);
+        float32_t f32Src2 = iemFpSoftF32FromIprt(&r32Src2);
+        if (f32_eq(f32Src1, f32Src2, &SoftState))
+            fEFlagsNew |= X86_EFL_ZF; /* EQUAL        100 */
+        else if (f32_lt(f32Src1, f32Src2, &SoftState))
+            fEFlagsNew |= X86_EFL_CF; /* LESS_THAN    001 */
+        /* else:                         GREATER_THAN 000 */
+
+        *pfMxcsr |= fDe;
+    }
+
+    *pfEFlags = fEFlagsNew;
+}
+#endif
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vcomiss_u128_fallback,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    iemAImpl_comiss_u128(pfMxcsr, pfEFlags, puSrc1, puSrc2);
+}
+
+
+/**
+ * [V]COMISD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_comisd_u128,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    uint32_t fEFlagsNew = *pfEFlags & ~X86_EFL_STATUS_BITS;
+
+    if (   RTFLOAT64U_IS_SIGNALLING_NAN(&puSrc1->ar64[0]) || RTFLOAT64U_IS_SIGNALLING_NAN(&puSrc2->ar64[0])
+        || RTFLOAT64U_IS_QUIET_NAN(&puSrc1->ar64[0]) || RTFLOAT64U_IS_QUIET_NAN(&puSrc2->ar64[0]))
+    {
+        *pfMxcsr   |= X86_MXCSR_IE;
+        fEFlagsNew |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF; /* UNORDERED 111 */
+    }
+    else
+    {
+        softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr);
+
+        RTFLOAT64U r64Src1, r64Src2;
+        uint32_t fDe  = iemSsePrepareValueR64(&r64Src1, *pfMxcsr, &puSrc1->ar64[0]);
+                 fDe |= iemSsePrepareValueR64(&r64Src2, *pfMxcsr, &puSrc2->ar64[0]);
+
+        float64_t f64Src1 = iemFpSoftF64FromIprt(&r64Src1);
+        float64_t f64Src2 = iemFpSoftF64FromIprt(&r64Src2);
+        if (f64_eq(f64Src1, f64Src2, &SoftState))
+            fEFlagsNew |= X86_EFL_ZF; /* EQUAL        100 */
+        else if (f64_lt(f64Src1, f64Src2, &SoftState))
+            fEFlagsNew |= X86_EFL_CF; /* LESS_THAN    001 */
+        /* else:                         GREATER_THAN 000 */
+
+        *pfMxcsr |= fDe;
+    }
+
+    *pfEFlags = fEFlagsNew;
+}
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_vcomisd_u128_fallback,(uint32_t *pfMxcsr, uint32_t *pfEFlags, PCX86XMMREG puSrc1, PCX86XMMREG puSrc2))
+{
+    iemAImpl_comisd_u128(pfMxcsr, pfEFlags, puSrc1, puSrc2);
+}
+
+
+/**
+ * CMPPS / CMPPD / CMPSS / CMPSD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+/**
+ * A compare truth table entry.
+ */
+typedef struct CMPTRUTHTBLENTRY
+{
+    /** Flag whether the \#IA is signalled when one of the source oeprans is a QNaN */
+    bool                fSignalsOnQNan;
+    /** The boolean result when the input operands are unordered. */
+    bool                fUnordered;
+    /** The boolean result when A = B. */
+    bool                fEqual;
+    /** The boolean result when A < B. */
+    bool                fLowerThan;
+    /** The boolean result when A > B. */
+    bool                fGreaterThan;
+} CMPTRUTHTBLENTRY;
+/** Pointer to a const truth table entry. */
+typedef const CMPTRUTHTBLENTRY *PCCMPTRUTHTBLENTRY;
+
+
+/** The compare truth table (indexed by immediate). */
+static const CMPTRUTHTBLENTRY g_aCmpTbl[] =
+{
+                            /* fSignalsOnQNan   fUnordered      fEqual      fLowerThan      fGreaterThan */
+    /* 00H (EQ_OQ)   */     {  false,           false,          true,       false,          false           },
+    /* 01H (LT_OS)   */     {  true,            false,          false,      true,           false           },
+    /* 02H (LE_OS)   */     {  true,            false,          true,       true,           false           },
+    /* 03H (UNORD_Q) */     {  false,           true,           false,      false,          false           },
+    /* 04H (NEQ_UQ)  */     {  false,           true,           false,      true,           true            },
+    /* 05H (NLT_US)  */     {  true,            true,           true,       false,          true            },
+    /* 06H (NLE_US)  */     {  true,            true,           false,      false,          true            },
+    /* 07H (ORQ_Q)   */     {  false,           false,          true,       true,           true            },
+    /** @todo AVX variants. */
+};
+
+
+static bool iemAImpl_cmp_worker_r32(uint32_t *pfMxcsr, PCRTFLOAT32U pr32Src1, PCRTFLOAT32U pr32Src2, uint8_t bEvil)
+{
+    bool fRes;
+    AssertRelease(bEvil < RT_ELEMENTS(g_aCmpTbl));
+
+    if (RTFLOAT32U_IS_SIGNALLING_NAN(pr32Src1) || RTFLOAT32U_IS_SIGNALLING_NAN(pr32Src2))
+    {
+        *pfMxcsr |= X86_MXCSR_IE;
+        fRes = g_aCmpTbl[bEvil].fUnordered;
+    }
+    else if (RTFLOAT32U_IS_QUIET_NAN(pr32Src1) || RTFLOAT32U_IS_QUIET_NAN(pr32Src2))
+    {
+        if (g_aCmpTbl[bEvil].fSignalsOnQNan)
+            *pfMxcsr |= X86_MXCSR_IE;
+        fRes = g_aCmpTbl[bEvil].fUnordered;
+    }
+    else
+    {
+        softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr);
+
+        RTFLOAT32U r32Src1, r32Src2;
+        uint32_t fDe  = iemSsePrepareValueR32(&r32Src1, *pfMxcsr, pr32Src1);
+                 fDe |= iemSsePrepareValueR32(&r32Src2, *pfMxcsr, pr32Src2);
+
+        *pfMxcsr |= fDe;
+        float32_t f32Src1 = iemFpSoftF32FromIprt(&r32Src1);
+        float32_t f32Src2 = iemFpSoftF32FromIprt(&r32Src2);
+        if (f32_eq(f32Src1, f32Src2, &SoftState))
+            fRes = g_aCmpTbl[bEvil].fEqual;
+        else if (f32_lt(f32Src1, f32Src2, &SoftState))
+            fRes = g_aCmpTbl[bEvil].fLowerThan;
+        else
+            fRes = g_aCmpTbl[bEvil].fGreaterThan;
+    }
+
+    return fRes;
+}
+
+
+static bool iemAImpl_cmp_worker_r64(uint32_t *pfMxcsr, PCRTFLOAT64U pr64Src1, PCRTFLOAT64U pr64Src2, uint8_t bEvil)
+{
+    bool fRes;
+    AssertRelease(bEvil < RT_ELEMENTS(g_aCmpTbl));
+
+    if (RTFLOAT64U_IS_SIGNALLING_NAN(pr64Src1) || RTFLOAT64U_IS_SIGNALLING_NAN(pr64Src2))
+    {
+        *pfMxcsr |= X86_MXCSR_IE;
+        fRes = g_aCmpTbl[bEvil].fUnordered;
+    }
+    else if (RTFLOAT64U_IS_QUIET_NAN(pr64Src1) || RTFLOAT64U_IS_QUIET_NAN(pr64Src2))
+    {
+        if (g_aCmpTbl[bEvil].fSignalsOnQNan)
+            *pfMxcsr |= X86_MXCSR_IE;
+        fRes = g_aCmpTbl[bEvil].fUnordered;
+    }
+    else
+    {
+        softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(*pfMxcsr);
+
+        RTFLOAT64U r64Src1, r64Src2;
+        uint32_t fDe  = iemSsePrepareValueR64(&r64Src1, *pfMxcsr, pr64Src1)
+                      | iemSsePrepareValueR64(&r64Src2, *pfMxcsr, pr64Src2);
+
+        *pfMxcsr |= fDe;
+        float64_t f64Src1 = iemFpSoftF64FromIprt(&r64Src1);
+        float64_t f64Src2 = iemFpSoftF64FromIprt(&r64Src2);
+        if (f64_eq(f64Src1, f64Src2, &SoftState))
+            fRes = g_aCmpTbl[bEvil].fEqual;
+        else if (f64_lt(f64Src1, f64Src2, &SoftState))
+            fRes = g_aCmpTbl[bEvil].fLowerThan;
+        else
+            fRes = g_aCmpTbl[bEvil].fGreaterThan;
+    }
+
+    return fRes;
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpps_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->ar32); i++)
+    {
+        if (iemAImpl_cmp_worker_r32(pfMxcsr, &pSrc->uSrc1.ar32[i], &pSrc->uSrc2.ar32[i], bEvil & 0x7))
+            puDst->au32[i] = UINT32_MAX;
+        else
+            puDst->au32[i] = 0;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmppd_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bEvil))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->ar64); i++)
+    {
+        if (iemAImpl_cmp_worker_r64(pfMxcsr, &pSrc->uSrc1.ar64[i], &pSrc->uSrc2.ar64[i], bEvil & 0x7))
+            puDst->au64[i] = UINT64_MAX;
+        else
+            puDst->au64[i] = 0;
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpss_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bEvil))
+{
+    if (iemAImpl_cmp_worker_r32(pfMxcsr, &pSrc->uSrc1.ar32[0], &pSrc->uSrc2.ar32[0], bEvil & 0x7))
+        puDst->au32[0] = UINT32_MAX;
+    else
+        puDst->au32[0] = 0;
+
+    puDst->au32[1] = pSrc->uSrc1.au32[1];
+    puDst->au64[1] = pSrc->uSrc1.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cmpsd_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bEvil))
+{
+    if (iemAImpl_cmp_worker_r64(pfMxcsr, &pSrc->uSrc1.ar64[0], &pSrc->uSrc2.ar64[0], bEvil & 0x7))
+        puDst->au64[0] = UINT64_MAX;
+    else
+        puDst->au64[0] = 0;
+
+    puDst->au64[1] = pSrc->uSrc1.au64[1];
+}
+#endif
+
+
+/**
+ * ROUNDPS / ROUNDPD / ROUNDSS / ROUNDSD
+ */
+
+#define X86_SSE_ROUNDXX_IMM_RC_MASK    UINT32_C(0x0003)
+#define X86_SSE_ROUNDXX_IMM_ROUND_SEL  UINT32_C(0x0004)
+#define X86_SSE_ROUNDXX_IMM_PRECISION  UINT32_C(0x0008)
+
+DECLINLINE(softfloat_state_t) iemSseRoundXXMxcsrAndImmToSoftState(uint32_t fMxcsr, uint8_t bImm)
+{
+    if (bImm & X86_SSE_ROUNDXX_IMM_ROUND_SEL)
+        return IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+
+    fMxcsr &= ~X86_MXCSR_RC_MASK;
+    fMxcsr |= (bImm & X86_SSE_ROUNDXX_IMM_RC_MASK) << X86_MXCSR_RC_SHIFT;
+    return IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+}
+
+static RTFLOAT32U iemAImpl_round_worker_r32(uint32_t *pfMxcsr, PCRTFLOAT32U pr32Src, uint8_t bImm)
+{
+    RTFLOAT32U r32Src, r32Dst;
+    float32_t f32Src;
+    softfloat_state_t SoftState = iemSseRoundXXMxcsrAndImmToSoftState(*pfMxcsr, bImm);
+    bool fExact = !RT_BOOL(bImm & X86_SSE_ROUNDXX_IMM_PRECISION);
+
+    iemSsePrepareValueR32(&r32Src, *pfMxcsr, pr32Src);
+    f32Src = f32_roundToInt(iemFpSoftF32FromIprt(&r32Src), SoftState.roundingMode, fExact, &SoftState);
+
+    iemFpSoftF32ToIprt(&r32Dst, f32Src);
+    return r32Dst;
+}
+
+static RTFLOAT64U iemAImpl_round_worker_r64(uint32_t *pfMxcsr, PCRTFLOAT64U pr64Src, uint8_t bImm)
+{
+    RTFLOAT64U r64Src, r64Dst;
+    float64_t f64Src;
+    softfloat_state_t SoftState = iemSseRoundXXMxcsrAndImmToSoftState(*pfMxcsr, bImm);
+    bool fExact = !RT_BOOL(bImm & X86_SSE_ROUNDXX_IMM_PRECISION);
+
+    iemSsePrepareValueR64(&r64Src, *pfMxcsr, pr64Src);
+    f64Src = f64_roundToInt(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, fExact, &SoftState);
+
+    iemFpSoftF64ToIprt(&r64Dst, f64Src);
+    return r64Dst;
+}
+
+#ifdef IEM_WITHOUT_ASSEMBLY
+IEM_DECL_IMPL_DEF(void, iemAImpl_roundss_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bImm))
+{
+    puDst->ar32[0] = iemAImpl_round_worker_r32(pfMxcsr, &pSrc->uSrc2.ar32[0], bImm & 0x7);
+    puDst->au32[1] = pSrc->uSrc1.au32[1];
+    puDst->au64[1] = pSrc->uSrc1.au64[1];
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_roundsd_u128,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bImm))
+{
+    puDst->ar64[0] = iemAImpl_round_worker_r64(pfMxcsr, &pSrc->uSrc2.ar64[0], bImm & 0x7);
+    puDst->au64[1] = pSrc->uSrc1.au64[1];
+}
+#endif
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_roundps_u128_fallback,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bImm))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->ar32); i++)
+    {
+        puDst->ar32[i] = iemAImpl_round_worker_r32(pfMxcsr, &pSrc->uSrc2.ar32[i], bImm & 0x7);
+    }
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_roundpd_u128_fallback,(uint32_t *pfMxcsr, PX86XMMREG puDst, PCIEMMEDIAF2XMMSRC pSrc, uint8_t bImm))
+{
+    for (uint8_t i = 0; i < RT_ELEMENTS(puDst->ar64); i++)
+    {
+        puDst->ar64[i] = iemAImpl_round_worker_r64(pfMxcsr, &pSrc->uSrc2.ar64[i], bImm & 0x7);
+    }
+}
+
+/**
+ * CVTPD2PI
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtpd2pi_u128_worker(uint32_t fMxcsr, int32_t *pi32Dst, PCRTFLOAT64U pr64Src)
+{
+    RTFLOAT64U r64Src;
+    iemSsePrepareValueR64(&r64Src, fMxcsr, pr64Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    *pi32Dst = f64_to_i32(iemFpSoftF64FromIprt(&r64Src), SoftState.roundingMode, true /*exact*/, &SoftState);
+    return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtpd2pi_u128,(uint32_t *pfMxcsr, uint64_t *pu64Dst, PCX86XMMREG pSrc))
+{
+    RTUINT64U u64Res;
+    uint32_t fMxcsrOut  = iemAImpl_cvtpd2pi_u128_worker(*pfMxcsr, &u64Res.ai32[0], &pSrc->ar64[0]);
+             fMxcsrOut |= iemAImpl_cvtpd2pi_u128_worker(*pfMxcsr, &u64Res.ai32[1], &pSrc->ar64[1]);
+
+    *pu64Dst = u64Res.u;
+    *pfMxcsr = fMxcsrOut;
+}
+#endif
+
+
+/**
+ * CVTTPD2PI
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvttpd2pi_u128_worker(uint32_t fMxcsr, int32_t *pi32Dst, PCRTFLOAT64U pr64Src)
+{
+    RTFLOAT64U r64Src;
+    iemSsePrepareValueR64(&r64Src, fMxcsr, pr64Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    *pi32Dst = f64_to_i32_r_minMag(iemFpSoftF64FromIprt(&r64Src), true /*exact*/, &SoftState);
+    return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvttpd2pi_u128,(uint32_t *pfMxcsr, uint64_t *pu64Dst, PCX86XMMREG pSrc))
+{
+    RTUINT64U u64Res;
+    uint32_t fMxcsrOut  = iemAImpl_cvttpd2pi_u128_worker(*pfMxcsr, &u64Res.ai32[0], &pSrc->ar64[0]);
+             fMxcsrOut |= iemAImpl_cvttpd2pi_u128_worker(*pfMxcsr, &u64Res.ai32[1], &pSrc->ar64[1]);
+
+    *pu64Dst = u64Res.u;
+    *pfMxcsr = fMxcsrOut;
+}
+#endif
+
+
+/**
+ * CVTPI2PS
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtpi2ps_u128_worker(uint32_t fMxcsr, PRTFLOAT32U pr32Dst, int32_t i32Src)
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float32_t r32Res = i32_to_f32(i32Src, &SoftState);
+    return iemSseSoftStateAndR32ToMxcsrAndIprtResult(&SoftState, r32Res, pr32Dst, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtpi2ps_u128,(uint32_t *pfMxcsr, PX86XMMREG pDst, uint64_t u64Src))
+{
+    RTUINT64U uSrc = { u64Src };
+    uint32_t fMxcsrOut  = iemAImpl_cvtpi2ps_u128_worker(*pfMxcsr, &pDst->ar32[0], uSrc.ai32[0]);
+             fMxcsrOut |= iemAImpl_cvtpi2ps_u128_worker(*pfMxcsr, &pDst->ar32[1], uSrc.ai32[1]);
+    *pfMxcsr = fMxcsrOut;
+}
+#endif
+
+
+/**
+ * CVTPI2PD
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtpi2pd_u128_worker(uint32_t fMxcsr, PRTFLOAT64U pr64Dst, int32_t i32Src)
+{
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    float64_t r64Res = i32_to_f64(i32Src, &SoftState);
+    return iemSseSoftStateAndR64ToMxcsrAndIprtResult(&SoftState, r64Res, pr64Dst, fMxcsr);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtpi2pd_u128,(uint32_t *pfMxcsr, PX86XMMREG pDst, uint64_t u64Src))
+{
+    RTUINT64U uSrc = { u64Src };
+    uint32_t fMxcsrOut  = iemAImpl_cvtpi2pd_u128_worker(*pfMxcsr, &pDst->ar64[0], uSrc.ai32[0]);
+             fMxcsrOut |= iemAImpl_cvtpi2pd_u128_worker(*pfMxcsr, &pDst->ar64[1], uSrc.ai32[1]);
+    *pfMxcsr = fMxcsrOut;
+}
+#endif
+
+
+/**
+ * CVTPS2PI
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvtps2pi_u128_worker(uint32_t fMxcsr, int32_t *pi32Dst, PCRTFLOAT32U pr32Src)
+{
+    RTFLOAT32U r32Src;
+    iemSsePrepareValueR32(&r32Src, fMxcsr, pr32Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    *pi32Dst = f32_to_i32(iemFpSoftF32FromIprt(&r32Src), SoftState.roundingMode, true /*exact*/, &SoftState);
+    return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvtps2pi_u128,(uint32_t *pfMxcsr, uint64_t *pu64Dst, uint64_t u64Src))
+{
+    RTUINT64U uDst;
+    RTUINT64U uSrc = { u64Src };
+    uint32_t fMxcsrOut  = iemAImpl_cvtps2pi_u128_worker(*pfMxcsr, &uDst.ai32[0], (PCRTFLOAT32U)&uSrc.au32[0]);
+             fMxcsrOut |= iemAImpl_cvtps2pi_u128_worker(*pfMxcsr, &uDst.ai32[1], (PCRTFLOAT32U)&uSrc.au32[1]);
+    *pu64Dst = uDst.u;
+    *pfMxcsr = fMxcsrOut;
+}
+#endif
+
+
+/**
+ * CVTTPS2PI
+ */
+#ifdef IEM_WITHOUT_ASSEMBLY
+static uint32_t iemAImpl_cvttps2pi_u128_worker(uint32_t fMxcsr, int32_t *pi32Dst, PCRTFLOAT32U pr32Src)
+{
+    RTFLOAT32U r32Src;
+    iemSsePrepareValueR32(&r32Src, fMxcsr, pr32Src); /* The de-normal flag is not set. */
+
+    softfloat_state_t SoftState = IEM_SOFTFLOAT_STATE_INITIALIZER_FROM_MXCSR(fMxcsr);
+    *pi32Dst = f32_to_i32_r_minMag(iemFpSoftF32FromIprt(&r32Src), true /*exact*/, &SoftState);
+    return fMxcsr | (SoftState.exceptionFlags & X86_MXCSR_XCPT_FLAGS);
+}
+
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_cvttps2pi_u128,(uint32_t *pfMxcsr, uint64_t *pu64Dst, uint64_t u64Src))
+{
+    RTUINT64U uDst;
+    RTUINT64U uSrc = { u64Src };
+    uint32_t fMxcsrOut  = iemAImpl_cvttps2pi_u128_worker(*pfMxcsr, &uDst.ai32[0], (PCRTFLOAT32U)&uSrc.au32[0]);
+             fMxcsrOut |= iemAImpl_cvttps2pi_u128_worker(*pfMxcsr, &uDst.ai32[1], (PCRTFLOAT32U)&uSrc.au32[1]);
+    *pu64Dst = uDst.u;
+    *pfMxcsr = fMxcsrOut;
+}
+#endif
+
+/**
+ * RDRAND
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_rdrand_u16_fallback,(uint16_t *puDst, uint32_t *pEFlags))
+{
+    *puDst = 0;
+    *pEFlags &= ~X86_EFL_STATUS_BITS;
+    *pEFlags |= X86_EFL_CF;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_rdrand_u32_fallback,(uint32_t *puDst, uint32_t *pEFlags))
+{
+    *puDst = 0;
+    *pEFlags &= ~X86_EFL_STATUS_BITS;
+    *pEFlags |= X86_EFL_CF;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_rdrand_u64_fallback,(uint64_t *puDst, uint32_t *pEFlags))
+{
+    *puDst = 0;
+    *pEFlags &= ~X86_EFL_STATUS_BITS;
+    *pEFlags |= X86_EFL_CF;
+}
+
+/**
+ * RDSEED
+ */
+IEM_DECL_IMPL_DEF(void, iemAImpl_rdseed_u16_fallback,(uint16_t *puDst, uint32_t *pEFlags))
+{
+    *puDst = 0;
+    *pEFlags &= ~X86_EFL_STATUS_BITS;
+    *pEFlags |= X86_EFL_CF;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_rdseed_u32_fallback,(uint32_t *puDst, uint32_t *pEFlags))
+{
+    *puDst = 0;
+    *pEFlags &= ~X86_EFL_STATUS_BITS;
+    *pEFlags |= X86_EFL_CF;
+}
+
+IEM_DECL_IMPL_DEF(void, iemAImpl_rdseed_u64_fallback,(uint64_t *puDst, uint32_t *pEFlags))
+{
+    *puDst = 0;
+    *pEFlags &= ~X86_EFL_STATUS_BITS;
+    *pEFlags |= X86_EFL_CF;
+}
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllCImpl.cpp b/src/VBox/VMM/VMMAll/IEMAllCImpl.cpp
new file mode 100644
index 00000000..c1a4d8d8
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllCImpl.cpp
@@ -0,0 +1,9671 @@
+/* $Id: IEMAllCImpl.cpp $ */
+/** @file
+ * IEM - Instruction Implementation in C/C++ (code include).
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP   LOG_GROUP_IEM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/gim.h>
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+# include <VBox/vmm/em.h>
+# include <VBox/vmm/hm_svm.h>
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+# include <VBox/vmm/hmvmxinline.h>
+#endif
+#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
+# include <VBox/vmm/cpuidcall.h>
+#endif
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/dbgftrace.h>
+#include "IEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <iprt/asm-math.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/x86.h>
+
+#include "IEMInline.h"
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/**
+ * Flushes the prefetch buffer, light version.
+ */
+#ifndef IEM_WITH_CODE_TLB
+# define IEM_FLUSH_PREFETCH_LIGHT(a_pVCpu, a_cbInstr) do { (a_pVCpu)->iem.s.cbOpcode   = (a_cbInstr); } while (0)
+#else
+# define IEM_FLUSH_PREFETCH_LIGHT(a_pVCpu, a_cbInstr) do { } while (0)
+#endif
+
+/**
+ * Flushes the prefetch buffer, heavy version.
+ */
+#ifndef IEM_WITH_CODE_TLB
+# define IEM_FLUSH_PREFETCH_HEAVY(a_pVCpu, a_cbInstr) do { (a_pVCpu)->iem.s.cbOpcode   = (a_cbInstr); } while (0)
+#else
+# if 1
+#  define IEM_FLUSH_PREFETCH_HEAVY(a_pVCpu, a_cbInstr) do { (a_pVCpu)->iem.s.pbInstrBuf = NULL; } while (0)
+# else
+#  define IEM_FLUSH_PREFETCH_HEAVY(a_pVCpu, a_cbInstr) do { } while (0)
+# endif
+#endif
+
+
+
+/** @name Misc Helpers
+ * @{
+ */
+
+
+/**
+ * Worker function for iemHlpCheckPortIOPermission, don't call directly.
+ *
+ * @returns Strict VBox status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16Port             The port number.
+ * @param   cbOperand           The operand size.
+ */
+static VBOXSTRICTRC iemHlpCheckPortIOPermissionBitmap(PVMCPUCC pVCpu, uint16_t u16Port, uint8_t cbOperand)
+{
+    /* The TSS bits we're interested in are the same on 386 and AMD64. */
+    AssertCompile(AMD64_SEL_TYPE_SYS_TSS_BUSY  == X86_SEL_TYPE_SYS_386_TSS_BUSY);
+    AssertCompile(AMD64_SEL_TYPE_SYS_TSS_AVAIL == X86_SEL_TYPE_SYS_386_TSS_AVAIL);
+    AssertCompileMembersAtSameOffset(X86TSS32, offIoBitmap, X86TSS64, offIoBitmap);
+    AssertCompile(sizeof(X86TSS32) == sizeof(X86TSS64));
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
+
+    /*
+     * Check the TSS type, 16-bit TSSes doesn't have any I/O permission bitmap.
+     */
+    Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType);
+    if (RT_UNLIKELY(   pVCpu->cpum.GstCtx.tr.Attr.n.u4Type != AMD64_SEL_TYPE_SYS_TSS_BUSY
+                    && pVCpu->cpum.GstCtx.tr.Attr.n.u4Type != AMD64_SEL_TYPE_SYS_TSS_AVAIL))
+    {
+        Log(("iemHlpCheckPortIOPermissionBitmap: Port=%#x cb=%d - TSS type %#x (attr=%#x) has no I/O bitmap -> #GP(0)\n",
+             u16Port, cbOperand, pVCpu->cpum.GstCtx.tr.Attr.n.u4Type, pVCpu->cpum.GstCtx.tr.Attr.u));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Read the bitmap offset (may #PF).
+     */
+    uint16_t offBitmap;
+    VBOXSTRICTRC rcStrict = iemMemFetchSysU16(pVCpu, &offBitmap, UINT8_MAX,
+                                              pVCpu->cpum.GstCtx.tr.u64Base + RT_UOFFSETOF(X86TSS64, offIoBitmap));
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemHlpCheckPortIOPermissionBitmap: Error reading offIoBitmap (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * The bit range from u16Port to (u16Port + cbOperand - 1), however intel
+     * describes the CPU actually reading two bytes regardless of whether the
+     * bit range crosses a byte boundrary.  Thus the + 1 in the test below.
+     */
+    uint32_t offFirstBit = (uint32_t)u16Port / 8 + offBitmap;
+    /** @todo check if real CPUs ensures that offBitmap has a minimum value of
+     *        for instance sizeof(X86TSS32). */
+    if (offFirstBit + 1 > pVCpu->cpum.GstCtx.tr.u32Limit) /* the limit is inclusive */
+    {
+        Log(("iemHlpCheckPortIOPermissionBitmap: offFirstBit=%#x + 1 is beyond u32Limit=%#x -> #GP(0)\n",
+             offFirstBit, pVCpu->cpum.GstCtx.tr.u32Limit));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Read the necessary bits.
+     */
+    /** @todo Test the assertion in the intel manual that the CPU reads two
+     *        bytes.  The question is how this works wrt to \#PF and \#GP on the
+     *        2nd byte when it's not required. */
+    uint16_t bmBytes = UINT16_MAX;
+    rcStrict = iemMemFetchSysU16(pVCpu, &bmBytes, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base + offFirstBit);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iemHlpCheckPortIOPermissionBitmap: Error reading I/O bitmap @%#x (%Rrc)\n", offFirstBit, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /*
+     * Perform the check.
+     */
+    uint16_t fPortMask = (1 << cbOperand) - 1;
+    bmBytes >>= (u16Port & 7);
+    if (bmBytes & fPortMask)
+    {
+        Log(("iemHlpCheckPortIOPermissionBitmap: u16Port=%#x LB %u - access denied (bm=%#x mask=%#x) -> #GP(0)\n",
+             u16Port, cbOperand, bmBytes, fPortMask));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if we are allowed to access the given I/O port, raising the
+ * appropriate exceptions if we aren't (or if the I/O bitmap is not
+ * accessible).
+ *
+ * @returns Strict VBox status code.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16Port             The port number.
+ * @param   cbOperand           The operand size.
+ */
+DECLINLINE(VBOXSTRICTRC) iemHlpCheckPortIOPermission(PVMCPUCC pVCpu, uint16_t u16Port, uint8_t cbOperand)
+{
+    X86EFLAGS Efl;
+    Efl.u = IEMMISC_GET_EFL(pVCpu);
+    if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
+        && (    pVCpu->iem.s.uCpl > Efl.Bits.u2IOPL
+            ||  Efl.Bits.u1VM) )
+        return iemHlpCheckPortIOPermissionBitmap(pVCpu, u16Port, cbOperand);
+    return VINF_SUCCESS;
+}
+
+
+#if 0
+/**
+ * Calculates the parity bit.
+ *
+ * @returns true if the bit is set, false if not.
+ * @param   u8Result            The least significant byte of the result.
+ */
+static bool iemHlpCalcParityFlag(uint8_t u8Result)
+{
+    /*
+     * Parity is set if the number of bits in the least significant byte of
+     * the result is even.
+     */
+    uint8_t cBits;
+    cBits  = u8Result & 1;              /* 0 */
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    u8Result >>= 1;
+    cBits += u8Result & 1;              /* 4 */
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    u8Result >>= 1;
+    cBits += u8Result & 1;
+    return !(cBits & 1);
+}
+#endif /* not used */
+
+
+/**
+ * Updates the specified flags according to a 8-bit result.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u8Result            The result to set the flags according to.
+ * @param   fToUpdate           The flags to update.
+ * @param   fUndefined          The flags that are specified as undefined.
+ */
+static void iemHlpUpdateArithEFlagsU8(PVMCPUCC pVCpu, uint8_t u8Result, uint32_t fToUpdate, uint32_t fUndefined)
+{
+    uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
+    iemAImpl_test_u8(&u8Result, u8Result, &fEFlags);
+    pVCpu->cpum.GstCtx.eflags.u &= ~(fToUpdate | fUndefined);
+    pVCpu->cpum.GstCtx.eflags.u |= (fToUpdate | fUndefined) & fEFlags;
+}
+
+
+/**
+ * Updates the specified flags according to a 16-bit result.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   u16Result           The result to set the flags according to.
+ * @param   fToUpdate           The flags to update.
+ * @param   fUndefined          The flags that are specified as undefined.
+ */
+static void iemHlpUpdateArithEFlagsU16(PVMCPUCC pVCpu, uint16_t u16Result, uint32_t fToUpdate, uint32_t fUndefined)
+{
+    uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
+    iemAImpl_test_u16(&u16Result, u16Result, &fEFlags);
+    pVCpu->cpum.GstCtx.eflags.u &= ~(fToUpdate | fUndefined);
+    pVCpu->cpum.GstCtx.eflags.u |= (fToUpdate | fUndefined) & fEFlags;
+}
+
+
+/**
+ * Helper used by iret.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   uCpl                The new CPL.
+ * @param   pSReg               Pointer to the segment register.
+ */
+static void iemHlpAdjustSelectorForNewCpl(PVMCPUCC pVCpu, uint8_t uCpl, PCPUMSELREG pSReg)
+{
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
+
+    if (   uCpl > pSReg->Attr.n.u2Dpl
+        && pSReg->Attr.n.u1DescType /* code or data, not system */
+        &&    (pSReg->Attr.n.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+           !=                         (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) /* not conforming code */
+        iemHlpLoadNullDataSelectorProt(pVCpu, pSReg, 0);
+}
+
+
+/**
+ * Indicates that we have modified the FPU state.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ */
+DECLINLINE(void) iemHlpUsedFpu(PVMCPUCC pVCpu)
+{
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_FPU_REM);
+}
+
+/** @} */
+
+/** @name C Implementations
+ * @{
+ */
+
+/**
+ * Implements a 16-bit popa.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_popa_16)
+{
+    RTGCPTR         GCPtrStart  = iemRegGetEffRsp(pVCpu);
+    RTGCPTR         GCPtrLast   = GCPtrStart + 15;
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * The docs are a bit hard to comprehend here, but it looks like we wrap
+     * around in real mode as long as none of the individual "popa" crosses the
+     * end of the stack segment.  In protected mode we check the whole access
+     * in one go.  For efficiency, only do the word-by-word thing if we're in
+     * danger of wrapping around.
+     */
+    /** @todo do popa boundary / wrap-around checks.  */
+    if (RT_UNLIKELY(   IEM_IS_REAL_OR_V86_MODE(pVCpu)
+                    && (pVCpu->cpum.GstCtx.cs.u32Limit < GCPtrLast)) ) /* ASSUMES 64-bit RTGCPTR */
+    {
+        /* word-by-word */
+        RTUINT64U TmpRsp;
+        TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+        rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.di, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.si, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.bp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            iemRegAddToRspEx(pVCpu, &TmpRsp, 2); /* sp */
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.bx, &TmpRsp);
+        }
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.dx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.cx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &pVCpu->cpum.GstCtx.ax, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+            rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+    }
+    else
+    {
+        uint16_t const *pa16Mem = NULL;
+        rcStrict = iemMemMap(pVCpu, (void **)&pa16Mem, 16, X86_SREG_SS, GCPtrStart, IEM_ACCESS_STACK_R, sizeof(*pa16Mem) - 1);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.di = pa16Mem[7 - X86_GREG_xDI];
+            pVCpu->cpum.GstCtx.si = pa16Mem[7 - X86_GREG_xSI];
+            pVCpu->cpum.GstCtx.bp = pa16Mem[7 - X86_GREG_xBP];
+            /* skip sp */
+            pVCpu->cpum.GstCtx.bx = pa16Mem[7 - X86_GREG_xBX];
+            pVCpu->cpum.GstCtx.dx = pa16Mem[7 - X86_GREG_xDX];
+            pVCpu->cpum.GstCtx.cx = pa16Mem[7 - X86_GREG_xCX];
+            pVCpu->cpum.GstCtx.ax = pa16Mem[7 - X86_GREG_xAX];
+            rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa16Mem, IEM_ACCESS_STACK_R);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                iemRegAddToRsp(pVCpu, 16);
+                rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            }
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements a 32-bit popa.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_popa_32)
+{
+    RTGCPTR         GCPtrStart  = iemRegGetEffRsp(pVCpu);
+    RTGCPTR         GCPtrLast   = GCPtrStart + 31;
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * The docs are a bit hard to comprehend here, but it looks like we wrap
+     * around in real mode as long as none of the individual "popa" crosses the
+     * end of the stack segment.  In protected mode we check the whole access
+     * in one go.  For efficiency, only do the word-by-word thing if we're in
+     * danger of wrapping around.
+     */
+    /** @todo do popa boundary / wrap-around checks.  */
+    if (RT_UNLIKELY(   IEM_IS_REAL_OR_V86_MODE(pVCpu)
+                    && (pVCpu->cpum.GstCtx.cs.u32Limit < GCPtrLast)) ) /* ASSUMES 64-bit RTGCPTR */
+    {
+        /* word-by-word */
+        RTUINT64U TmpRsp;
+        TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+        rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.edi, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.esi, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ebp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            iemRegAddToRspEx(pVCpu, &TmpRsp, 2); /* sp */
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ebx, &TmpRsp);
+        }
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.edx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.ecx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &pVCpu->cpum.GstCtx.eax, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+#if 1  /** @todo what actually happens with the high bits when we're in 16-bit mode? */
+            pVCpu->cpum.GstCtx.rdi &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rsi &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rbp &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rbx &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rdx &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rcx &= UINT32_MAX;
+            pVCpu->cpum.GstCtx.rax &= UINT32_MAX;
+#endif
+            pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+            rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+    }
+    else
+    {
+        uint32_t const *pa32Mem;
+        rcStrict = iemMemMap(pVCpu, (void **)&pa32Mem, 32, X86_SREG_SS, GCPtrStart, IEM_ACCESS_STACK_R, sizeof(*pa32Mem) - 1);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rdi = pa32Mem[7 - X86_GREG_xDI];
+            pVCpu->cpum.GstCtx.rsi = pa32Mem[7 - X86_GREG_xSI];
+            pVCpu->cpum.GstCtx.rbp = pa32Mem[7 - X86_GREG_xBP];
+            /* skip esp */
+            pVCpu->cpum.GstCtx.rbx = pa32Mem[7 - X86_GREG_xBX];
+            pVCpu->cpum.GstCtx.rdx = pa32Mem[7 - X86_GREG_xDX];
+            pVCpu->cpum.GstCtx.rcx = pa32Mem[7 - X86_GREG_xCX];
+            pVCpu->cpum.GstCtx.rax = pa32Mem[7 - X86_GREG_xAX];
+            rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa32Mem, IEM_ACCESS_STACK_R);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                iemRegAddToRsp(pVCpu, 32);
+                rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            }
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements a 16-bit pusha.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_pusha_16)
+{
+    RTGCPTR         GCPtrTop    = iemRegGetEffRsp(pVCpu);
+    RTGCPTR         GCPtrBottom = GCPtrTop - 15;
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * The docs are a bit hard to comprehend here, but it looks like we wrap
+     * around in real mode as long as none of the individual "pushd" crosses the
+     * end of the stack segment.  In protected mode we check the whole access
+     * in one go.  For efficiency, only do the word-by-word thing if we're in
+     * danger of wrapping around.
+     */
+    /** @todo do pusha boundary / wrap-around checks.  */
+    if (RT_UNLIKELY(   GCPtrBottom > GCPtrTop
+                    && IEM_IS_REAL_OR_V86_MODE(pVCpu) ) )
+    {
+        /* word-by-word */
+        RTUINT64U TmpRsp;
+        TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+        rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.ax, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.cx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.dx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.bx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.sp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.bp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.si, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU16Ex(pVCpu, pVCpu->cpum.GstCtx.di, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+            rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+    }
+    else
+    {
+        GCPtrBottom--;
+        uint16_t *pa16Mem = NULL;
+        rcStrict = iemMemMap(pVCpu, (void **)&pa16Mem, 16, X86_SREG_SS, GCPtrBottom, IEM_ACCESS_STACK_W, sizeof(*pa16Mem) - 1);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pa16Mem[7 - X86_GREG_xDI] = pVCpu->cpum.GstCtx.di;
+            pa16Mem[7 - X86_GREG_xSI] = pVCpu->cpum.GstCtx.si;
+            pa16Mem[7 - X86_GREG_xBP] = pVCpu->cpum.GstCtx.bp;
+            pa16Mem[7 - X86_GREG_xSP] = pVCpu->cpum.GstCtx.sp;
+            pa16Mem[7 - X86_GREG_xBX] = pVCpu->cpum.GstCtx.bx;
+            pa16Mem[7 - X86_GREG_xDX] = pVCpu->cpum.GstCtx.dx;
+            pa16Mem[7 - X86_GREG_xCX] = pVCpu->cpum.GstCtx.cx;
+            pa16Mem[7 - X86_GREG_xAX] = pVCpu->cpum.GstCtx.ax;
+            rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pa16Mem, IEM_ACCESS_STACK_W);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                iemRegSubFromRsp(pVCpu, 16);
+                rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            }
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements a 32-bit pusha.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_pusha_32)
+{
+    RTGCPTR         GCPtrTop    = iemRegGetEffRsp(pVCpu);
+    RTGCPTR         GCPtrBottom = GCPtrTop - 31;
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * The docs are a bit hard to comprehend here, but it looks like we wrap
+     * around in real mode as long as none of the individual "pusha" crosses the
+     * end of the stack segment.  In protected mode we check the whole access
+     * in one go.  For efficiency, only do the word-by-word thing if we're in
+     * danger of wrapping around.
+     */
+    /** @todo do pusha boundary / wrap-around checks.  */
+    if (RT_UNLIKELY(   GCPtrBottom > GCPtrTop
+                    && IEM_IS_REAL_OR_V86_MODE(pVCpu) ) )
+    {
+        /* word-by-word */
+        RTUINT64U TmpRsp;
+        TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+        rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.eax, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ecx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.edx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ebx, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.esp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.ebp, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.esi, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, pVCpu->cpum.GstCtx.edi, &TmpRsp);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+            rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+    }
+    else
+    {
+        GCPtrBottom--;
+        uint32_t *pa32Mem;
+        rcStrict = iemMemMap(pVCpu, (void **)&pa32Mem, 32, X86_SREG_SS, GCPtrBottom, IEM_ACCESS_STACK_W, sizeof(*pa32Mem) - 1);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pa32Mem[7 - X86_GREG_xDI] = pVCpu->cpum.GstCtx.edi;
+            pa32Mem[7 - X86_GREG_xSI] = pVCpu->cpum.GstCtx.esi;
+            pa32Mem[7 - X86_GREG_xBP] = pVCpu->cpum.GstCtx.ebp;
+            pa32Mem[7 - X86_GREG_xSP] = pVCpu->cpum.GstCtx.esp;
+            pa32Mem[7 - X86_GREG_xBX] = pVCpu->cpum.GstCtx.ebx;
+            pa32Mem[7 - X86_GREG_xDX] = pVCpu->cpum.GstCtx.edx;
+            pa32Mem[7 - X86_GREG_xCX] = pVCpu->cpum.GstCtx.ecx;
+            pa32Mem[7 - X86_GREG_xAX] = pVCpu->cpum.GstCtx.eax;
+            rcStrict = iemMemCommitAndUnmap(pVCpu, pa32Mem, IEM_ACCESS_STACK_W);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                iemRegSubFromRsp(pVCpu, 32);
+                rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            }
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements pushf.
+ *
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_pushf, IEMMODE, enmEffOpSize)
+{
+    VBOXSTRICTRC rcStrict;
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_PUSHF))
+    {
+        Log2(("pushf: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_PUSHF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * If we're in V8086 mode some care is required (which is why we're in
+     * doing this in a C implementation).
+     */
+    uint32_t fEfl = IEMMISC_GET_EFL(pVCpu);
+    if (   (fEfl & X86_EFL_VM)
+        && X86_EFL_GET_IOPL(fEfl) != 3 )
+    {
+        Assert(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE);
+        if (   enmEffOpSize != IEMMODE_16BIT
+            || !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME))
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        fEfl &= ~X86_EFL_IF;          /* (RF and VM are out of range) */
+        fEfl |= (fEfl & X86_EFL_VIF) >> (19 - 9);
+        rcStrict = iemMemStackPushU16(pVCpu, (uint16_t)fEfl);
+    }
+    else
+    {
+
+        /*
+         * Ok, clear RF and VM, adjust for ancient CPUs, and push the flags.
+         */
+        fEfl &= ~(X86_EFL_RF | X86_EFL_VM);
+
+        switch (enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                AssertCompile(IEMTARGETCPU_8086 <= IEMTARGETCPU_186 && IEMTARGETCPU_V20 <= IEMTARGETCPU_186 && IEMTARGETCPU_286 > IEMTARGETCPU_186);
+                if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_186)
+                    fEfl |= UINT16_C(0xf000);
+                rcStrict = iemMemStackPushU16(pVCpu, (uint16_t)fEfl);
+                break;
+            case IEMMODE_32BIT:
+                rcStrict = iemMemStackPushU32(pVCpu, fEfl);
+                break;
+            case IEMMODE_64BIT:
+                rcStrict = iemMemStackPushU64(pVCpu, fEfl);
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+
+    if (rcStrict == VINF_SUCCESS)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements popf.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_popf, IEMMODE, enmEffOpSize)
+{
+    uint32_t const  fEflOld = IEMMISC_GET_EFL(pVCpu);
+    VBOXSTRICTRC    rcStrict;
+    uint32_t        fEflNew;
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_POPF))
+    {
+        Log2(("popf: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_POPF, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * V8086 is special as usual.
+     */
+    if (fEflOld & X86_EFL_VM)
+    {
+        /*
+         * Almost anything goes if IOPL is 3.
+         */
+        if (X86_EFL_GET_IOPL(fEflOld) == 3)
+        {
+            switch (enmEffOpSize)
+            {
+                case IEMMODE_16BIT:
+                {
+                    uint16_t u16Value;
+                    rcStrict = iemMemStackPopU16(pVCpu, &u16Value);
+                    if (rcStrict != VINF_SUCCESS)
+                        return rcStrict;
+                    fEflNew = u16Value | (fEflOld & UINT32_C(0xffff0000));
+                    break;
+                }
+                case IEMMODE_32BIT:
+                    rcStrict = iemMemStackPopU32(pVCpu, &fEflNew);
+                    if (rcStrict != VINF_SUCCESS)
+                        return rcStrict;
+                    break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+
+            const uint32_t fPopfBits = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386
+                                     ? X86_EFL_POPF_BITS : X86_EFL_POPF_BITS_386;
+            fEflNew &=   fPopfBits & ~(X86_EFL_IOPL);
+            fEflNew |= ~(fPopfBits & ~(X86_EFL_IOPL)) & fEflOld;
+        }
+        /*
+         * Interrupt flag virtualization with CR4.VME=1.
+         */
+        else if (   enmEffOpSize == IEMMODE_16BIT
+                 && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) )
+        {
+            uint16_t    u16Value;
+            RTUINT64U   TmpRsp;
+            TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Value, &TmpRsp);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /** @todo Is the popf VME \#GP(0) delivered after updating RSP+RIP
+             *        or before? */
+            if (    (   (u16Value & X86_EFL_IF)
+                     && (fEflOld  & X86_EFL_VIP))
+                ||  (u16Value & X86_EFL_TF) )
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+
+            fEflNew = u16Value | (fEflOld & UINT32_C(0xffff0000) & ~X86_EFL_VIF);
+            fEflNew |= (fEflNew & X86_EFL_IF) << (19 - 9);
+            fEflNew &=   X86_EFL_POPF_BITS & ~(X86_EFL_IOPL | X86_EFL_IF);
+            fEflNew |= ~(X86_EFL_POPF_BITS & ~(X86_EFL_IOPL | X86_EFL_IF)) & fEflOld;
+
+            pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+        }
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    }
+    /*
+     * Not in V8086 mode.
+     */
+    else
+    {
+        /* Pop the flags. */
+        switch (enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                uint16_t u16Value;
+                rcStrict = iemMemStackPopU16(pVCpu, &u16Value);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                fEflNew = u16Value | (fEflOld & UINT32_C(0xffff0000));
+
+                /*
+                 * Ancient CPU adjustments:
+                 *  - 8086, 80186, V20/30:
+                 *    Fixed bits 15:12 bits are not kept correctly internally, mostly for
+                 *    practical reasons (masking below).  We add them when pushing flags.
+                 *  - 80286:
+                 *    The NT and IOPL flags cannot be popped from real mode and are
+                 *    therefore always zero (since a 286 can never exit from PM and
+                 *    their initial value is zero).  This changed on a 386 and can
+                 *    therefore be used to detect 286 or 386 CPU in real mode.
+                 */
+                if (   IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_286
+                    && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
+                    fEflNew &= ~(X86_EFL_NT | X86_EFL_IOPL);
+                break;
+            }
+            case IEMMODE_32BIT:
+                rcStrict = iemMemStackPopU32(pVCpu, &fEflNew);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                break;
+            case IEMMODE_64BIT:
+            {
+                uint64_t u64Value;
+                rcStrict = iemMemStackPopU64(pVCpu, &u64Value);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                fEflNew = u64Value;  /** @todo testcase: Check exactly what happens if high bits are set. */
+                break;
+            }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+
+        /* Merge them with the current flags. */
+        const uint32_t fPopfBits = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.enmMicroarch != kCpumMicroarch_Intel_80386
+                                 ? X86_EFL_POPF_BITS : X86_EFL_POPF_BITS_386;
+        if (   (fEflNew & (X86_EFL_IOPL | X86_EFL_IF)) == (fEflOld & (X86_EFL_IOPL | X86_EFL_IF))
+            || pVCpu->iem.s.uCpl == 0)
+        {
+            fEflNew &=  fPopfBits;
+            fEflNew |= ~fPopfBits & fEflOld;
+        }
+        else if (pVCpu->iem.s.uCpl <= X86_EFL_GET_IOPL(fEflOld))
+        {
+            fEflNew &=   fPopfBits & ~(X86_EFL_IOPL);
+            fEflNew |= ~(fPopfBits & ~(X86_EFL_IOPL)) & fEflOld;
+        }
+        else
+        {
+            fEflNew &=   fPopfBits & ~(X86_EFL_IOPL | X86_EFL_IF);
+            fEflNew |= ~(fPopfBits & ~(X86_EFL_IOPL | X86_EFL_IF)) & fEflOld;
+        }
+    }
+
+    /*
+     * Commit the flags.
+     */
+    Assert(fEflNew & RT_BIT_32(1));
+    IEMMISC_SET_EFL(pVCpu, fEflNew);
+    return iemRegAddToRipAndFinishingClearingRfEx(pVCpu, cbInstr, fEflOld);
+}
+
+
+/**
+ * Implements an indirect call.
+ *
+ * @param   uNewPC          The new program counter (RIP) value (loaded from the
+ *                          operand).
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_16, uint16_t, uNewPC)
+{
+    uint16_t const uOldPC = pVCpu->cpum.GstCtx.ip + cbInstr;
+    if (uNewPC <= pVCpu->cpum.GstCtx.cs.u32Limit)
+    {
+        VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldPC);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rip = uNewPC;
+            IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr);
+            return iemRegFinishClearingRF(pVCpu);
+        }
+        return rcStrict;
+    }
+    return iemRaiseGeneralProtectionFault0(pVCpu);
+}
+
+
+/**
+ * Implements a 16-bit relative call.
+ *
+ * @param   offDisp      The displacment offset.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_rel_16, int16_t, offDisp)
+{
+    uint16_t const uOldPC = pVCpu->cpum.GstCtx.ip + cbInstr;
+    uint16_t const uNewPC = uOldPC + offDisp;
+    if (uNewPC <= pVCpu->cpum.GstCtx.cs.u32Limit)
+    {
+        VBOXSTRICTRC rcStrict = iemMemStackPushU16(pVCpu, uOldPC);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rip = uNewPC;
+            IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr);
+            return iemRegFinishClearingRF(pVCpu);
+        }
+        return rcStrict;
+    }
+    return iemRaiseGeneralProtectionFault0(pVCpu);
+}
+
+
+/**
+ * Implements a 32-bit indirect call.
+ *
+ * @param   uNewPC          The new program counter (RIP) value (loaded from the
+ *                          operand).
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_32, uint32_t, uNewPC)
+{
+    uint32_t const uOldPC = pVCpu->cpum.GstCtx.eip + cbInstr;
+    if (uNewPC <= pVCpu->cpum.GstCtx.cs.u32Limit)
+    {
+        VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldPC);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rip = uNewPC;
+            IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr);
+            return iemRegFinishClearingRF(pVCpu);
+        }
+        return rcStrict;
+    }
+    return iemRaiseGeneralProtectionFault0(pVCpu);
+}
+
+
+/**
+ * Implements a 32-bit relative call.
+ *
+ * @param   offDisp      The displacment offset.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_rel_32, int32_t, offDisp)
+{
+    uint32_t const uOldPC = pVCpu->cpum.GstCtx.eip + cbInstr;
+    uint32_t const uNewPC = uOldPC + offDisp;
+    if (uNewPC <= pVCpu->cpum.GstCtx.cs.u32Limit)
+    {
+        VBOXSTRICTRC rcStrict = iemMemStackPushU32(pVCpu, uOldPC);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rip = uNewPC;
+            IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr);
+            return iemRegFinishClearingRF(pVCpu);
+        }
+        return rcStrict;
+    }
+    return iemRaiseGeneralProtectionFault0(pVCpu);
+}
+
+
+/**
+ * Implements a 64-bit indirect call.
+ *
+ * @param   uNewPC          The new program counter (RIP) value (loaded from the
+ *                          operand).
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_64, uint64_t, uNewPC)
+{
+    uint64_t const uOldPC = pVCpu->cpum.GstCtx.rip + cbInstr;
+    if (IEM_IS_CANONICAL(uNewPC))
+    {
+        VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldPC);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rip = uNewPC;
+            IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr);
+            return iemRegFinishClearingRF(pVCpu);
+        }
+        return rcStrict;
+    }
+    return iemRaiseGeneralProtectionFault0(pVCpu);
+}
+
+
+/**
+ * Implements a 64-bit relative call.
+ *
+ * @param   offDisp      The displacment offset.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_call_rel_64, int64_t, offDisp)
+{
+    uint64_t const uOldPC = pVCpu->cpum.GstCtx.rip + cbInstr;
+    uint64_t const uNewPC = uOldPC + offDisp;
+    if (IEM_IS_CANONICAL(uNewPC))
+    {
+        VBOXSTRICTRC rcStrict = iemMemStackPushU64(pVCpu, uOldPC);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.rip = uNewPC;
+            IEM_FLUSH_PREFETCH_LIGHT(pVCpu, cbInstr);
+            return iemRegFinishClearingRF(pVCpu);
+        }
+        return rcStrict;
+    }
+    return iemRaiseNotCanonical(pVCpu);
+}
+
+
+/**
+ * Implements far jumps and calls thru task segments (TSS).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling thread.
+ * @param   cbInstr         The current instruction length.
+ * @param   uSel            The selector.
+ * @param   enmBranch       The kind of branching we're performing.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   pDesc           The descriptor corresponding to @a uSel. The type is
+ *                          task gate.
+ */
+static VBOXSTRICTRC iemCImpl_BranchTaskSegment(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch,
+                                               IEMMODE enmEffOpSize, PIEMSELDESC pDesc)
+{
+#ifndef IEM_IMPLEMENTS_TASKSWITCH
+    IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
+#else
+    Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL);
+    Assert(   pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL
+           || pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL);
+    RT_NOREF_PV(enmEffOpSize);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    if (   pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl
+        || pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
+    {
+        Log(("BranchTaskSegment invalid priv. uSel=%04x TSS DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
+             pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL)));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /** @todo This is checked earlier for far jumps (see iemCImpl_FarJmp) but not
+     *        far calls (see iemCImpl_callf). Most likely in both cases it should be
+     *        checked here, need testcases. */
+    if (!pDesc->Legacy.Gen.u1Present)
+    {
+        Log(("BranchTaskSegment TSS not present uSel=%04x -> #NP\n", uSel));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
+    return iemTaskSwitch(pVCpu, enmBranch == IEMBRANCH_JUMP ? IEMTASKSWITCH_JUMP : IEMTASKSWITCH_CALL,
+                         uNextEip, 0 /* fFlags */, 0 /* uErr */, 0 /* uCr2 */, uSel, pDesc);
+#endif
+}
+
+
+/**
+ * Implements far jumps and calls thru task gates.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling thread.
+ * @param   cbInstr         The current instruction length.
+ * @param   uSel            The selector.
+ * @param   enmBranch       The kind of branching we're performing.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   pDesc           The descriptor corresponding to @a uSel. The type is
+ *                          task gate.
+ */
+static VBOXSTRICTRC iemCImpl_BranchTaskGate(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch,
+                                            IEMMODE enmEffOpSize, PIEMSELDESC pDesc)
+{
+#ifndef IEM_IMPLEMENTS_TASKSWITCH
+    IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
+#else
+    Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL);
+    RT_NOREF_PV(enmEffOpSize);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    if (   pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl
+        || pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
+    {
+        Log(("BranchTaskGate invalid priv. uSel=%04x TSS DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
+             pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL)));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /** @todo This is checked earlier for far jumps (see iemCImpl_FarJmp) but not
+     *        far calls (see iemCImpl_callf). Most likely in both cases it should be
+     *        checked here, need testcases. */
+    if (!pDesc->Legacy.Gen.u1Present)
+    {
+        Log(("BranchTaskSegment segment not present uSel=%04x -> #NP\n", uSel));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    /*
+     * Fetch the new TSS descriptor from the GDT.
+     */
+    RTSEL uSelTss = pDesc->Legacy.Gate.u16Sel;
+    if (uSelTss  & X86_SEL_LDT)
+    {
+        Log(("BranchTaskGate TSS is in LDT. uSel=%04x uSelTss=%04x -> #GP\n", uSel, uSelTss));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    IEMSELDESC TssDesc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &TssDesc, uSelTss, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    if (TssDesc.Legacy.Gate.u4Type & X86_SEL_TYPE_SYS_TSS_BUSY_MASK)
+    {
+        Log(("BranchTaskGate TSS is busy. uSel=%04x uSelTss=%04x DescType=%#x -> #GP\n", uSel, uSelTss,
+             TssDesc.Legacy.Gate.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel & X86_SEL_MASK_OFF_RPL);
+    }
+
+    if (!TssDesc.Legacy.Gate.u1Present)
+    {
+        Log(("BranchTaskGate TSS is not present. uSel=%04x uSelTss=%04x -> #NP\n", uSel, uSelTss));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSelTss & X86_SEL_MASK_OFF_RPL);
+    }
+
+    uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
+    return iemTaskSwitch(pVCpu, enmBranch == IEMBRANCH_JUMP ? IEMTASKSWITCH_JUMP : IEMTASKSWITCH_CALL,
+                         uNextEip, 0 /* fFlags */, 0 /* uErr */, 0 /* uCr2 */, uSelTss, &TssDesc);
+#endif
+}
+
+
+/**
+ * Implements far jumps and calls thru call gates.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling thread.
+ * @param   cbInstr         The current instruction length.
+ * @param   uSel            The selector.
+ * @param   enmBranch       The kind of branching we're performing.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   pDesc           The descriptor corresponding to @a uSel. The type is
+ *                          call gate.
+ */
+static VBOXSTRICTRC iemCImpl_BranchCallGate(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch,
+                                            IEMMODE enmEffOpSize, PIEMSELDESC pDesc)
+{
+#define IEM_IMPLEMENTS_CALLGATE
+#ifndef IEM_IMPLEMENTS_CALLGATE
+    IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
+#else
+    RT_NOREF_PV(enmEffOpSize);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    /* NB: Far jumps can only do intra-privilege transfers. Far calls support
+     * inter-privilege calls and are much more complex.
+     *
+     * NB: 64-bit call gate has the same type as a 32-bit call gate! If
+     * EFER.LMA=1, the gate must be 64-bit. Conversely if EFER.LMA=0, the gate
+     * must be 16-bit or 32-bit.
+     */
+    /** @todo effective operand size is probably irrelevant here, only the
+     *        call gate bitness matters??
+     */
+    VBOXSTRICTRC    rcStrict;
+    RTPTRUNION      uPtrRet;
+    uint64_t        uNewRsp;
+    uint64_t        uNewRip;
+    uint64_t        u64Base;
+    uint32_t        cbLimit;
+    RTSEL           uNewCS;
+    IEMSELDESC      DescCS;
+
+    AssertCompile(X86_SEL_TYPE_SYS_386_CALL_GATE == AMD64_SEL_TYPE_SYS_CALL_GATE);
+    Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL);
+    Assert(   pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE
+           || pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE);
+
+    /* Determine the new instruction pointer from the gate descriptor. */
+    uNewRip = pDesc->Legacy.Gate.u16OffsetLow
+            | ((uint32_t)pDesc->Legacy.Gate.u16OffsetHigh << 16)
+            | ((uint64_t)pDesc->Long.Gate.u32OffsetTop    << 32);
+
+    /* Perform DPL checks on the gate descriptor. */
+    if (   pDesc->Legacy.Gate.u2Dpl < pVCpu->iem.s.uCpl
+        || pDesc->Legacy.Gate.u2Dpl < (uSel & X86_SEL_RPL))
+    {
+        Log(("BranchCallGate invalid priv. uSel=%04x Gate DPL=%d CPL=%u Sel RPL=%u -> #GP\n", uSel, pDesc->Legacy.Gate.u2Dpl,
+             pVCpu->iem.s.uCpl, (uSel & X86_SEL_RPL)));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+
+    /** @todo does this catch NULL selectors, too? */
+    if (!pDesc->Legacy.Gen.u1Present)
+    {
+        Log(("BranchCallGate Gate not present uSel=%04x -> #NP\n", uSel));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
+    }
+
+    /*
+     * Fetch the target CS descriptor from the GDT or LDT.
+     */
+    uNewCS = pDesc->Legacy.Gate.u16Sel;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCS, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Target CS must be a code selector. */
+    if (   !DescCS.Legacy.Gen.u1DescType
+        || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) )
+    {
+        Log(("BranchCallGate %04x:%08RX64 -> not a code selector (u1DescType=%u u4Type=%#x).\n",
+             uNewCS, uNewRip, DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
+    }
+
+    /* Privilege checks on target CS. */
+    if (enmBranch == IEMBRANCH_JUMP)
+    {
+        if (DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+        {
+            if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
+            {
+                Log(("BranchCallGate jump (conforming) bad DPL uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
+                     uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
+            }
+        }
+        else
+        {
+            if (DescCS.Legacy.Gen.u2Dpl != pVCpu->iem.s.uCpl)
+            {
+                Log(("BranchCallGate jump (non-conforming) bad DPL uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
+                     uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
+            }
+        }
+    }
+    else
+    {
+        Assert(enmBranch == IEMBRANCH_CALL);
+        if (DescCS.Legacy.Gen.u2Dpl > pVCpu->iem.s.uCpl)
+        {
+            Log(("BranchCallGate call invalid priv. uNewCS=%04x Gate DPL=%d CPL=%u -> #GP\n",
+                 uNewCS, DescCS.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS & X86_SEL_MASK_OFF_RPL);
+        }
+    }
+
+    /* Additional long mode checks. */
+    if (IEM_IS_LONG_MODE(pVCpu))
+    {
+        if (!DescCS.Legacy.Gen.u1Long)
+        {
+            Log(("BranchCallGate uNewCS %04x -> not a 64-bit code segment.\n", uNewCS));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
+        }
+
+        /* L vs D. */
+        if (   DescCS.Legacy.Gen.u1Long
+            && DescCS.Legacy.Gen.u1DefBig)
+        {
+            Log(("BranchCallGate uNewCS %04x -> both L and D are set.\n", uNewCS));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCS);
+        }
+    }
+
+    if (!DescCS.Legacy.Gate.u1Present)
+    {
+        Log(("BranchCallGate target CS is not present. uSel=%04x uNewCS=%04x -> #NP(CS)\n", uSel, uNewCS));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCS);
+    }
+
+    if (enmBranch == IEMBRANCH_JUMP)
+    {
+        /** @todo This is very similar to regular far jumps; merge! */
+        /* Jumps are fairly simple... */
+
+        /* Chop the high bits off if 16-bit gate (Intel says so). */
+        if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
+            uNewRip = (uint16_t)uNewRip;
+
+        /* Limit check for non-long segments. */
+        cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
+        if (DescCS.Legacy.Gen.u1Long)
+            u64Base = 0;
+        else
+        {
+            if (uNewRip > cbLimit)
+            {
+                Log(("BranchCallGate jump %04x:%08RX64 -> out of bounds (%#x) -> #GP(0)\n", uNewCS, uNewRip, cbLimit));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
+            }
+            u64Base = X86DESC_BASE(&DescCS.Legacy);
+        }
+
+        /* Canonical address check. */
+        if (!IEM_IS_CANONICAL(uNewRip))
+        {
+            Log(("BranchCallGate jump %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
+            return iemRaiseNotCanonical(pVCpu);
+        }
+
+        /*
+         * Ok, everything checked out fine.  Now set the accessed bit before
+         * committing the result into CS, CSHID and RIP.
+         */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            /** @todo check what VT-x and AMD-V does. */
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* commit */
+        pVCpu->cpum.GstCtx.rip         = uNewRip;
+        pVCpu->cpum.GstCtx.cs.Sel      = uNewCS & X86_SEL_MASK_OFF_RPL;
+        pVCpu->cpum.GstCtx.cs.Sel     |= pVCpu->iem.s.uCpl; /** @todo is this right for conforming segs? or in general? */
+        pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
+        pVCpu->cpum.GstCtx.cs.fFlags   = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.Attr.u   = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
+        pVCpu->cpum.GstCtx.cs.u64Base  = u64Base;
+        pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+    }
+    else
+    {
+        Assert(enmBranch == IEMBRANCH_CALL);
+        /* Calls are much more complicated. */
+
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF) && (DescCS.Legacy.Gen.u2Dpl < pVCpu->iem.s.uCpl))
+        {
+            uint16_t    offNewStack;    /* Offset of new stack in TSS. */
+            uint16_t    cbNewStack;     /* Number of bytes the stack information takes up in TSS. */
+            uint8_t     uNewCSDpl;
+            uint8_t     cbWords;
+            RTSEL       uNewSS;
+            RTSEL       uOldSS;
+            uint64_t    uOldRsp;
+            IEMSELDESC  DescSS;
+            RTPTRUNION  uPtrTSS;
+            RTGCPTR     GCPtrTSS;
+            RTPTRUNION  uPtrParmWds;
+            RTGCPTR     GCPtrParmWds;
+
+            /* More privilege. This is the fun part. */
+            Assert(!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF));    /* Filtered out above. */
+
+            /*
+             * Determine new SS:rSP from the TSS.
+             */
+            Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType);
+
+            /* Figure out where the new stack pointer is stored in the TSS. */
+            uNewCSDpl = DescCS.Legacy.Gen.u2Dpl;
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY)
+                {
+                    offNewStack = RT_UOFFSETOF(X86TSS32, esp0) + uNewCSDpl * 8;
+                    cbNewStack  = RT_SIZEOFMEMB(X86TSS32, esp0) + RT_SIZEOFMEMB(X86TSS32, ss0);
+                }
+                else
+                {
+                    Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY);
+                    offNewStack = RT_UOFFSETOF(X86TSS16, sp0) + uNewCSDpl * 4;
+                    cbNewStack  = RT_SIZEOFMEMB(X86TSS16, sp0) + RT_SIZEOFMEMB(X86TSS16, ss0);
+                }
+            }
+            else
+            {
+                Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY);
+                offNewStack = RT_UOFFSETOF(X86TSS64, rsp0) + uNewCSDpl * RT_SIZEOFMEMB(X86TSS64, rsp0);
+                cbNewStack  = RT_SIZEOFMEMB(X86TSS64, rsp0);
+            }
+
+            /* Check against TSS limit. */
+            if ((uint16_t)(offNewStack + cbNewStack - 1) > pVCpu->cpum.GstCtx.tr.u32Limit)
+            {
+                Log(("BranchCallGate inner stack past TSS limit - %u > %u -> #TS(TSS)\n", offNewStack + cbNewStack - 1, pVCpu->cpum.GstCtx.tr.u32Limit));
+                return iemRaiseTaskSwitchFaultBySelector(pVCpu, pVCpu->cpum.GstCtx.tr.Sel);
+            }
+
+            GCPtrTSS = pVCpu->cpum.GstCtx.tr.u64Base + offNewStack;
+            rcStrict = iemMemMap(pVCpu, &uPtrTSS.pv, cbNewStack, UINT8_MAX, GCPtrTSS, IEM_ACCESS_SYS_R, 0);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("BranchCallGate: TSS mapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY)
+                {
+                    uNewRsp = uPtrTSS.pu32[0];
+                    uNewSS  = uPtrTSS.pu16[2];
+                }
+                else
+                {
+                    Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY);
+                    uNewRsp = uPtrTSS.pu16[0];
+                    uNewSS  = uPtrTSS.pu16[1];
+                }
+            }
+            else
+            {
+                Assert(pVCpu->cpum.GstCtx.tr.Attr.n.u4Type == AMD64_SEL_TYPE_SYS_TSS_BUSY);
+                /* SS will be a NULL selector, but that's valid. */
+                uNewRsp = uPtrTSS.pu64[0];
+                uNewSS  = uNewCSDpl;
+            }
+
+            /* Done with the TSS now. */
+            rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrTSS.pv, IEM_ACCESS_SYS_R);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("BranchCallGate: TSS unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+
+            /* Only used outside of long mode. */
+            cbWords = pDesc->Legacy.Gate.u5ParmCount;
+
+            /* If EFER.LMA is 0, there's extra work to do. */
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if ((uNewSS & X86_SEL_MASK_OFF_RPL) == 0)
+                {
+                    Log(("BranchCallGate new SS NULL -> #TS(NewSS)\n"));
+                    return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
+                }
+
+                /* Grab the new SS descriptor. */
+                rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_SS);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+
+                /* Ensure that CS.DPL == SS.RPL == SS.DPL. */
+                if (   (DescCS.Legacy.Gen.u2Dpl != (uNewSS & X86_SEL_RPL))
+                    || (DescCS.Legacy.Gen.u2Dpl != DescSS.Legacy.Gen.u2Dpl))
+                {
+                    Log(("BranchCallGate call bad RPL/DPL uNewSS=%04x SS DPL=%d CS DPL=%u -> #TS(NewSS)\n",
+                         uNewSS, DescCS.Legacy.Gen.u2Dpl, DescCS.Legacy.Gen.u2Dpl));
+                    return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
+                }
+
+                /* Ensure new SS is a writable data segment. */
+                if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
+                {
+                    Log(("BranchCallGate call new SS -> not a writable data selector (u4Type=%#x)\n", DescSS.Legacy.Gen.u4Type));
+                    return iemRaiseTaskSwitchFaultBySelector(pVCpu, uNewSS);
+                }
+
+                if (!DescSS.Legacy.Gen.u1Present)
+                {
+                    Log(("BranchCallGate New stack not present uSel=%04x -> #SS(NewSS)\n", uNewSS));
+                    return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSS);
+                }
+                if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
+                    cbNewStack = (uint16_t)sizeof(uint32_t) * (4 + cbWords);
+                else
+                    cbNewStack = (uint16_t)sizeof(uint16_t) * (4 + cbWords);
+            }
+            else
+            {
+                /* Just grab the new (NULL) SS descriptor. */
+                /** @todo testcase: Check whether the zero GDT entry is actually loaded here
+                 *        like we do... */
+                rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_SS);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+
+                cbNewStack = sizeof(uint64_t) * 4;
+            }
+
+            /** @todo According to Intel, new stack is checked for enough space first,
+             *        then switched. According to AMD, the stack is switched first and
+             *        then pushes might fault!
+             *        NB: OS/2 Warp 3/4 actively relies on the fact that possible
+             *        incoming stack \#PF happens before actual stack switch. AMD is
+             *        either lying or implicitly assumes that new state is committed
+             *        only if and when an instruction doesn't fault.
+             */
+
+            /** @todo According to AMD, CS is loaded first, then SS.
+             *        According to Intel, it's the other way around!?
+             */
+
+            /** @todo Intel and AMD disagree on when exactly the CPL changes! */
+
+            /* Set the accessed bit before committing new SS. */
+            if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+            {
+                rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+            }
+
+            /* Remember the old SS:rSP and their linear address. */
+            uOldSS  = pVCpu->cpum.GstCtx.ss.Sel;
+            uOldRsp = pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig ? pVCpu->cpum.GstCtx.rsp : pVCpu->cpum.GstCtx.sp;
+
+            GCPtrParmWds = pVCpu->cpum.GstCtx.ss.u64Base + uOldRsp;
+
+            /* HACK ALERT! Probe if the write to the new stack will succeed. May #SS(NewSS)
+                           or #PF, the former is not implemented in this workaround. */
+            /** @todo Proper fix callgate target stack exceptions. */
+            /** @todo testcase: Cover callgates with partially or fully inaccessible
+             *        target stacks. */
+            void    *pvNewFrame;
+            RTGCPTR  GCPtrNewStack = X86DESC_BASE(&DescSS.Legacy) + uNewRsp - cbNewStack;
+            rcStrict = iemMemMap(pVCpu, &pvNewFrame, cbNewStack, UINT8_MAX, GCPtrNewStack, IEM_ACCESS_SYS_RW, 0);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("BranchCallGate: Incoming stack (%04x:%08RX64) not accessible, rc=%Rrc\n", uNewSS, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+            rcStrict = iemMemCommitAndUnmap(pVCpu, pvNewFrame, IEM_ACCESS_SYS_RW);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("BranchCallGate: New stack probe unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+
+            /* Commit new SS:rSP. */
+            pVCpu->cpum.GstCtx.ss.Sel      = uNewSS;
+            pVCpu->cpum.GstCtx.ss.ValidSel = uNewSS;
+            pVCpu->cpum.GstCtx.ss.Attr.u   = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
+            pVCpu->cpum.GstCtx.ss.u32Limit = X86DESC_LIMIT_G(&DescSS.Legacy);
+            pVCpu->cpum.GstCtx.ss.u64Base  = X86DESC_BASE(&DescSS.Legacy);
+            pVCpu->cpum.GstCtx.ss.fFlags   = CPUMSELREG_FLAGS_VALID;
+            pVCpu->cpum.GstCtx.rsp         = uNewRsp;
+            pVCpu->iem.s.uCpl = uNewCSDpl; /** @todo is the parameter words accessed using the new CPL or the old CPL? */
+            Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+            CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+
+            /* At this point the stack access must not fail because new state was already committed. */
+            /** @todo this can still fail due to SS.LIMIT not check.   */
+            rcStrict = iemMemStackPushBeginSpecial(pVCpu, cbNewStack,
+                                                   IEM_IS_LONG_MODE(pVCpu) ? 7
+                                                   : pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 3 : 1,
+                                                   &uPtrRet.pv, &uNewRsp);
+            AssertMsgReturn(rcStrict == VINF_SUCCESS, ("BranchCallGate: New stack mapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)),
+                            VERR_INTERNAL_ERROR_5);
+
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
+                {
+                    if (cbWords)
+                    {
+                        /* Map the relevant chunk of the old stack. */
+                        rcStrict = iemMemMap(pVCpu, &uPtrParmWds.pv, cbWords * 4, UINT8_MAX, GCPtrParmWds,
+                                             IEM_ACCESS_DATA_R, 0 /** @todo Can uNewCSDpl == 3? Then we need alignment mask here! */);
+                        if (rcStrict != VINF_SUCCESS)
+                        {
+                            Log(("BranchCallGate: Old stack mapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                            return rcStrict;
+                        }
+
+                        /* Copy the parameter (d)words. */
+                        for (int i = 0; i < cbWords; ++i)
+                            uPtrRet.pu32[2 + i] = uPtrParmWds.pu32[i];
+
+                        /* Unmap the old stack. */
+                        rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrParmWds.pv, IEM_ACCESS_DATA_R);
+                        if (rcStrict != VINF_SUCCESS)
+                        {
+                            Log(("BranchCallGate: Old stack unmapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                            return rcStrict;
+                        }
+                    }
+
+                    /* Push the old CS:rIP. */
+                    uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
+                    uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when pushing CS? */
+
+                    /* Push the old SS:rSP. */
+                    uPtrRet.pu32[2 + cbWords + 0] = uOldRsp;
+                    uPtrRet.pu32[2 + cbWords + 1] = uOldSS;
+                }
+                else
+                {
+                    Assert(pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE);
+
+                    if (cbWords)
+                    {
+                        /* Map the relevant chunk of the old stack. */
+                        rcStrict = iemMemMap(pVCpu, &uPtrParmWds.pv, cbWords * 2, UINT8_MAX, GCPtrParmWds,
+                                             IEM_ACCESS_DATA_R, 0 /** @todo Can uNewCSDpl == 3? Then we need alignment mask here! */);
+                        if (rcStrict != VINF_SUCCESS)
+                        {
+                            Log(("BranchCallGate: Old stack mapping (16-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                            return rcStrict;
+                        }
+
+                        /* Copy the parameter words. */
+                        for (int i = 0; i < cbWords; ++i)
+                            uPtrRet.pu16[2 + i] = uPtrParmWds.pu16[i];
+
+                        /* Unmap the old stack. */
+                        rcStrict = iemMemCommitAndUnmap(pVCpu, uPtrParmWds.pv, IEM_ACCESS_DATA_R);
+                        if (rcStrict != VINF_SUCCESS)
+                        {
+                            Log(("BranchCallGate: Old stack unmapping (32-bit) failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                            return rcStrict;
+                        }
+                    }
+
+                    /* Push the old CS:rIP. */
+                    uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
+                    uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
+
+                    /* Push the old SS:rSP. */
+                    uPtrRet.pu16[2 + cbWords + 0] = uOldRsp;
+                    uPtrRet.pu16[2 + cbWords + 1] = uOldSS;
+                }
+            }
+            else
+            {
+                Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
+
+                /* For 64-bit gates, no parameters are copied. Just push old SS:rSP and CS:rIP. */
+                uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
+                uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when pushing CS? */
+                uPtrRet.pu64[2] = uOldRsp;
+                uPtrRet.pu64[3] = uOldSS;       /** @todo Testcase: What is written to the high words when pushing SS? */
+            }
+
+            rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
+            if (rcStrict != VINF_SUCCESS)
+            {
+                Log(("BranchCallGate: New stack unmapping failed (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+                return rcStrict;
+            }
+
+            /* Chop the high bits off if 16-bit gate (Intel says so). */
+            if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
+                uNewRip = (uint16_t)uNewRip;
+
+            /* Limit / canonical check. */
+            cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (uNewRip > cbLimit)
+                {
+                    Log(("BranchCallGate %04x:%08RX64 -> out of bounds (%#x)\n", uNewCS, uNewRip, cbLimit));
+                    return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
+                }
+                u64Base = X86DESC_BASE(&DescCS.Legacy);
+            }
+            else
+            {
+                Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
+                if (!IEM_IS_CANONICAL(uNewRip))
+                {
+                    Log(("BranchCallGate call %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
+                    return iemRaiseNotCanonical(pVCpu);
+                }
+                u64Base = 0;
+            }
+
+            /*
+             * Now set the accessed bit before
+             * writing the return address to the stack and committing the result into
+             * CS, CSHID and RIP.
+             */
+            /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
+            if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+            {
+                rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                /** @todo check what VT-x and AMD-V does. */
+                DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+            }
+
+            /* Commit new CS:rIP. */
+            pVCpu->cpum.GstCtx.rip         = uNewRip;
+            pVCpu->cpum.GstCtx.cs.Sel      = uNewCS & X86_SEL_MASK_OFF_RPL;
+            pVCpu->cpum.GstCtx.cs.Sel     |= pVCpu->iem.s.uCpl;
+            pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
+            pVCpu->cpum.GstCtx.cs.fFlags   = CPUMSELREG_FLAGS_VALID;
+            pVCpu->cpum.GstCtx.cs.Attr.u   = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+            pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
+            pVCpu->cpum.GstCtx.cs.u64Base  = u64Base;
+            pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+        }
+        else
+        {
+            /* Same privilege. */
+            /** @todo This is very similar to regular far calls; merge! */
+
+            /* Check stack first - may #SS(0). */
+            /** @todo check how gate size affects pushing of CS! Does callf 16:32 in
+             *        16-bit code cause a two or four byte CS to be pushed? */
+            rcStrict = iemMemStackPushBeginSpecial(pVCpu,
+                                                   IEM_IS_LONG_MODE(pVCpu) ? 8+8
+                                                   : pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 4+4 : 2+2,
+                                                   IEM_IS_LONG_MODE(pVCpu) ? 7
+                                                   : pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE ? 3 : 2,
+                                                   &uPtrRet.pv, &uNewRsp);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /* Chop the high bits off if 16-bit gate (Intel says so). */
+            if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE)
+                uNewRip = (uint16_t)uNewRip;
+
+            /* Limit / canonical check. */
+            cbLimit = X86DESC_LIMIT_G(&DescCS.Legacy);
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (uNewRip > cbLimit)
+                {
+                    Log(("BranchCallGate %04x:%08RX64 -> out of bounds (%#x)\n", uNewCS, uNewRip, cbLimit));
+                    return iemRaiseGeneralProtectionFaultBySelector(pVCpu, 0);
+                }
+                u64Base = X86DESC_BASE(&DescCS.Legacy);
+            }
+            else
+            {
+                if (!IEM_IS_CANONICAL(uNewRip))
+                {
+                    Log(("BranchCallGate call %04x:%016RX64 - not canonical -> #GP\n", uNewCS, uNewRip));
+                    return iemRaiseNotCanonical(pVCpu);
+                }
+                u64Base = 0;
+            }
+
+            /*
+             * Now set the accessed bit before
+             * writing the return address to the stack and committing the result into
+             * CS, CSHID and RIP.
+             */
+            /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
+            if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+            {
+                rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCS);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+                /** @todo check what VT-x and AMD-V does. */
+                DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+            }
+
+            /* stack */
+            if (!IEM_IS_LONG_MODE(pVCpu))
+            {
+                if (pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_386_CALL_GATE)
+                {
+                    uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
+                    uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when pushing CS? */
+                }
+                else
+                {
+                    Assert(pDesc->Legacy.Gate.u4Type == X86_SEL_TYPE_SYS_286_CALL_GATE);
+                    uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
+                    uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
+                }
+            }
+            else
+            {
+                Assert(pDesc->Legacy.Gate.u4Type == AMD64_SEL_TYPE_SYS_CALL_GATE);
+                uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
+                uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when pushing CS? */
+            }
+
+            rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /* commit */
+            pVCpu->cpum.GstCtx.rip         = uNewRip;
+            pVCpu->cpum.GstCtx.cs.Sel      = uNewCS & X86_SEL_MASK_OFF_RPL;
+            pVCpu->cpum.GstCtx.cs.Sel     |= pVCpu->iem.s.uCpl;
+            pVCpu->cpum.GstCtx.cs.ValidSel = pVCpu->cpum.GstCtx.cs.Sel;
+            pVCpu->cpum.GstCtx.cs.fFlags   = CPUMSELREG_FLAGS_VALID;
+            pVCpu->cpum.GstCtx.cs.Attr.u   = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+            pVCpu->cpum.GstCtx.cs.u32Limit = cbLimit;
+            pVCpu->cpum.GstCtx.cs.u64Base  = u64Base;
+            pVCpu->iem.s.enmCpuMode  = iemCalcCpuMode(pVCpu);
+        }
+    }
+    pVCpu->cpum.GstCtx.eflags.Bits.u1RF = 0;
+/** @todo single stepping   */
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
+    return VINF_SUCCESS;
+#endif /* IEM_IMPLEMENTS_CALLGATE */
+}
+
+
+/**
+ * Implements far jumps and calls thru system selectors.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling thread.
+ * @param   cbInstr         The current instruction length.
+ * @param   uSel            The selector.
+ * @param   enmBranch       The kind of branching we're performing.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   pDesc           The descriptor corresponding to @a uSel.
+ */
+static VBOXSTRICTRC iemCImpl_BranchSysSel(PVMCPUCC pVCpu, uint8_t cbInstr, uint16_t uSel, IEMBRANCH enmBranch,
+                                          IEMMODE enmEffOpSize, PIEMSELDESC pDesc)
+{
+    Assert(enmBranch == IEMBRANCH_JUMP || enmBranch == IEMBRANCH_CALL);
+    Assert((uSel & X86_SEL_MASK_OFF_RPL));
+    IEM_CTX_IMPORT_RET(pVCpu, IEM_CPUMCTX_EXTRN_XCPT_MASK);
+
+    if (IEM_IS_LONG_MODE(pVCpu))
+        switch (pDesc->Legacy.Gen.u4Type)
+        {
+            case AMD64_SEL_TYPE_SYS_CALL_GATE:
+                return iemCImpl_BranchCallGate(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc);
+
+            default:
+            case AMD64_SEL_TYPE_SYS_LDT:
+            case AMD64_SEL_TYPE_SYS_TSS_BUSY:
+            case AMD64_SEL_TYPE_SYS_TSS_AVAIL:
+            case AMD64_SEL_TYPE_SYS_TRAP_GATE:
+            case AMD64_SEL_TYPE_SYS_INT_GATE:
+                Log(("branch %04x -> wrong sys selector (64-bit): %d\n", uSel, pDesc->Legacy.Gen.u4Type));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+
+    switch (pDesc->Legacy.Gen.u4Type)
+    {
+        case X86_SEL_TYPE_SYS_286_CALL_GATE:
+        case X86_SEL_TYPE_SYS_386_CALL_GATE:
+            return iemCImpl_BranchCallGate(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc);
+
+        case X86_SEL_TYPE_SYS_TASK_GATE:
+            return iemCImpl_BranchTaskGate(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc);
+
+        case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
+        case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
+            return iemCImpl_BranchTaskSegment(pVCpu, cbInstr, uSel, enmBranch, enmEffOpSize, pDesc);
+
+        case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+            Log(("branch %04x -> busy 286 TSS\n", uSel));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+
+        case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+            Log(("branch %04x -> busy 386 TSS\n", uSel));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+
+        default:
+        case X86_SEL_TYPE_SYS_LDT:
+        case X86_SEL_TYPE_SYS_286_INT_GATE:
+        case X86_SEL_TYPE_SYS_286_TRAP_GATE:
+        case X86_SEL_TYPE_SYS_386_INT_GATE:
+        case X86_SEL_TYPE_SYS_386_TRAP_GATE:
+            Log(("branch %04x -> wrong sys selector: %d\n", uSel, pDesc->Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+}
+
+
+/**
+ * Implements far jumps.
+ *
+ * @param   uSel            The selector.
+ * @param   offSeg          The segment offset.
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_FarJmp, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize)
+{
+    NOREF(cbInstr);
+    Assert(offSeg <= UINT32_MAX || (!IEM_IS_GUEST_CPU_AMD(pVCpu) && pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT));
+
+    /*
+     * Real mode and V8086 mode are easy.  The only snag seems to be that
+     * CS.limit doesn't change and the limit check is done against the current
+     * limit.
+     */
+    /** @todo Robert Collins claims (The Segment Descriptor Cache, DDJ August
+     *        1998) that up to and including the Intel 486, far control
+     *        transfers in real mode set default CS attributes (0x93) and also
+     *        set a 64K segment limit. Starting with the Pentium, the
+     *        attributes and limit are left alone but the access rights are
+     *        ignored. We only implement the Pentium+ behavior.
+     *  */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        Assert(enmEffOpSize == IEMMODE_16BIT || enmEffOpSize == IEMMODE_32BIT);
+        if (offSeg > pVCpu->cpum.GstCtx.cs.u32Limit)
+        {
+            Log(("iemCImpl_FarJmp: 16-bit limit\n"));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        if (enmEffOpSize == IEMMODE_16BIT) /** @todo WRONG, must pass this. */
+            pVCpu->cpum.GstCtx.rip       = offSeg;
+        else
+            pVCpu->cpum.GstCtx.rip       = offSeg & UINT16_MAX;
+        pVCpu->cpum.GstCtx.cs.Sel        = uSel;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uSel;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.u64Base    = (uint32_t)uSel << 4;
+
+        return iemRegFinishClearingRF(pVCpu);
+    }
+
+    /*
+     * Protected mode. Need to parse the specified descriptor...
+     */
+    if (!(uSel & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("jmpf %04x:%08RX64 -> invalid selector, #GP(0)\n", uSel, offSeg));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Fetch the descriptor. */
+    IEMSELDESC Desc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Is it there? */
+    if (!Desc.Legacy.Gen.u1Present) /** @todo this is probably checked too early. Testcase! */
+    {
+        Log(("jmpf %04x:%08RX64 -> segment not present\n", uSel, offSeg));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
+    }
+
+    /*
+     * Deal with it according to its type.  We do the standard code selectors
+     * here and dispatch the system selectors to worker functions.
+     */
+    if (!Desc.Legacy.Gen.u1DescType)
+        return iemCImpl_BranchSysSel(pVCpu, cbInstr, uSel, IEMBRANCH_JUMP, enmEffOpSize, &Desc);
+
+    /* Only code segments. */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+    {
+        Log(("jmpf %04x:%08RX64 -> not a code selector (u4Type=%#x).\n", uSel, offSeg, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+
+    /* L vs D. */
+    if (   Desc.Legacy.Gen.u1Long
+        && Desc.Legacy.Gen.u1DefBig
+        && IEM_IS_LONG_MODE(pVCpu))
+    {
+        Log(("jmpf %04x:%08RX64 -> both L and D are set.\n", uSel, offSeg));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+
+    /* DPL/RPL/CPL check, where conforming segments makes a difference. */
+    if (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+    {
+        if (pVCpu->iem.s.uCpl < Desc.Legacy.Gen.u2Dpl)
+        {
+            Log(("jmpf %04x:%08RX64 -> DPL violation (conforming); DPL=%d CPL=%u\n",
+                 uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+    else
+    {
+        if (pVCpu->iem.s.uCpl != Desc.Legacy.Gen.u2Dpl)
+        {
+            Log(("jmpf %04x:%08RX64 -> CPL != DPL; DPL=%d CPL=%u\n", uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+        if ((uSel & X86_SEL_RPL) > pVCpu->iem.s.uCpl)
+        {
+            Log(("jmpf %04x:%08RX64 -> RPL > DPL; RPL=%d CPL=%u\n", uSel, offSeg, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+
+    /* Chop the high bits if 16-bit (Intel says so). */
+    if (enmEffOpSize == IEMMODE_16BIT)
+        offSeg &= UINT16_MAX;
+
+    /* Limit check and get the base.  */
+    uint64_t u64Base;
+    uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
+    if (   !Desc.Legacy.Gen.u1Long
+        || !IEM_IS_LONG_MODE(pVCpu))
+    {
+        if (RT_LIKELY(offSeg <= cbLimit))
+            u64Base = X86DESC_BASE(&Desc.Legacy);
+        else
+        {
+            Log(("jmpf %04x:%08RX64 -> out of bounds (%#x)\n", uSel, offSeg, cbLimit));
+            /** @todo Intel says this is \#GP(0)! */
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+    else
+        u64Base = 0;
+
+    /*
+     * Ok, everything checked out fine.  Now set the accessed bit before
+     * committing the result into CS, CSHID and RIP.
+     */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        /** @todo check what VT-x and AMD-V does. */
+        Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    /* commit */
+    pVCpu->cpum.GstCtx.rip = offSeg;
+    pVCpu->cpum.GstCtx.cs.Sel         = uSel & X86_SEL_MASK_OFF_RPL;
+    pVCpu->cpum.GstCtx.cs.Sel        |= pVCpu->iem.s.uCpl; /** @todo is this right for conforming segs? or in general? */
+    pVCpu->cpum.GstCtx.cs.ValidSel    = pVCpu->cpum.GstCtx.cs.Sel;
+    pVCpu->cpum.GstCtx.cs.fFlags      = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.Attr.u      = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.cs.u32Limit    = cbLimit;
+    pVCpu->cpum.GstCtx.cs.u64Base     = u64Base;
+    pVCpu->iem.s.enmCpuMode  = iemCalcCpuMode(pVCpu);
+    /** @todo check if the hidden bits are loaded correctly for 64-bit
+     *        mode.  */
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
+
+    return iemRegFinishClearingRF(pVCpu);
+}
+
+
+/**
+ * Implements far calls.
+ *
+ * This very similar to iemCImpl_FarJmp.
+ *
+ * @param   uSel            The selector.
+ * @param   offSeg          The segment offset.
+ * @param   enmEffOpSize    The operand size (in case we need it).
+ */
+IEM_CIMPL_DEF_3(iemCImpl_callf, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmEffOpSize)
+{
+    VBOXSTRICTRC    rcStrict;
+    uint64_t        uNewRsp;
+    RTPTRUNION      uPtrRet;
+
+    /*
+     * Real mode and V8086 mode are easy.  The only snag seems to be that
+     * CS.limit doesn't change and the limit check is done against the current
+     * limit.
+     */
+    /** @todo See comment for similar code in iemCImpl_FarJmp */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        Assert(enmEffOpSize == IEMMODE_16BIT || enmEffOpSize == IEMMODE_32BIT);
+
+        /* Check stack first - may #SS(0). */
+        rcStrict = iemMemStackPushBeginSpecial(pVCpu, enmEffOpSize == IEMMODE_32BIT ? 4+4 : 2+2,
+                                               enmEffOpSize == IEMMODE_32BIT ? 3 : 1,
+                                               &uPtrRet.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /* Check the target address range. */
+/** @todo this must be wrong! Write unreal mode tests! */
+        if (offSeg > UINT32_MAX)
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+
+        /* Everything is fine, push the return address. */
+        if (enmEffOpSize == IEMMODE_16BIT)
+        {
+            uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
+            uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
+        }
+        else
+        {
+            uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
+            uPtrRet.pu16[2] = pVCpu->cpum.GstCtx.cs.Sel;
+        }
+        rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /* Branch. */
+        pVCpu->cpum.GstCtx.rip           = offSeg;
+        pVCpu->cpum.GstCtx.cs.Sel        = uSel;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uSel;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.u64Base    = (uint32_t)uSel << 4;
+
+        return iemRegFinishClearingRF(pVCpu);
+    }
+
+    /*
+     * Protected mode. Need to parse the specified descriptor...
+     */
+    if (!(uSel & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("callf %04x:%08RX64 -> invalid selector, #GP(0)\n", uSel, offSeg));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Fetch the descriptor. */
+    IEMSELDESC Desc;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Deal with it according to its type.  We do the standard code selectors
+     * here and dispatch the system selectors to worker functions.
+     */
+    if (!Desc.Legacy.Gen.u1DescType)
+        return iemCImpl_BranchSysSel(pVCpu, cbInstr, uSel, IEMBRANCH_CALL, enmEffOpSize, &Desc);
+
+    /* Only code segments. */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+    {
+        Log(("callf %04x:%08RX64 -> not a code selector (u4Type=%#x).\n", uSel, offSeg, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+
+    /* L vs D. */
+    if (   Desc.Legacy.Gen.u1Long
+        && Desc.Legacy.Gen.u1DefBig
+        && IEM_IS_LONG_MODE(pVCpu))
+    {
+        Log(("callf %04x:%08RX64 -> both L and D are set.\n", uSel, offSeg));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+    }
+
+    /* DPL/RPL/CPL check, where conforming segments makes a difference. */
+    if (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+    {
+        if (pVCpu->iem.s.uCpl < Desc.Legacy.Gen.u2Dpl)
+        {
+            Log(("callf %04x:%08RX64 -> DPL violation (conforming); DPL=%d CPL=%u\n",
+                 uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+    else
+    {
+        if (pVCpu->iem.s.uCpl != Desc.Legacy.Gen.u2Dpl)
+        {
+            Log(("callf %04x:%08RX64 -> CPL != DPL; DPL=%d CPL=%u\n", uSel, offSeg, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+        if ((uSel & X86_SEL_RPL) > pVCpu->iem.s.uCpl)
+        {
+            Log(("callf %04x:%08RX64 -> RPL > DPL; RPL=%d CPL=%u\n", uSel, offSeg, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+
+    /* Is it there? */
+    if (!Desc.Legacy.Gen.u1Present)
+    {
+        Log(("callf %04x:%08RX64 -> segment not present\n", uSel, offSeg));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
+    }
+
+    /* Check stack first - may #SS(0). */
+    /** @todo check how operand prefix affects pushing of CS! Does callf 16:32 in
+     *        16-bit code cause a two or four byte CS to be pushed? */
+    rcStrict = iemMemStackPushBeginSpecial(pVCpu,
+                                           enmEffOpSize == IEMMODE_64BIT ? 8+8 : enmEffOpSize == IEMMODE_32BIT ? 4+4 : 2+2,
+                                           enmEffOpSize == IEMMODE_64BIT ? 7   : enmEffOpSize == IEMMODE_32BIT ? 3   : 1,
+                                           &uPtrRet.pv, &uNewRsp);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Chop the high bits if 16-bit (Intel says so). */
+    if (enmEffOpSize == IEMMODE_16BIT)
+        offSeg &= UINT16_MAX;
+
+    /* Limit / canonical check. */
+    uint64_t u64Base;
+    uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
+    if (   !Desc.Legacy.Gen.u1Long
+        || !IEM_IS_LONG_MODE(pVCpu))
+    {
+        if (RT_LIKELY(offSeg <= cbLimit))
+            u64Base = X86DESC_BASE(&Desc.Legacy);
+        else
+        {
+            Log(("jmpf %04x:%08RX64 -> out of bounds (%#x)\n", uSel, offSeg, cbLimit));
+            /** @todo Intel says this is \#GP(0)! */
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+    }
+    else if (IEM_IS_CANONICAL(offSeg))
+        u64Base = 0;
+    else
+    {
+        Log(("callf %04x:%016RX64 - not canonical -> #GP\n", uSel, offSeg));
+        return iemRaiseNotCanonical(pVCpu);
+    }
+
+    /*
+     * Now set the accessed bit before
+     * writing the return address to the stack and committing the result into
+     * CS, CSHID and RIP.
+     */
+    /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
+    if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        /** @todo check what VT-x and AMD-V does. */
+        Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    /* stack */
+    if (enmEffOpSize == IEMMODE_16BIT)
+    {
+        uPtrRet.pu16[0] = pVCpu->cpum.GstCtx.ip + cbInstr;
+        uPtrRet.pu16[1] = pVCpu->cpum.GstCtx.cs.Sel;
+    }
+    else if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        uPtrRet.pu32[0] = pVCpu->cpum.GstCtx.eip + cbInstr;
+        uPtrRet.pu32[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high word when callf is pushing CS? */
+    }
+    else
+    {
+        uPtrRet.pu64[0] = pVCpu->cpum.GstCtx.rip + cbInstr;
+        uPtrRet.pu64[1] = pVCpu->cpum.GstCtx.cs.Sel; /** @todo Testcase: What is written to the high words when callf is pushing CS? */
+    }
+    rcStrict = iemMemStackPushCommitSpecial(pVCpu, uPtrRet.pv, uNewRsp);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* commit */
+    pVCpu->cpum.GstCtx.rip = offSeg;
+    pVCpu->cpum.GstCtx.cs.Sel         = uSel & X86_SEL_MASK_OFF_RPL;
+    pVCpu->cpum.GstCtx.cs.Sel        |= pVCpu->iem.s.uCpl;
+    pVCpu->cpum.GstCtx.cs.ValidSel    = pVCpu->cpum.GstCtx.cs.Sel;
+    pVCpu->cpum.GstCtx.cs.fFlags      = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.Attr.u      = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.cs.u32Limit    = cbLimit;
+    pVCpu->cpum.GstCtx.cs.u64Base     = u64Base;
+    pVCpu->iem.s.enmCpuMode  = iemCalcCpuMode(pVCpu);
+    /** @todo check if the hidden bits are loaded correctly for 64-bit
+     *        mode.  */
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
+
+    return iemRegFinishClearingRF(pVCpu);
+}
+
+
+/**
+ * Implements retf.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   cbPop           The amount of arguments to pop from the stack
+ *                          (bytes).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_retf, IEMMODE, enmEffOpSize, uint16_t, cbPop)
+{
+    VBOXSTRICTRC    rcStrict;
+    RTCPTRUNION     uPtrFrame;
+    RTUINT64U       NewRsp;
+    uint64_t        uNewRip;
+    uint16_t        uNewCs;
+    NOREF(cbInstr);
+
+    /*
+     * Read the stack values first.
+     */
+    uint32_t        cbRetPtr = enmEffOpSize == IEMMODE_16BIT ? 2+2
+                             : enmEffOpSize == IEMMODE_32BIT ? 4+4 : 8+8;
+    rcStrict = iemMemStackPopBeginSpecial(pVCpu, cbRetPtr,
+                                          enmEffOpSize == IEMMODE_16BIT ? 1 : enmEffOpSize == IEMMODE_32BIT ? 3 : 7,
+                                          &uPtrFrame.pv, &NewRsp.u);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    if (enmEffOpSize == IEMMODE_16BIT)
+    {
+        uNewRip = uPtrFrame.pu16[0];
+        uNewCs  = uPtrFrame.pu16[1];
+    }
+    else if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        uNewRip = uPtrFrame.pu32[0];
+        uNewCs  = uPtrFrame.pu16[2];
+    }
+    else
+    {
+        uNewRip = uPtrFrame.pu64[0];
+        uNewCs  = uPtrFrame.pu16[4];
+    }
+    rcStrict = iemMemStackPopDoneSpecial(pVCpu, uPtrFrame.pv);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* extremely likely */ }
+    else
+        return rcStrict;
+
+    /*
+     * Real mode and V8086 mode are easy.
+     */
+    /** @todo See comment for similar code in iemCImpl_FarJmp */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
+        /** @todo check how this is supposed to work if sp=0xfffe. */
+
+        /* Check the limit of the new EIP. */
+        /** @todo Intel pseudo code only does the limit check for 16-bit
+         *        operands, AMD does not make any distinction. What is right? */
+        if (uNewRip > pVCpu->cpum.GstCtx.cs.u32Limit)
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+
+        /* commit the operation. */
+        if (cbPop)
+            iemRegAddToRspEx(pVCpu, &NewRsp, cbPop);
+        pVCpu->cpum.GstCtx.rsp           = NewRsp.u;
+        pVCpu->cpum.GstCtx.rip           = uNewRip;
+        pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.u64Base    = (uint32_t)uNewCs << 4;
+        return iemRegFinishClearingRF(pVCpu);
+    }
+
+    /*
+     * Protected mode is complicated, of course.
+     */
+    if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("retf %04x:%08RX64 -> invalid selector, #GP(0)\n", uNewCs, uNewRip));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
+
+    /* Fetch the descriptor. */
+    IEMSELDESC DescCs;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCs, uNewCs, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Can only return to a code selector. */
+    if (   !DescCs.Legacy.Gen.u1DescType
+        || !(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE) )
+    {
+        Log(("retf %04x:%08RX64 -> not a code selector (u1DescType=%u u4Type=%#x).\n",
+             uNewCs, uNewRip, DescCs.Legacy.Gen.u1DescType, DescCs.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* L vs D. */
+    if (   DescCs.Legacy.Gen.u1Long /** @todo Testcase: far return to a selector with both L and D set. */
+        && DescCs.Legacy.Gen.u1DefBig
+        && IEM_IS_LONG_MODE(pVCpu))
+    {
+        Log(("retf %04x:%08RX64 -> both L & D set.\n", uNewCs, uNewRip));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* DPL/RPL/CPL checks. */
+    if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl)
+    {
+        Log(("retf %04x:%08RX64 -> RPL < CPL(%d).\n", uNewCs, uNewRip, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    if (DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF)
+    {
+        if ((uNewCs & X86_SEL_RPL) < DescCs.Legacy.Gen.u2Dpl)
+        {
+            Log(("retf %04x:%08RX64 -> DPL violation (conforming); DPL=%u RPL=%u\n",
+                 uNewCs, uNewRip, DescCs.Legacy.Gen.u2Dpl, (uNewCs & X86_SEL_RPL)));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+        }
+    }
+    else
+    {
+        if ((uNewCs & X86_SEL_RPL) != DescCs.Legacy.Gen.u2Dpl)
+        {
+            Log(("retf %04x:%08RX64 -> RPL != DPL; DPL=%u RPL=%u\n",
+                 uNewCs, uNewRip, DescCs.Legacy.Gen.u2Dpl, (uNewCs & X86_SEL_RPL)));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+        }
+    }
+
+    /* Is it there? */
+    if (!DescCs.Legacy.Gen.u1Present)
+    {
+        Log(("retf %04x:%08RX64 -> segment not present\n", uNewCs, uNewRip));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
+    }
+
+    /*
+     * Return to outer privilege? (We'll typically have entered via a call gate.)
+     */
+    if ((uNewCs & X86_SEL_RPL) != pVCpu->iem.s.uCpl)
+    {
+        /* Read the outer stack pointer stored *after* the parameters. */
+        rcStrict = iemMemStackPopContinueSpecial(pVCpu, cbPop /*off*/, cbRetPtr, &uPtrFrame.pv, NewRsp.u);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        uint16_t uNewOuterSs;
+        RTUINT64U NewOuterRsp;
+        if (enmEffOpSize == IEMMODE_16BIT)
+        {
+            NewOuterRsp.u = uPtrFrame.pu16[0];
+            uNewOuterSs   = uPtrFrame.pu16[1];
+        }
+        else if (enmEffOpSize == IEMMODE_32BIT)
+        {
+            NewOuterRsp.u = uPtrFrame.pu32[0];
+            uNewOuterSs   = uPtrFrame.pu16[2];
+        }
+        else
+        {
+            NewOuterRsp.u = uPtrFrame.pu64[0];
+            uNewOuterSs   = uPtrFrame.pu16[4];
+        }
+        rcStrict = iemMemStackPopDoneSpecial(pVCpu, uPtrFrame.pv);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* extremely likely */ }
+        else
+            return rcStrict;
+
+        /* Check for NULL stack selector (invalid in ring-3 and non-long mode)
+           and read the selector. */
+        IEMSELDESC DescSs;
+        if (!(uNewOuterSs & X86_SEL_MASK_OFF_RPL))
+        {
+            if (   !DescCs.Legacy.Gen.u1Long
+                || (uNewOuterSs & X86_SEL_RPL) == 3)
+            {
+                Log(("retf %04x:%08RX64 %04x:%08RX64 -> invalid stack selector, #GP\n",
+                     uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+            /** @todo Testcase: Return far to ring-1 or ring-2 with SS=0. */
+            iemMemFakeStackSelDesc(&DescSs, (uNewOuterSs & X86_SEL_RPL));
+        }
+        else
+        {
+            /* Fetch the descriptor for the new stack segment. */
+            rcStrict = iemMemFetchSelDesc(pVCpu, &DescSs, uNewOuterSs, X86_XCPT_GP);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+
+        /* Check that RPL of stack and code selectors match. */
+        if ((uNewCs & X86_SEL_RPL) != (uNewOuterSs & X86_SEL_RPL))
+        {
+            Log(("retf %04x:%08RX64 %04x:%08RX64 - SS.RPL != CS.RPL -> #GP(SS)\n", uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
+        }
+
+        /* Must be a writable data segment. */
+        if (   !DescSs.Legacy.Gen.u1DescType
+            || (DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
+            || !(DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
+        {
+            Log(("retf %04x:%08RX64 %04x:%08RX64 - SS not a writable data segment (u1DescType=%u u4Type=%#x) -> #GP(SS).\n",
+                 uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u, DescSs.Legacy.Gen.u1DescType, DescSs.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
+        }
+
+        /* L vs D. (Not mentioned by intel.) */
+        if (   DescSs.Legacy.Gen.u1Long /** @todo Testcase: far return to a stack selector with both L and D set. */
+            && DescSs.Legacy.Gen.u1DefBig
+            && IEM_IS_LONG_MODE(pVCpu))
+        {
+            Log(("retf %04x:%08RX64 %04x:%08RX64 - SS has both L & D set -> #GP(SS).\n",
+                 uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
+        }
+
+        /* DPL/RPL/CPL checks. */
+        if (DescSs.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
+        {
+            Log(("retf %04x:%08RX64 %04x:%08RX64 - SS.DPL(%u) != CS.RPL (%u) -> #GP(SS).\n",
+                 uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u, DescSs.Legacy.Gen.u2Dpl, uNewCs & X86_SEL_RPL));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewOuterSs);
+        }
+
+        /* Is it there? */
+        if (!DescSs.Legacy.Gen.u1Present)
+        {
+            Log(("retf %04x:%08RX64 %04x:%08RX64 - SS not present -> #NP(SS).\n", uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u));
+            return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
+        }
+
+        /* Calc SS limit.*/
+        uint32_t cbLimitSs = X86DESC_LIMIT_G(&DescSs.Legacy);
+
+        /* Is RIP canonical or within CS.limit? */
+        uint64_t u64Base;
+        uint32_t cbLimitCs = X86DESC_LIMIT_G(&DescCs.Legacy);
+
+        /** @todo Testcase: Is this correct? */
+        if (   DescCs.Legacy.Gen.u1Long
+            && IEM_IS_LONG_MODE(pVCpu) )
+        {
+            if (!IEM_IS_CANONICAL(uNewRip))
+            {
+                Log(("retf %04x:%08RX64 %04x:%08RX64 - not canonical -> #GP.\n", uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u));
+                return iemRaiseNotCanonical(pVCpu);
+            }
+            u64Base = 0;
+        }
+        else
+        {
+            if (uNewRip > cbLimitCs)
+            {
+                Log(("retf %04x:%08RX64 %04x:%08RX64 - out of bounds (%#x)-> #GP(CS).\n",
+                     uNewCs, uNewRip, uNewOuterSs, NewOuterRsp.u, cbLimitCs));
+                /** @todo Intel says this is \#GP(0)! */
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+            }
+            u64Base = X86DESC_BASE(&DescCs.Legacy);
+        }
+
+        /*
+         * Now set the accessed bit before
+         * writing the return address to the stack and committing the result into
+         * CS, CSHID and RIP.
+         */
+        /** @todo Testcase: Need to check WHEN exactly the CS accessed bit is set. */
+        if (!(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            /** @todo check what VT-x and AMD-V does. */
+            DescCs.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+        /** @todo Testcase: Need to check WHEN exactly the SS accessed bit is set. */
+        if (!(DescSs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewOuterSs);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            /** @todo check what VT-x and AMD-V does. */
+            DescSs.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* commit */
+        if (enmEffOpSize == IEMMODE_16BIT)
+            pVCpu->cpum.GstCtx.rip           = uNewRip & UINT16_MAX; /** @todo Testcase: When exactly does this occur? With call it happens prior to the limit check according to Intel... */
+        else
+            pVCpu->cpum.GstCtx.rip           = uNewRip;
+        pVCpu->cpum.GstCtx.cs.Sel            = uNewCs;
+        pVCpu->cpum.GstCtx.cs.ValidSel       = uNewCs;
+        pVCpu->cpum.GstCtx.cs.fFlags         = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.Attr.u         = X86DESC_GET_HID_ATTR(&DescCs.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit       = cbLimitCs;
+        pVCpu->cpum.GstCtx.cs.u64Base        = u64Base;
+        pVCpu->iem.s.enmCpuMode              = iemCalcCpuMode(pVCpu);
+        pVCpu->cpum.GstCtx.ss.Sel            = uNewOuterSs;
+        pVCpu->cpum.GstCtx.ss.ValidSel       = uNewOuterSs;
+        pVCpu->cpum.GstCtx.ss.fFlags         = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.Attr.u         = X86DESC_GET_HID_ATTR(&DescSs.Legacy);
+        pVCpu->cpum.GstCtx.ss.u32Limit       = cbLimitSs;
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+            pVCpu->cpum.GstCtx.ss.u64Base    = 0;
+        else
+            pVCpu->cpum.GstCtx.ss.u64Base    = X86DESC_BASE(&DescSs.Legacy);
+        if (cbPop)
+            iemRegAddToRspEx(pVCpu, &NewOuterRsp, cbPop);
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+            pVCpu->cpum.GstCtx.rsp           = NewOuterRsp.u;
+        else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            pVCpu->cpum.GstCtx.rsp           = (uint32_t)NewOuterRsp.u;
+        else
+            pVCpu->cpum.GstCtx.sp            = (uint16_t)NewOuterRsp.u;
+
+        pVCpu->iem.s.uCpl                    = (uNewCs & X86_SEL_RPL);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.ds);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.es);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.fs);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.gs);
+
+        /** @todo check if the hidden bits are loaded correctly for 64-bit
+         *        mode. */
+    }
+    /*
+     * Return to the same privilege level
+     */
+    else
+    {
+        /* Limit / canonical check. */
+        uint64_t u64Base;
+        uint32_t cbLimitCs = X86DESC_LIMIT_G(&DescCs.Legacy);
+
+        /** @todo Testcase: Is this correct? */
+        if (   DescCs.Legacy.Gen.u1Long
+            && IEM_IS_LONG_MODE(pVCpu) )
+        {
+            if (!IEM_IS_CANONICAL(uNewRip))
+            {
+                Log(("retf %04x:%08RX64 - not canonical -> #GP\n", uNewCs, uNewRip));
+                return iemRaiseNotCanonical(pVCpu);
+            }
+            u64Base = 0;
+        }
+        else
+        {
+            if (uNewRip > cbLimitCs)
+            {
+                Log(("retf %04x:%08RX64 -> out of bounds (%#x)\n", uNewCs, uNewRip, cbLimitCs));
+                /** @todo Intel says this is \#GP(0)! */
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+            }
+            u64Base = X86DESC_BASE(&DescCs.Legacy);
+        }
+
+        /*
+         * Now set the accessed bit before
+         * writing the return address to the stack and committing the result into
+         * CS, CSHID and RIP.
+         */
+        /** @todo Testcase: Need to check WHEN exactly the accessed bit is set. */
+        if (!(DescCs.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            /** @todo check what VT-x and AMD-V does. */
+            DescCs.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* commit */
+        if (cbPop)
+            iemRegAddToRspEx(pVCpu, &NewRsp, cbPop);
+        if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            pVCpu->cpum.GstCtx.sp        = (uint16_t)NewRsp.u;
+        else
+            pVCpu->cpum.GstCtx.rsp       = NewRsp.u;
+        if (enmEffOpSize == IEMMODE_16BIT)
+            pVCpu->cpum.GstCtx.rip       = uNewRip & UINT16_MAX; /** @todo Testcase: When exactly does this occur? With call it happens prior to the limit check according to Intel... */
+        else
+            pVCpu->cpum.GstCtx.rip       = uNewRip;
+        pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESC_GET_HID_ATTR(&DescCs.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit   = cbLimitCs;
+        pVCpu->cpum.GstCtx.cs.u64Base    = u64Base;
+        /** @todo check if the hidden bits are loaded correctly for 64-bit
+         *        mode.  */
+        pVCpu->iem.s.enmCpuMode          = iemCalcCpuMode(pVCpu);
+    }
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo use light flush for same privlege? */
+
+    return iemRegFinishClearingRF(pVCpu);
+}
+
+
+/**
+ * Implements retn and retn imm16.
+ *
+ * We're doing this in C because of the \#GP that might be raised if the popped
+ * program counter is out of bounds.
+ *
+ * The hope with this forced inline worker function, is that the compiler will
+ * be clever enough to eliminate unused code for the constant enmEffOpSize and
+ * maybe cbPop parameters.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling thread.
+ * @param   cbInstr         The current instruction length.
+ * @param   enmEffOpSize    The effective operand size.  This is constant.
+ * @param   cbPop           The amount of arguments to pop from the stack
+ *                          (bytes).  This can be constant (zero).
+ */
+DECL_FORCE_INLINE(VBOXSTRICTRC) iemCImpl_ReturnNearCommon(PVMCPUCC pVCpu, uint8_t cbInstr, IEMMODE enmEffOpSize, uint16_t cbPop)
+{
+    /* Fetch the RSP from the stack. */
+    VBOXSTRICTRC    rcStrict;
+    RTUINT64U       NewRip;
+    RTUINT64U       NewRsp;
+    NewRsp.u = pVCpu->cpum.GstCtx.rsp;
+
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            NewRip.u = 0;
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRip.Words.w0, &NewRsp);
+            break;
+        case IEMMODE_32BIT:
+            NewRip.u = 0;
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRip.DWords.dw0, &NewRsp);
+            break;
+        case IEMMODE_64BIT:
+            rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRip.u, &NewRsp);
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Check the new RSP before loading it. */
+    /** @todo Should test this as the intel+amd pseudo code doesn't mention half
+     *        of it.  The canonical test is performed here and for call. */
+    if (enmEffOpSize != IEMMODE_64BIT)
+    {
+        if (RT_LIKELY(NewRip.DWords.dw0 <= pVCpu->cpum.GstCtx.cs.u32Limit))
+        { /* likely */ }
+        else
+        {
+            Log(("retn newrip=%llx - out of bounds (%x) -> #GP\n", NewRip.u, pVCpu->cpum.GstCtx.cs.u32Limit));
+            return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+        }
+    }
+    else
+    {
+        if (RT_LIKELY(IEM_IS_CANONICAL(NewRip.u)))
+        { /* likely */ }
+        else
+        {
+            Log(("retn newrip=%llx - not canonical -> #GP\n", NewRip.u));
+            return iemRaiseNotCanonical(pVCpu);
+        }
+    }
+
+    /* Apply cbPop */
+    if (cbPop)
+        iemRegAddToRspEx(pVCpu, &NewRsp, cbPop);
+
+    /* Commit it. */
+    pVCpu->cpum.GstCtx.rip = NewRip.u;
+    pVCpu->cpum.GstCtx.rsp = NewRsp.u;
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo only need a light flush here, don't we?  We don't really need any flushing... */
+    RT_NOREF(cbInstr);
+
+    return iemRegFinishClearingRF(pVCpu);
+}
+
+
+/**
+ * Implements retn imm16 with 16-bit effective operand size.
+ *
+ * @param   cbPop The amount of arguments to pop from the stack (bytes).
+ */
+IEM_CIMPL_DEF_1(iemCImpl_retn_iw_16, uint16_t, cbPop)
+{
+    return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_16BIT, cbPop);
+}
+
+
+/**
+ * Implements retn imm16 with 32-bit effective operand size.
+ *
+ * @param   cbPop The amount of arguments to pop from the stack (bytes).
+ */
+IEM_CIMPL_DEF_1(iemCImpl_retn_iw_32, uint16_t, cbPop)
+{
+    return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_32BIT, cbPop);
+}
+
+
+/**
+ * Implements retn imm16 with 64-bit effective operand size.
+ *
+ * @param   cbPop The amount of arguments to pop from the stack (bytes).
+ */
+IEM_CIMPL_DEF_1(iemCImpl_retn_iw_64, uint16_t, cbPop)
+{
+    return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_64BIT, cbPop);
+}
+
+
+/**
+ * Implements retn with 16-bit effective operand size.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_retn_16)
+{
+    return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_16BIT, 0);
+}
+
+
+/**
+ * Implements retn with 32-bit effective operand size.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_retn_32)
+{
+    return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_32BIT, 0);
+}
+
+
+/**
+ * Implements retn with 64-bit effective operand size.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_retn_64)
+{
+    return iemCImpl_ReturnNearCommon(pVCpu, cbInstr, IEMMODE_64BIT, 0);
+}
+
+
+/**
+ * Implements enter.
+ *
+ * We're doing this in C because the instruction is insane, even for the
+ * u8NestingLevel=0 case dealing with the stack is tedious.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   cbFrame         Frame size.
+ * @param   cParameters     Frame parameter count.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_enter, IEMMODE, enmEffOpSize, uint16_t, cbFrame, uint8_t, cParameters)
+{
+    /* Push RBP, saving the old value in TmpRbp. */
+    RTUINT64U       NewRsp; NewRsp.u = pVCpu->cpum.GstCtx.rsp;
+    RTUINT64U       TmpRbp; TmpRbp.u = pVCpu->cpum.GstCtx.rbp;
+    RTUINT64U       NewRbp;
+    VBOXSTRICTRC    rcStrict;
+    if (enmEffOpSize == IEMMODE_64BIT)
+    {
+        rcStrict = iemMemStackPushU64Ex(pVCpu, TmpRbp.u, &NewRsp);
+        NewRbp = NewRsp;
+    }
+    else if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        rcStrict = iemMemStackPushU32Ex(pVCpu, TmpRbp.DWords.dw0, &NewRsp);
+        NewRbp = NewRsp;
+    }
+    else
+    {
+        rcStrict = iemMemStackPushU16Ex(pVCpu, TmpRbp.Words.w0, &NewRsp);
+        NewRbp = TmpRbp;
+        NewRbp.Words.w0 = NewRsp.Words.w0;
+    }
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Copy the parameters (aka nesting levels by Intel). */
+    cParameters &= 0x1f;
+    if (cParameters > 0)
+    {
+        switch (enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+                    TmpRbp.DWords.dw0 -= 2;
+                else
+                    TmpRbp.Words.w0   -= 2;
+                do
+                {
+                    uint16_t u16Tmp;
+                    rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Tmp, &TmpRbp);
+                    if (rcStrict != VINF_SUCCESS)
+                        break;
+                    rcStrict = iemMemStackPushU16Ex(pVCpu, u16Tmp, &NewRsp);
+                } while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
+                break;
+
+            case IEMMODE_32BIT:
+                if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+                    TmpRbp.DWords.dw0 -= 4;
+                else
+                    TmpRbp.Words.w0   -= 4;
+                do
+                {
+                    uint32_t u32Tmp;
+                    rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Tmp, &TmpRbp);
+                    if (rcStrict != VINF_SUCCESS)
+                        break;
+                    rcStrict = iemMemStackPushU32Ex(pVCpu, u32Tmp, &NewRsp);
+                } while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
+                break;
+
+            case IEMMODE_64BIT:
+                TmpRbp.u -= 8;
+                do
+                {
+                    uint64_t u64Tmp;
+                    rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Tmp, &TmpRbp);
+                    if (rcStrict != VINF_SUCCESS)
+                        break;
+                    rcStrict = iemMemStackPushU64Ex(pVCpu, u64Tmp, &NewRsp);
+                } while (--cParameters > 0 && rcStrict == VINF_SUCCESS);
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        if (rcStrict != VINF_SUCCESS)
+            return VINF_SUCCESS;
+
+        /* Push the new RBP */
+        if (enmEffOpSize == IEMMODE_64BIT)
+            rcStrict = iemMemStackPushU64Ex(pVCpu, NewRbp.u, &NewRsp);
+        else if (enmEffOpSize == IEMMODE_32BIT)
+            rcStrict = iemMemStackPushU32Ex(pVCpu, NewRbp.DWords.dw0, &NewRsp);
+        else
+            rcStrict = iemMemStackPushU16Ex(pVCpu, NewRbp.Words.w0, &NewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+    }
+
+    /* Recalc RSP. */
+    iemRegSubFromRspEx(pVCpu, &NewRsp, cbFrame);
+
+    /** @todo Should probe write access at the new RSP according to AMD. */
+    /** @todo Should handle accesses to the VMX APIC-access page. */
+
+    /* Commit it. */
+    pVCpu->cpum.GstCtx.rbp = NewRbp.u;
+    pVCpu->cpum.GstCtx.rsp = NewRsp.u;
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+
+/**
+ * Implements leave.
+ *
+ * We're doing this in C because messing with the stack registers is annoying
+ * since they depends on SS attributes.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_leave, IEMMODE, enmEffOpSize)
+{
+    /* Calculate the intermediate RSP from RBP and the stack attributes. */
+    RTUINT64U       NewRsp;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        NewRsp.u = pVCpu->cpum.GstCtx.rbp;
+    else if (pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+        NewRsp.u = pVCpu->cpum.GstCtx.ebp;
+    else
+    {
+        /** @todo Check that LEAVE actually preserve the high EBP bits. */
+        NewRsp.u = pVCpu->cpum.GstCtx.rsp;
+        NewRsp.Words.w0 = pVCpu->cpum.GstCtx.bp;
+    }
+
+    /* Pop RBP according to the operand size. */
+    VBOXSTRICTRC    rcStrict;
+    RTUINT64U       NewRbp;
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            NewRbp.u = pVCpu->cpum.GstCtx.rbp;
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &NewRbp.Words.w0, &NewRsp);
+            break;
+        case IEMMODE_32BIT:
+            NewRbp.u = 0;
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &NewRbp.DWords.dw0, &NewRsp);
+            break;
+        case IEMMODE_64BIT:
+            rcStrict = iemMemStackPopU64Ex(pVCpu, &NewRbp.u, &NewRsp);
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+
+    /* Commit it. */
+    pVCpu->cpum.GstCtx.rbp = NewRbp.u;
+    pVCpu->cpum.GstCtx.rsp = NewRsp.u;
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements int3 and int XX.
+ *
+ * @param   u8Int       The interrupt vector number.
+ * @param   enmInt      The int instruction type.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_int, uint8_t, u8Int, IEMINT, enmInt)
+{
+    Assert(pVCpu->iem.s.cXcptRecursions == 0);
+
+    /*
+     * We must check if this INT3 might belong to DBGF before raising a #BP.
+     */
+    if (u8Int == 3)
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        if (pVM->dbgf.ro.cEnabledInt3Breakpoints == 0)
+        { /* likely: No vbox debugger breakpoints */ }
+        else
+        {
+            VBOXSTRICTRC rcStrict = DBGFTrap03Handler(pVM, pVCpu, &pVCpu->cpum.GstCtx);
+            Log(("iemCImpl_int: DBGFTrap03Handler -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
+            if (rcStrict != VINF_EM_RAW_GUEST_TRAP)
+                return iemSetPassUpStatus(pVCpu, rcStrict);
+        }
+    }
+/** @todo single stepping   */
+    return iemRaiseXcptOrInt(pVCpu,
+                             cbInstr,
+                             u8Int,
+                             IEM_XCPT_FLAGS_T_SOFT_INT | enmInt,
+                             0,
+                             0);
+}
+
+
+/**
+ * Implements iret for real mode and V8086 mode.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_iret_real_v8086, IEMMODE, enmEffOpSize)
+{
+    X86EFLAGS Efl;
+    Efl.u = IEMMISC_GET_EFL(pVCpu);
+    NOREF(cbInstr);
+
+    /*
+     * iret throws an exception if VME isn't enabled.
+     */
+    if (   Efl.Bits.u1VM
+        && Efl.Bits.u2IOPL != 3
+        && !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME))
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    /*
+     * Do the stack bits, but don't commit RSP before everything checks
+     * out right.
+     */
+    Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
+    VBOXSTRICTRC    rcStrict;
+    RTCPTRUNION     uFrame;
+    uint16_t        uNewCs;
+    uint32_t        uNewEip;
+    uint32_t        uNewFlags;
+    uint64_t        uNewRsp;
+    if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 12, 1, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewEip    = uFrame.pu32[0];
+        if (uNewEip > UINT16_MAX)
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+
+        uNewCs     = (uint16_t)uFrame.pu32[1];
+        uNewFlags  = uFrame.pu32[2];
+        uNewFlags &= X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF
+                   | X86_EFL_TF | X86_EFL_IF | X86_EFL_DF | X86_EFL_OF | X86_EFL_IOPL | X86_EFL_NT
+                   | X86_EFL_RF /*| X86_EFL_VM*/ | X86_EFL_AC /*|X86_EFL_VIF*/ /*|X86_EFL_VIP*/
+                   | X86_EFL_ID;
+        if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
+            uNewFlags &= ~(X86_EFL_AC | X86_EFL_ID | X86_EFL_VIF | X86_EFL_VIP);
+        uNewFlags |= Efl.u & (X86_EFL_VM | X86_EFL_VIF | X86_EFL_VIP | X86_EFL_1);
+    }
+    else
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 6, 1, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewEip    = uFrame.pu16[0];
+        uNewCs     = uFrame.pu16[1];
+        uNewFlags  = uFrame.pu16[2];
+        uNewFlags &= X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF
+                   | X86_EFL_TF | X86_EFL_IF | X86_EFL_DF | X86_EFL_OF | X86_EFL_IOPL | X86_EFL_NT;
+        uNewFlags |= Efl.u & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF);
+        /** @todo The intel pseudo code does not indicate what happens to
+         *        reserved flags. We just ignore them. */
+        /* Ancient CPU adjustments: See iemCImpl_popf. */
+        if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_286)
+            uNewFlags &= ~(X86_EFL_NT | X86_EFL_IOPL);
+    }
+    rcStrict = iemMemStackPopDoneSpecial(pVCpu, uFrame.pv);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* extremely likely */ }
+    else
+        return rcStrict;
+
+    /** @todo Check how this is supposed to work if sp=0xfffe. */
+    Log7(("iemCImpl_iret_real_v8086: uNewCs=%#06x uNewRip=%#010x uNewFlags=%#x uNewRsp=%#18llx\n",
+          uNewCs, uNewEip, uNewFlags, uNewRsp));
+
+    /*
+     * Check the limit of the new EIP.
+     */
+    /** @todo Only the AMD pseudo code check the limit here, what's
+     *        right? */
+    if (uNewEip > pVCpu->cpum.GstCtx.cs.u32Limit)
+        return iemRaiseSelectorBounds(pVCpu, X86_SREG_CS, IEM_ACCESS_INSTRUCTION);
+
+    /*
+     * V8086 checks and flag adjustments
+     */
+    if (Efl.Bits.u1VM)
+    {
+        if (Efl.Bits.u2IOPL == 3)
+        {
+            /* Preserve IOPL and clear RF. */
+            uNewFlags &=        ~(X86_EFL_IOPL | X86_EFL_RF);
+            uNewFlags |= Efl.u & (X86_EFL_IOPL);
+        }
+        else if (   enmEffOpSize == IEMMODE_16BIT
+                 && (   !(uNewFlags & X86_EFL_IF)
+                     || !Efl.Bits.u1VIP )
+                 && !(uNewFlags & X86_EFL_TF)   )
+        {
+            /* Move IF to VIF, clear RF and preserve IF and IOPL.*/
+            uNewFlags &= ~X86_EFL_VIF;
+            uNewFlags |= (uNewFlags & X86_EFL_IF) << (19 - 9);
+            uNewFlags &=        ~(X86_EFL_IF | X86_EFL_IOPL | X86_EFL_RF);
+            uNewFlags |= Efl.u & (X86_EFL_IF | X86_EFL_IOPL);
+        }
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        Log7(("iemCImpl_iret_real_v8086: u1VM=1: adjusted uNewFlags=%#x\n", uNewFlags));
+    }
+
+    /*
+     * Commit the operation.
+     */
+#ifdef DBGFTRACE_ENABLED
+    RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/rm %04x:%04x -> %04x:%04x %x %04llx",
+                      pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewRsp);
+#endif
+    pVCpu->cpum.GstCtx.rsp           = uNewRsp;
+    pVCpu->cpum.GstCtx.rip           = uNewEip;
+    pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+    pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+    pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.u64Base    = (uint32_t)uNewCs << 4;
+    /** @todo do we load attribs and limit as well? */
+    Assert(uNewFlags & X86_EFL_1);
+    IEMMISC_SET_EFL(pVCpu, uNewFlags);
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo can do light flush in real mode at least */
+
+/** @todo single stepping   */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads a segment register when entering V8086 mode.
+ *
+ * @param   pSReg           The segment register.
+ * @param   uSeg            The segment to load.
+ */
+static void iemCImplCommonV8086LoadSeg(PCPUMSELREG pSReg, uint16_t uSeg)
+{
+    pSReg->Sel        = uSeg;
+    pSReg->ValidSel   = uSeg;
+    pSReg->fFlags     = CPUMSELREG_FLAGS_VALID;
+    pSReg->u64Base    = (uint32_t)uSeg << 4;
+    pSReg->u32Limit   = 0xffff;
+    pSReg->Attr.u     = X86_SEL_TYPE_RW_ACC | RT_BIT(4) /*!sys*/ | RT_BIT(7) /*P*/ | (3 /*DPL*/ << 5); /* VT-x wants 0xf3 */
+    /** @todo Testcase: Check if VT-x really needs this and what it does itself when
+     *        IRET'ing to V8086. */
+}
+
+
+/**
+ * Implements iret for protected mode returning to V8086 mode.
+ *
+ * @param   uNewEip         The new EIP.
+ * @param   uNewCs          The new CS.
+ * @param   uNewFlags       The new EFLAGS.
+ * @param   uNewRsp         The RSP after the initial IRET frame.
+ *
+ * @note    This can only be a 32-bit iret du to the X86_EFL_VM position.
+ */
+IEM_CIMPL_DEF_4(iemCImpl_iret_prot_v8086, uint32_t, uNewEip, uint16_t, uNewCs, uint32_t, uNewFlags, uint64_t, uNewRsp)
+{
+    RT_NOREF_PV(cbInstr);
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
+
+    /*
+     * Pop the V8086 specific frame bits off the stack.
+     */
+    VBOXSTRICTRC    rcStrict;
+    RTCPTRUNION     uFrame;
+    rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0 /*off*/, 24 /*cbMem*/, &uFrame.pv, uNewRsp);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    uint32_t uNewEsp = uFrame.pu32[0];
+    uint16_t uNewSs  = uFrame.pu32[1];
+    uint16_t uNewEs  = uFrame.pu32[2];
+    uint16_t uNewDs  = uFrame.pu32[3];
+    uint16_t uNewFs  = uFrame.pu32[4];
+    uint16_t uNewGs  = uFrame.pu32[5];
+    rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uFrame.pv, IEM_ACCESS_STACK_R); /* don't use iemMemStackPopCommitSpecial here. */
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Commit the operation.
+     */
+    uNewFlags &= X86_EFL_LIVE_MASK;
+    uNewFlags |= X86_EFL_RA1_MASK;
+#ifdef DBGFTRACE_ENABLED
+    RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/p/v %04x:%08x -> %04x:%04x %x %04x:%04x",
+                      pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewSs, uNewEsp);
+#endif
+    Log7(("iemCImpl_iret_prot_v8086: %04x:%08x -> %04x:%04x %x %04x:%04x\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, uNewCs, uNewEip, uNewFlags, uNewSs, uNewEsp));
+
+    IEMMISC_SET_EFL(pVCpu, uNewFlags);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.cs, uNewCs);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.ss, uNewSs);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.es, uNewEs);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.ds, uNewDs);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.fs, uNewFs);
+    iemCImplCommonV8086LoadSeg(&pVCpu->cpum.GstCtx.gs, uNewGs);
+    pVCpu->cpum.GstCtx.rip      = (uint16_t)uNewEip;
+    pVCpu->cpum.GstCtx.rsp      = uNewEsp; /** @todo check this out! */
+    pVCpu->iem.s.uCpl  = 3;
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
+
+/** @todo single stepping   */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements iret for protected mode returning via a nested task.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_iret_prot_NestedTask, IEMMODE, enmEffOpSize)
+{
+    Log7(("iemCImpl_iret_prot_NestedTask:\n"));
+#ifndef IEM_IMPLEMENTS_TASKSWITCH
+    IEM_RETURN_ASPECT_NOT_IMPLEMENTED();
+#else
+    RT_NOREF_PV(enmEffOpSize);
+
+    /*
+     * Read the segment selector in the link-field of the current TSS.
+     */
+    RTSEL        uSelRet;
+    VBOXSTRICTRC rcStrict = iemMemFetchSysU16(pVCpu, &uSelRet, UINT8_MAX, pVCpu->cpum.GstCtx.tr.u64Base);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Fetch the returning task's TSS descriptor from the GDT.
+     */
+    if (uSelRet & X86_SEL_LDT)
+    {
+        Log(("iret_prot_NestedTask TSS not in LDT. uSelRet=%04x -> #TS\n", uSelRet));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet);
+    }
+
+    IEMSELDESC TssDesc;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &TssDesc, uSelRet, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    if (TssDesc.Legacy.Gate.u1DescType)
+    {
+        Log(("iret_prot_NestedTask Invalid TSS type. uSelRet=%04x -> #TS\n", uSelRet));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
+    }
+
+    if (   TssDesc.Legacy.Gate.u4Type != X86_SEL_TYPE_SYS_286_TSS_BUSY
+        && TssDesc.Legacy.Gate.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
+    {
+        Log(("iret_prot_NestedTask TSS is not busy. uSelRet=%04x DescType=%#x -> #TS\n", uSelRet, TssDesc.Legacy.Gate.u4Type));
+        return iemRaiseTaskSwitchFaultBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
+    }
+
+    if (!TssDesc.Legacy.Gate.u1Present)
+    {
+        Log(("iret_prot_NestedTask TSS is not present. uSelRet=%04x -> #NP\n", uSelRet));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uSelRet & X86_SEL_MASK_OFF_RPL);
+    }
+
+    uint32_t uNextEip = pVCpu->cpum.GstCtx.eip + cbInstr;
+    return iemTaskSwitch(pVCpu, IEMTASKSWITCH_IRET, uNextEip, 0 /* fFlags */, 0 /* uErr */,
+                         0 /* uCr2 */, uSelRet, &TssDesc);
+#endif
+}
+
+
+/**
+ * Implements iret for protected mode
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_iret_prot, IEMMODE, enmEffOpSize)
+{
+    NOREF(cbInstr);
+    Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
+
+    /*
+     * Nested task return.
+     */
+    if (pVCpu->cpum.GstCtx.eflags.Bits.u1NT)
+        return IEM_CIMPL_CALL_1(iemCImpl_iret_prot_NestedTask, enmEffOpSize);
+
+    /*
+     * Normal return.
+     *
+     * Do the stack bits, but don't commit RSP before everything checks
+     * out right.
+     */
+    Assert(enmEffOpSize == IEMMODE_32BIT || enmEffOpSize == IEMMODE_16BIT);
+    VBOXSTRICTRC    rcStrict;
+    RTCPTRUNION     uFrame;
+    uint16_t        uNewCs;
+    uint32_t        uNewEip;
+    uint32_t        uNewFlags;
+    uint64_t        uNewRsp;
+    if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 12, 3, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewEip    = uFrame.pu32[0];
+        uNewCs     = (uint16_t)uFrame.pu32[1];
+        uNewFlags  = uFrame.pu32[2];
+    }
+    else
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 6, 1, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewEip    = uFrame.pu16[0];
+        uNewCs     = uFrame.pu16[1];
+        uNewFlags  = uFrame.pu16[2];
+    }
+    rcStrict = iemMemStackPopDoneSpecial(pVCpu, (void *)uFrame.pv); /* don't use iemMemStackPopCommitSpecial here. */
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* extremely likely */ }
+    else
+        return rcStrict;
+    Log7(("iemCImpl_iret_prot: uNewCs=%#06x uNewEip=%#010x uNewFlags=%#x uNewRsp=%#18llx uCpl=%u\n", uNewCs, uNewEip, uNewFlags, uNewRsp, pVCpu->iem.s.uCpl));
+
+    /*
+     * We're hopefully not returning to V8086 mode...
+     */
+    if (   (uNewFlags & X86_EFL_VM)
+        && pVCpu->iem.s.uCpl == 0)
+    {
+        Assert(enmEffOpSize == IEMMODE_32BIT);
+        return IEM_CIMPL_CALL_4(iemCImpl_iret_prot_v8086, uNewEip, uNewCs, uNewFlags, uNewRsp);
+    }
+
+    /*
+     * Protected mode.
+     */
+    /* Read the CS descriptor. */
+    if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("iret %04x:%08x -> invalid CS selector, #GP(0)\n", uNewCs, uNewEip));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    IEMSELDESC DescCS;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCs, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iret %04x:%08x - rcStrict=%Rrc when fetching CS\n", uNewCs, uNewEip, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /* Must be a code descriptor. */
+    if (!DescCS.Legacy.Gen.u1DescType)
+    {
+        Log(("iret %04x:%08x - CS is system segment (%#x) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+    {
+        Log(("iret %04x:%08x - not code segment (%#x) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* Privilege checks. */
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF))
+    {
+        if ((uNewCs & X86_SEL_RPL) != DescCS.Legacy.Gen.u2Dpl)
+        {
+            Log(("iret %04x:%08x - RPL != DPL (%d) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u2Dpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+        }
+    }
+    else if ((uNewCs & X86_SEL_RPL) < DescCS.Legacy.Gen.u2Dpl)
+    {
+        Log(("iret %04x:%08x - RPL < DPL (%d) -> #GP\n", uNewCs, uNewEip, DescCS.Legacy.Gen.u2Dpl));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+    if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl)
+    {
+        Log(("iret %04x:%08x - RPL < CPL (%d) -> #GP\n", uNewCs, uNewEip, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* Present? */
+    if (!DescCS.Legacy.Gen.u1Present)
+    {
+        Log(("iret %04x:%08x - CS not present -> #NP\n", uNewCs, uNewEip));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
+    }
+
+    uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
+
+    /*
+     * Return to outer level?
+     */
+    if ((uNewCs & X86_SEL_RPL) != pVCpu->iem.s.uCpl)
+    {
+        uint16_t    uNewSS;
+        uint32_t    uNewESP;
+        if (enmEffOpSize == IEMMODE_32BIT)
+        {
+            rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0/*off*/, 8 /*cbMem*/, &uFrame.pv, uNewRsp);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+/** @todo We might be popping a 32-bit ESP from the IRET frame, but whether
+ *        16-bit or 32-bit are being loaded into SP depends on the D/B
+ *        bit of the popped SS selector it turns out. */
+            uNewESP = uFrame.pu32[0];
+            uNewSS  = (uint16_t)uFrame.pu32[1];
+        }
+        else
+        {
+            rcStrict = iemMemStackPopContinueSpecial(pVCpu, 0 /*off*/, 4 /*cbMem*/, &uFrame.pv, uNewRsp);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            uNewESP = uFrame.pu16[0];
+            uNewSS  = uFrame.pu16[1];
+        }
+        rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uFrame.pv, IEM_ACCESS_STACK_R);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        Log7(("iemCImpl_iret_prot: uNewSS=%#06x uNewESP=%#010x\n", uNewSS, uNewESP));
+
+        /* Read the SS descriptor. */
+        if (!(uNewSS & X86_SEL_MASK_OFF_RPL))
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> invalid SS selector, #GP(0)\n", uNewCs, uNewEip, uNewSS, uNewESP));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        IEMSELDESC DescSS;
+        rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSS, X86_XCPT_GP); /** @todo Correct exception? */
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iret %04x:%08x/%04x:%08x - %Rrc when fetching SS\n",
+                 uNewCs, uNewEip, uNewSS, uNewESP, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /* Privilege checks. */
+        if ((uNewSS & X86_SEL_RPL) != (uNewCs & X86_SEL_RPL))
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> SS.RPL != CS.RPL -> #GP\n", uNewCs, uNewEip, uNewSS, uNewESP));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
+        }
+        if (DescSS.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> SS.DPL (%d) != CS.RPL -> #GP\n",
+                 uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u2Dpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
+        }
+
+        /* Must be a writeable data segment descriptor. */
+        if (!DescSS.Legacy.Gen.u1DescType)
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> SS is system segment (%#x) -> #GP\n",
+                 uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
+        }
+        if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
+        {
+            Log(("iret %04x:%08x/%04x:%08x - not writable data segment (%#x) -> #GP\n",
+                 uNewCs, uNewEip, uNewSS, uNewESP, DescSS.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSS);
+        }
+
+        /* Present? */
+        if (!DescSS.Legacy.Gen.u1Present)
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> SS not present -> #SS\n", uNewCs, uNewEip, uNewSS, uNewESP));
+            return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSS);
+        }
+
+        uint32_t cbLimitSs = X86DESC_LIMIT_G(&DescSS.Legacy);
+
+        /* Check EIP. */
+        if (uNewEip > cbLimitCS)
+        {
+            Log(("iret %04x:%08x/%04x:%08x -> EIP is out of bounds (%#x) -> #GP(0)\n",
+                 uNewCs, uNewEip, uNewSS, uNewESP, cbLimitCS));
+            /** @todo Which is it, \#GP(0) or \#GP(sel)? */
+            return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
+        }
+
+        /*
+         * Commit the changes, marking CS and SS accessed first since
+         * that may fail.
+         */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+        if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSS);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        uint32_t fEFlagsMask = X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF
+                             | X86_EFL_TF | X86_EFL_DF | X86_EFL_OF | X86_EFL_NT;
+        if (enmEffOpSize != IEMMODE_16BIT)
+            fEFlagsMask |= X86_EFL_RF | X86_EFL_AC | X86_EFL_ID;
+        if (pVCpu->iem.s.uCpl == 0)
+            fEFlagsMask |= X86_EFL_IF | X86_EFL_IOPL | X86_EFL_VIF | X86_EFL_VIP; /* VM is 0 */
+        else if (pVCpu->iem.s.uCpl <= pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL)
+            fEFlagsMask |= X86_EFL_IF;
+        if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
+            fEFlagsMask &= ~(X86_EFL_AC | X86_EFL_ID | X86_EFL_VIF | X86_EFL_VIP);
+        uint32_t fEFlagsNew = IEMMISC_GET_EFL(pVCpu);
+        fEFlagsNew         &= ~fEFlagsMask;
+        fEFlagsNew         |= uNewFlags & fEFlagsMask;
+#ifdef DBGFTRACE_ENABLED
+        RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%up%u %04x:%08x -> %04x:%04x %x %04x:%04x",
+                          pVCpu->iem.s.uCpl, uNewCs & X86_SEL_RPL, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
+                          uNewCs, uNewEip, uNewFlags,  uNewSS, uNewESP);
+#endif
+
+        IEMMISC_SET_EFL(pVCpu, fEFlagsNew);
+        pVCpu->cpum.GstCtx.rip           = uNewEip;
+        pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit   = cbLimitCS;
+        pVCpu->cpum.GstCtx.cs.u64Base    = X86DESC_BASE(&DescCS.Legacy);
+        pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+
+        pVCpu->cpum.GstCtx.ss.Sel        = uNewSS;
+        pVCpu->cpum.GstCtx.ss.ValidSel   = uNewSS;
+        pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
+        pVCpu->cpum.GstCtx.ss.u32Limit   = cbLimitSs;
+        pVCpu->cpum.GstCtx.ss.u64Base    = X86DESC_BASE(&DescSS.Legacy);
+        if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            pVCpu->cpum.GstCtx.sp        = (uint16_t)uNewESP;
+        else
+            pVCpu->cpum.GstCtx.rsp       = uNewESP;
+
+        pVCpu->iem.s.uCpl       = uNewCs & X86_SEL_RPL;
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.ds);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.es);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.fs);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCs & X86_SEL_RPL, &pVCpu->cpum.GstCtx.gs);
+
+        /* Done! */
+
+    }
+    /*
+     * Return to the same level.
+     */
+    else
+    {
+        /* Check EIP. */
+        if (uNewEip > cbLimitCS)
+        {
+            Log(("iret %04x:%08x - EIP is out of bounds (%#x) -> #GP(0)\n", uNewCs, uNewEip, cbLimitCS));
+            /** @todo Which is it, \#GP(0) or \#GP(sel)? */
+            return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
+        }
+
+        /*
+         * Commit the changes, marking CS first since it may fail.
+         */
+        if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        X86EFLAGS NewEfl;
+        NewEfl.u = IEMMISC_GET_EFL(pVCpu);
+        uint32_t fEFlagsMask = X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF
+                             | X86_EFL_TF | X86_EFL_DF | X86_EFL_OF | X86_EFL_NT;
+        if (enmEffOpSize != IEMMODE_16BIT)
+            fEFlagsMask |= X86_EFL_RF | X86_EFL_AC | X86_EFL_ID;
+        if (pVCpu->iem.s.uCpl == 0)
+            fEFlagsMask |= X86_EFL_IF | X86_EFL_IOPL | X86_EFL_VIF | X86_EFL_VIP; /* VM is 0 */
+        else if (pVCpu->iem.s.uCpl <= NewEfl.Bits.u2IOPL)
+            fEFlagsMask |= X86_EFL_IF;
+        if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
+            fEFlagsMask &= ~(X86_EFL_AC | X86_EFL_ID | X86_EFL_VIF | X86_EFL_VIP);
+        NewEfl.u           &= ~fEFlagsMask;
+        NewEfl.u           |= fEFlagsMask & uNewFlags;
+#ifdef DBGFTRACE_ENABLED
+        RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%up %04x:%08x -> %04x:%04x %x %04x:%04llx",
+                          pVCpu->iem.s.uCpl, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip,
+                          uNewCs, uNewEip, uNewFlags, pVCpu->cpum.GstCtx.ss.Sel, uNewRsp);
+#endif
+
+        IEMMISC_SET_EFL(pVCpu, NewEfl.u);
+        pVCpu->cpum.GstCtx.rip           = uNewEip;
+        pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+        pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+        pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+        pVCpu->cpum.GstCtx.cs.u32Limit   = cbLimitCS;
+        pVCpu->cpum.GstCtx.cs.u64Base    = X86DESC_BASE(&DescCS.Legacy);
+        pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+        if (!pVCpu->cpum.GstCtx.ss.Attr.n.u1DefBig)
+            pVCpu->cpum.GstCtx.sp        = (uint16_t)uNewRsp;
+        else
+            pVCpu->cpum.GstCtx.rsp       = uNewRsp;
+        /* Done! */
+    }
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo may light flush if same ring? */
+
+/** @todo single stepping   */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements iret for long mode
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_iret_64bit, IEMMODE, enmEffOpSize)
+{
+    NOREF(cbInstr);
+
+    /*
+     * Nested task return is not supported in long mode.
+     */
+    if (pVCpu->cpum.GstCtx.eflags.Bits.u1NT)
+    {
+        Log(("iretq with NT=1 (eflags=%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.eflags.u));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Normal return.
+     *
+     * Do the stack bits, but don't commit RSP before everything checks
+     * out right.
+     */
+    VBOXSTRICTRC    rcStrict;
+    RTCPTRUNION     uFrame;
+    uint64_t        uNewRip;
+    uint16_t        uNewCs;
+    uint16_t        uNewSs;
+    uint32_t        uNewFlags;
+    uint64_t        uNewRsp;
+    if (enmEffOpSize == IEMMODE_64BIT)
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*8, 7, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewRip    = uFrame.pu64[0];
+        uNewCs     = (uint16_t)uFrame.pu64[1];
+        uNewFlags  = (uint32_t)uFrame.pu64[2];
+        uNewRsp    = uFrame.pu64[3];
+        uNewSs     = (uint16_t)uFrame.pu64[4];
+    }
+    else if (enmEffOpSize == IEMMODE_32BIT)
+    {
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*4, 3, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewRip    = uFrame.pu32[0];
+        uNewCs     = (uint16_t)uFrame.pu32[1];
+        uNewFlags  = uFrame.pu32[2];
+        uNewRsp    = uFrame.pu32[3];
+        uNewSs     = (uint16_t)uFrame.pu32[4];
+    }
+    else
+    {
+        Assert(enmEffOpSize == IEMMODE_16BIT);
+        rcStrict = iemMemStackPopBeginSpecial(pVCpu, 5*2, 1, &uFrame.pv, &uNewRsp);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        uNewRip    = uFrame.pu16[0];
+        uNewCs     = uFrame.pu16[1];
+        uNewFlags  = uFrame.pu16[2];
+        uNewRsp    = uFrame.pu16[3];
+        uNewSs     = uFrame.pu16[4];
+    }
+    rcStrict = iemMemStackPopDoneSpecial(pVCpu, (void *)uFrame.pv); /* don't use iemMemStackPopCommitSpecial here. */
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* extremely like */ }
+    else
+        return rcStrict;
+    Log7(("iretq stack: cs:rip=%04x:%016RX64 rflags=%016RX64 ss:rsp=%04x:%016RX64\n", uNewCs, uNewRip, uNewFlags, uNewSs, uNewRsp));
+
+    /*
+     * Check stuff.
+     */
+    /* Read the CS descriptor. */
+    if (!(uNewCs & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("iret %04x:%016RX64/%04x:%016RX64 -> invalid CS selector, #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    IEMSELDESC DescCS;
+    rcStrict = iemMemFetchSelDesc(pVCpu, &DescCS, uNewCs, X86_XCPT_GP);
+    if (rcStrict != VINF_SUCCESS)
+    {
+        Log(("iret %04x:%016RX64/%04x:%016RX64 - rcStrict=%Rrc when fetching CS\n",
+             uNewCs, uNewRip, uNewSs, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
+        return rcStrict;
+    }
+
+    /* Must be a code descriptor. */
+    if (   !DescCS.Legacy.Gen.u1DescType
+        || !(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE))
+    {
+        Log(("iret %04x:%016RX64/%04x:%016RX64 - CS is not a code segment T=%u T=%#xu -> #GP\n",
+             uNewCs, uNewRip, uNewSs, uNewRsp, DescCS.Legacy.Gen.u1DescType, DescCS.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* Privilege checks. */
+    uint8_t const uNewCpl = uNewCs & X86_SEL_RPL;
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_CONF))
+    {
+        if ((uNewCs & X86_SEL_RPL) != DescCS.Legacy.Gen.u2Dpl)
+        {
+            Log(("iret %04x:%016RX64 - RPL != DPL (%d) -> #GP\n", uNewCs, uNewRip, DescCS.Legacy.Gen.u2Dpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+        }
+    }
+    else if ((uNewCs & X86_SEL_RPL) < DescCS.Legacy.Gen.u2Dpl)
+    {
+        Log(("iret %04x:%016RX64 - RPL < DPL (%d) -> #GP\n", uNewCs, uNewRip, DescCS.Legacy.Gen.u2Dpl));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+    if ((uNewCs & X86_SEL_RPL) < pVCpu->iem.s.uCpl)
+    {
+        Log(("iret %04x:%016RX64 - RPL < CPL (%d) -> #GP\n", uNewCs, uNewRip, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewCs);
+    }
+
+    /* Present? */
+    if (!DescCS.Legacy.Gen.u1Present)
+    {
+        Log(("iret %04x:%016RX64/%04x:%016RX64 - CS not present -> #NP\n", uNewCs, uNewRip, uNewSs, uNewRsp));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewCs);
+    }
+
+    uint32_t cbLimitCS = X86DESC_LIMIT_G(&DescCS.Legacy);
+
+    /* Read the SS descriptor. */
+    IEMSELDESC DescSS;
+    if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
+    {
+        if (   !DescCS.Legacy.Gen.u1Long
+            || DescCS.Legacy.Gen.u1DefBig /** @todo exactly how does iret (and others) behave with u1Long=1 and u1DefBig=1? \#GP(sel)? */
+            || uNewCpl > 2) /** @todo verify SS=0 impossible for ring-3. */
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> invalid SS selector, #GP(0)\n", uNewCs, uNewRip, uNewSs, uNewRsp));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        /* Make sure SS is sensible, marked as accessed etc. */
+        iemMemFakeStackSelDesc(&DescSS, (uNewSs & X86_SEL_RPL));
+    }
+    else
+    {
+        rcStrict = iemMemFetchSelDesc(pVCpu, &DescSS, uNewSs, X86_XCPT_GP); /** @todo Correct exception? */
+        if (rcStrict != VINF_SUCCESS)
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 - %Rrc when fetching SS\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /* Privilege checks. */
+    if ((uNewSs & X86_SEL_RPL) != (uNewCs & X86_SEL_RPL))
+    {
+        Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS.RPL != CS.RPL -> #GP\n", uNewCs, uNewRip, uNewSs, uNewRsp));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
+    }
+
+    uint32_t cbLimitSs;
+    if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
+        cbLimitSs = UINT32_MAX;
+    else
+    {
+        if (DescSS.Legacy.Gen.u2Dpl != (uNewCs & X86_SEL_RPL))
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS.DPL (%d) != CS.RPL -> #GP\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u2Dpl));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
+        }
+
+        /* Must be a writeable data segment descriptor. */
+        if (!DescSS.Legacy.Gen.u1DescType)
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS is system segment (%#x) -> #GP\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
+        }
+        if ((DescSS.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE)
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 - not writable data segment (%#x) -> #GP\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp, DescSS.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewSs);
+        }
+
+        /* Present? */
+        if (!DescSS.Legacy.Gen.u1Present)
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> SS not present -> #SS\n", uNewCs, uNewRip, uNewSs, uNewRsp));
+            return iemRaiseStackSelectorNotPresentBySelector(pVCpu, uNewSs);
+        }
+        cbLimitSs = X86DESC_LIMIT_G(&DescSS.Legacy);
+    }
+
+    /* Check EIP. */
+    if (DescCS.Legacy.Gen.u1Long)
+    {
+        if (!IEM_IS_CANONICAL(uNewRip))
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> RIP is not canonical -> #GP(0)\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp));
+            return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
+        }
+    }
+    else
+    {
+        if (uNewRip > cbLimitCS)
+        {
+            Log(("iret %04x:%016RX64/%04x:%016RX64 -> EIP is out of bounds (%#x) -> #GP(0)\n",
+                 uNewCs, uNewRip, uNewSs, uNewRsp, cbLimitCS));
+            /** @todo Which is it, \#GP(0) or \#GP(sel)? */
+            return iemRaiseSelectorBoundsBySelector(pVCpu, uNewCs);
+        }
+    }
+
+    /*
+     * Commit the changes, marking CS and SS accessed first since
+     * that may fail.
+     */
+    /** @todo where exactly are these actually marked accessed by a real CPU? */
+    if (!(DescCS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewCs);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        DescCS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+    if (!(DescSS.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+    {
+        rcStrict = iemMemMarkSelDescAccessed(pVCpu, uNewSs);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+        DescSS.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+    }
+
+    uint32_t fEFlagsMask = X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_SF
+                         | X86_EFL_TF | X86_EFL_DF | X86_EFL_OF | X86_EFL_NT;
+    if (enmEffOpSize != IEMMODE_16BIT)
+        fEFlagsMask |= X86_EFL_RF | X86_EFL_AC | X86_EFL_ID;
+    if (pVCpu->iem.s.uCpl == 0)
+        fEFlagsMask |= X86_EFL_IF | X86_EFL_IOPL | X86_EFL_VIF | X86_EFL_VIP; /* VM is ignored */
+    else if (pVCpu->iem.s.uCpl <= pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL)
+        fEFlagsMask |= X86_EFL_IF;
+    uint32_t fEFlagsNew = IEMMISC_GET_EFL(pVCpu);
+    fEFlagsNew         &= ~fEFlagsMask;
+    fEFlagsNew         |= uNewFlags & fEFlagsMask;
+#ifdef DBGFTRACE_ENABLED
+    RTTraceBufAddMsgF(pVCpu->CTX_SUFF(pVM)->CTX_SUFF(hTraceBuf), "iret/%ul%u %08llx -> %04x:%04llx %llx %04x:%04llx",
+                      pVCpu->iem.s.uCpl, uNewCpl, pVCpu->cpum.GstCtx.rip, uNewCs, uNewRip, uNewFlags, uNewSs, uNewRsp);
+#endif
+
+    IEMMISC_SET_EFL(pVCpu, fEFlagsNew);
+    pVCpu->cpum.GstCtx.rip           = uNewRip;
+    pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+    pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+    pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESC_GET_HID_ATTR(&DescCS.Legacy);
+    pVCpu->cpum.GstCtx.cs.u32Limit   = cbLimitCS;
+    pVCpu->cpum.GstCtx.cs.u64Base    = X86DESC_BASE(&DescCS.Legacy);
+    pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+    if (pVCpu->cpum.GstCtx.cs.Attr.n.u1Long || pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig)
+        pVCpu->cpum.GstCtx.rsp       = uNewRsp;
+    else
+        pVCpu->cpum.GstCtx.sp        = (uint16_t)uNewRsp;
+    pVCpu->cpum.GstCtx.ss.Sel        = uNewSs;
+    pVCpu->cpum.GstCtx.ss.ValidSel   = uNewSs;
+    if (!(uNewSs & X86_SEL_MASK_OFF_RPL))
+    {
+        pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = X86DESCATTR_UNUSABLE | (uNewCpl << X86DESCATTR_DPL_SHIFT);
+        pVCpu->cpum.GstCtx.ss.u32Limit   = UINT32_MAX;
+        pVCpu->cpum.GstCtx.ss.u64Base    = 0;
+        Log2(("iretq new SS: NULL\n"));
+    }
+    else
+    {
+        pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = X86DESC_GET_HID_ATTR(&DescSS.Legacy);
+        pVCpu->cpum.GstCtx.ss.u32Limit   = cbLimitSs;
+        pVCpu->cpum.GstCtx.ss.u64Base    = X86DESC_BASE(&DescSS.Legacy);
+        Log2(("iretq new SS: base=%#RX64 lim=%#x attr=%#x\n", pVCpu->cpum.GstCtx.ss.u64Base, pVCpu->cpum.GstCtx.ss.u32Limit, pVCpu->cpum.GstCtx.ss.Attr.u));
+    }
+
+    if (pVCpu->iem.s.uCpl != uNewCpl)
+    {
+        pVCpu->iem.s.uCpl = uNewCpl;
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.ds);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.es);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.fs);
+        iemHlpAdjustSelectorForNewCpl(pVCpu, uNewCpl, &pVCpu->cpum.GstCtx.gs);
+    }
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr); /** @todo may light flush if the ring + mode doesn't change */
+
+/** @todo single stepping   */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements iret.
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_iret, IEMMODE, enmEffOpSize)
+{
+    bool fBlockingNmi = CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        /*
+         * Record whether NMI (or virtual-NMI) blocking is in effect during the execution
+         * of this IRET instruction. We need to provide this information as part of some
+         * VM-exits.
+         *
+         * See Intel spec. 27.2.2 "Information for VM Exits Due to Vectored Events".
+         */
+        if (IEM_VMX_IS_PINCTLS_SET(pVCpu, VMX_PIN_CTLS_VIRT_NMI))
+            pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking;
+        else
+            pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret = fBlockingNmi;
+
+        /*
+         * If "NMI exiting" is set, IRET does not affect blocking of NMIs.
+         * See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation".
+         */
+        if (IEM_VMX_IS_PINCTLS_SET(pVCpu, VMX_PIN_CTLS_NMI_EXIT))
+            fBlockingNmi = false;
+
+        /* Clear virtual-NMI blocking, if any, before causing any further exceptions. */
+        pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false;
+    }
+#endif
+
+    /*
+     * The SVM nested-guest intercept for IRET takes priority over all exceptions,
+     * The NMI is still held pending (which I assume means blocking of further NMIs
+     * is in effect).
+     *
+     * See AMD spec. 15.9 "Instruction Intercepts".
+     * See AMD spec. 15.21.9 "NMI Support".
+     */
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IRET))
+    {
+        Log(("iret: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IRET, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Clear NMI blocking, if any, before causing any further exceptions.
+     * See Intel spec. 6.7.1 "Handling Multiple NMIs".
+     */
+    if (fBlockingNmi)
+        CPUMClearInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
+
+    /*
+     * Call a mode specific worker.
+     */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+        return IEM_CIMPL_CALL_1(iemCImpl_iret_real_v8086, enmEffOpSize);
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_LDTR);
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        return IEM_CIMPL_CALL_1(iemCImpl_iret_64bit, enmEffOpSize);
+    return     IEM_CIMPL_CALL_1(iemCImpl_iret_prot, enmEffOpSize);
+}
+
+
+static void iemLoadallSetSelector(PVMCPUCC pVCpu, uint8_t iSegReg, uint16_t uSel)
+{
+    PCPUMSELREGHID  pHid = iemSRegGetHid(pVCpu, iSegReg);
+
+    pHid->Sel      = uSel;
+    pHid->ValidSel = uSel;
+    pHid->fFlags   = CPUMSELREG_FLAGS_VALID;
+}
+
+
+static void iemLoadall286SetDescCache(PVMCPUCC pVCpu, uint8_t iSegReg, uint8_t const *pbMem)
+{
+    PCPUMSELREGHID  pHid = iemSRegGetHid(pVCpu, iSegReg);
+
+    /* The base is in the first three bytes. */
+    pHid->u64Base  = pbMem[0] + (pbMem[1] << 8) + (pbMem[2] << 16);
+    /* The attributes are in the fourth byte. */
+    pHid->Attr.u   = pbMem[3];
+    /* The limit is in the last two bytes. */
+    pHid->u32Limit = pbMem[4] + (pbMem[5] << 8);
+}
+
+
+/**
+ * Implements 286 LOADALL (286 CPUs only).
+ */
+IEM_CIMPL_DEF_0(iemCImpl_loadall286)
+{
+    NOREF(cbInstr);
+
+    /* Data is loaded from a buffer at 800h. No checks are done on the
+     * validity of loaded state.
+     *
+     * LOADALL only loads the internal CPU state, it does not access any
+     * GDT, LDT, or similar tables.
+     */
+
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("loadall286: CPL must be 0 not %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    uint8_t const *pbMem = NULL;
+    uint16_t const *pa16Mem;
+    uint8_t const *pa8Mem;
+    RTGCPHYS GCPtrStart = 0x800;    /* Fixed table location. */
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&pbMem, 0x66, UINT8_MAX, GCPtrStart, IEM_ACCESS_SYS_R, 0);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* The MSW is at offset 0x06. */
+    pa16Mem = (uint16_t const *)(pbMem + 0x06);
+    /* Even LOADALL can't clear the MSW.PE bit, though it can set it. */
+    uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0 & ~(X86_CR0_MP | X86_CR0_EM | X86_CR0_TS);
+    uNewCr0 |= *pa16Mem & (X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS);
+    uint64_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
+
+    CPUMSetGuestCR0(pVCpu, uNewCr0);
+    Assert(pVCpu->cpum.GstCtx.cr0 == uNewCr0);
+
+    /* Inform PGM if mode changed. */
+    if ((uNewCr0 & X86_CR0_PE) != (uOldCr0 & X86_CR0_PE))
+    {
+        int rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
+        AssertRCReturn(rc, rc);
+        /* ignore informational status codes */
+    }
+    rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
+                             false /* fForce */);
+
+    /* TR selector is at offset 0x16. */
+    pa16Mem = (uint16_t const *)(pbMem + 0x16);
+    pVCpu->cpum.GstCtx.tr.Sel      = pa16Mem[0];
+    pVCpu->cpum.GstCtx.tr.ValidSel = pa16Mem[0];
+    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
+
+    /* Followed by FLAGS... */
+    pVCpu->cpum.GstCtx.eflags.u = pa16Mem[1] | X86_EFL_1;
+    pVCpu->cpum.GstCtx.ip       = pa16Mem[2];   /* ...and IP. */
+
+    /* LDT is at offset 0x1C. */
+    pa16Mem = (uint16_t const *)(pbMem + 0x1C);
+    pVCpu->cpum.GstCtx.ldtr.Sel      = pa16Mem[0];
+    pVCpu->cpum.GstCtx.ldtr.ValidSel = pa16Mem[0];
+    pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+
+    /* Segment registers are at offset 0x1E. */
+    pa16Mem = (uint16_t const *)(pbMem + 0x1E);
+    iemLoadallSetSelector(pVCpu, X86_SREG_DS, pa16Mem[0]);
+    iemLoadallSetSelector(pVCpu, X86_SREG_SS, pa16Mem[1]);
+    iemLoadallSetSelector(pVCpu, X86_SREG_CS, pa16Mem[2]);
+    iemLoadallSetSelector(pVCpu, X86_SREG_ES, pa16Mem[3]);
+
+    /* GPRs are at offset 0x26. */
+    pa16Mem = (uint16_t const *)(pbMem + 0x26);
+    pVCpu->cpum.GstCtx.di = pa16Mem[0];
+    pVCpu->cpum.GstCtx.si = pa16Mem[1];
+    pVCpu->cpum.GstCtx.bp = pa16Mem[2];
+    pVCpu->cpum.GstCtx.sp = pa16Mem[3];
+    pVCpu->cpum.GstCtx.bx = pa16Mem[4];
+    pVCpu->cpum.GstCtx.dx = pa16Mem[5];
+    pVCpu->cpum.GstCtx.cx = pa16Mem[6];
+    pVCpu->cpum.GstCtx.ax = pa16Mem[7];
+
+    /* Descriptor caches are at offset 0x36, 6 bytes per entry. */
+    iemLoadall286SetDescCache(pVCpu, X86_SREG_ES, pbMem + 0x36);
+    iemLoadall286SetDescCache(pVCpu, X86_SREG_CS, pbMem + 0x3C);
+    iemLoadall286SetDescCache(pVCpu, X86_SREG_SS, pbMem + 0x42);
+    iemLoadall286SetDescCache(pVCpu, X86_SREG_DS, pbMem + 0x48);
+
+    /* GDTR contents are at offset 0x4E, 6 bytes. */
+    RTGCPHYS GCPtrBase;
+    uint16_t cbLimit;
+    pa8Mem = pbMem + 0x4E;
+    /* NB: Fourth byte "should be zero"; we are ignoring it. */
+    GCPtrBase = pa8Mem[0] + (pa8Mem[1] << 8) + (pa8Mem[2] << 16);
+    cbLimit = pa8Mem[4] + (pa8Mem[5] << 8);
+    CPUMSetGuestGDTR(pVCpu, GCPtrBase, cbLimit);
+
+    /* IDTR contents are at offset 0x5A, 6 bytes. */
+    pa8Mem = pbMem + 0x5A;
+    GCPtrBase = pa8Mem[0] + (pa8Mem[1] << 8) + (pa8Mem[2] << 16);
+    cbLimit = pa8Mem[4] + (pa8Mem[5] << 8);
+    CPUMSetGuestIDTR(pVCpu, GCPtrBase, cbLimit);
+
+    Log(("LOADALL: GDTR:%08RX64/%04X, IDTR:%08RX64/%04X\n", pVCpu->cpum.GstCtx.gdtr.pGdt, pVCpu->cpum.GstCtx.gdtr.cbGdt, pVCpu->cpum.GstCtx.idtr.pIdt, pVCpu->cpum.GstCtx.idtr.cbIdt));
+    Log(("LOADALL: CS:%04X, CS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.cs.u64Base, pVCpu->cpum.GstCtx.cs.u32Limit, pVCpu->cpum.GstCtx.cs.Attr.u));
+    Log(("LOADALL: DS:%04X, DS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.ds.Sel, pVCpu->cpum.GstCtx.ds.u64Base, pVCpu->cpum.GstCtx.ds.u32Limit, pVCpu->cpum.GstCtx.ds.Attr.u));
+    Log(("LOADALL: ES:%04X, ES base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.es.Sel, pVCpu->cpum.GstCtx.es.u64Base, pVCpu->cpum.GstCtx.es.u32Limit, pVCpu->cpum.GstCtx.es.Attr.u));
+    Log(("LOADALL: SS:%04X, SS base:%08X, limit:%04X, attrs:%02X\n", pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.ss.u64Base, pVCpu->cpum.GstCtx.ss.u32Limit, pVCpu->cpum.GstCtx.ss.Attr.u));
+    Log(("LOADALL: SI:%04X, DI:%04X, AX:%04X, BX:%04X, CX:%04X, DX:%04X\n", pVCpu->cpum.GstCtx.si, pVCpu->cpum.GstCtx.di, pVCpu->cpum.GstCtx.bx, pVCpu->cpum.GstCtx.bx, pVCpu->cpum.GstCtx.cx, pVCpu->cpum.GstCtx.dx));
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pbMem, IEM_ACCESS_SYS_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* The CPL may change. It is taken from the "DPL fields of the SS and CS
+     * descriptor caches" but there is no word as to what happens if those are
+     * not identical (probably bad things).
+     */
+    pVCpu->iem.s.uCpl = pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl;
+
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS | CPUM_CHANGED_IDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_TR | CPUM_CHANGED_LDTR);
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
+
+/** @todo single stepping   */
+    return rcStrict;
+}
+
+
+/**
+ * Implements SYSCALL (AMD and Intel64).
+ */
+IEM_CIMPL_DEF_0(iemCImpl_syscall)
+{
+    /** @todo hack, LOADALL should be decoded as such on a 286. */
+    if (RT_UNLIKELY(pVCpu->iem.s.uTargetCpu == IEMTARGETCPU_286))
+        return iemCImpl_loadall286(pVCpu, cbInstr);
+
+    /*
+     * Check preconditions.
+     *
+     * Note that CPUs described in the documentation may load a few odd values
+     * into CS and SS than we allow here.  This has yet to be checked on real
+     * hardware.
+     */
+    if (!(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SCE))
+    {
+        Log(("syscall: Not enabled in EFER -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
+    {
+        Log(("syscall: Protected mode is required -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu) && !CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+    {
+        Log(("syscall: Only available in long mode on intel -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS);
+
+    /** @todo verify RPL ignoring and CS=0xfff8 (i.e. SS == 0). */
+    /** @todo what about LDT selectors? Shouldn't matter, really. */
+    uint16_t uNewCs = (pVCpu->cpum.GstCtx.msrSTAR >> MSR_K6_STAR_SYSCALL_CS_SS_SHIFT) & X86_SEL_MASK_OFF_RPL;
+    uint16_t uNewSs = uNewCs + 8;
+    if (uNewCs == 0 || uNewSs == 0)
+    {
+        /** @todo Neither Intel nor AMD document this check. */
+        Log(("syscall: msrSTAR.CS = 0 or SS = 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Long mode and legacy mode differs. */
+    if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+    {
+        uint64_t uNewRip = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.msrLSTAR : pVCpu->cpum.GstCtx. msrCSTAR;
+
+        /* This test isn't in the docs, but I'm not trusting the guys writing
+           the MSRs to have validated the values as canonical like they should. */
+        if (!IEM_IS_CANONICAL(uNewRip))
+        {
+            /** @todo Intel claims this can't happen because IA32_LSTAR MSR can't be written with non-canonical address. */
+            Log(("syscall: New RIP not canonical -> #UD\n"));
+            return iemRaiseUndefinedOpcode(pVCpu);
+        }
+
+        /*
+         * Commit it.
+         */
+        Log(("syscall: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, uNewRip));
+        pVCpu->cpum.GstCtx.rcx           = pVCpu->cpum.GstCtx.rip + cbInstr;
+        pVCpu->cpum.GstCtx.rip           = uNewRip;
+
+        pVCpu->cpum.GstCtx.rflags.u     &= ~X86_EFL_RF;
+        pVCpu->cpum.GstCtx.r11           = pVCpu->cpum.GstCtx.rflags.u;
+        pVCpu->cpum.GstCtx.rflags.u     &= ~pVCpu->cpum.GstCtx.msrSFMASK;
+        pVCpu->cpum.GstCtx.rflags.u     |= X86_EFL_1;
+
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_L | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_RW_ACC;
+    }
+    else
+    {
+        /*
+         * Commit it.
+         */
+        Log(("syscall: %04x:%08RX32 [efl=%#x] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, uNewCs, (uint32_t)(pVCpu->cpum.GstCtx.msrSTAR & MSR_K6_STAR_SYSCALL_EIP_MASK)));
+        pVCpu->cpum.GstCtx.rcx           = pVCpu->cpum.GstCtx.eip + cbInstr;
+        pVCpu->cpum.GstCtx.rip           = pVCpu->cpum.GstCtx.msrSTAR & MSR_K6_STAR_SYSCALL_EIP_MASK;
+        pVCpu->cpum.GstCtx.rflags.u     &= ~(X86_EFL_VM | X86_EFL_IF | X86_EFL_RF);
+
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC;
+        pVCpu->cpum.GstCtx.ss.Attr.u     = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_RW_ACC;
+    }
+    pVCpu->cpum.GstCtx.cs.Sel        = uNewCs;
+    pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs;
+    pVCpu->cpum.GstCtx.cs.u64Base    = 0;
+    pVCpu->cpum.GstCtx.cs.u32Limit   = UINT32_MAX;
+    pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+
+    pVCpu->cpum.GstCtx.ss.Sel        = uNewSs;
+    pVCpu->cpum.GstCtx.ss.ValidSel   = uNewSs;
+    pVCpu->cpum.GstCtx.ss.u64Base    = 0;
+    pVCpu->cpum.GstCtx.ss.u32Limit   = UINT32_MAX;
+    pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+
+    pVCpu->iem.s.uCpl       = 0;
+    pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
+
+/** @todo single step   */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements SYSRET (AMD and Intel64).
+ */
+IEM_CIMPL_DEF_0(iemCImpl_sysret)
+
+{
+    RT_NOREF_PV(cbInstr);
+
+    /*
+     * Check preconditions.
+     *
+     * Note that CPUs described in the documentation may load a few odd values
+     * into CS and SS than we allow here.  This has yet to be checked on real
+     * hardware.
+     */
+    if (!(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_SCE))
+    {
+        Log(("sysret: Not enabled in EFER -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (IEM_IS_GUEST_CPU_INTEL(pVCpu) && !CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+    {
+        Log(("sysret: Only available in long mode on intel -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
+    {
+        Log(("sysret: Protected mode is required -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("sysret: CPL must be 0 not %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS);
+
+    /** @todo Does SYSRET verify CS != 0 and SS != 0? Neither is valid in ring-3. */
+    uint16_t uNewCs = (pVCpu->cpum.GstCtx.msrSTAR >> MSR_K6_STAR_SYSRET_CS_SS_SHIFT) & X86_SEL_MASK_OFF_RPL;
+    uint16_t uNewSs = uNewCs + 8;
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        uNewCs += 16;
+    if (uNewCs == 0 || uNewSs == 0)
+    {
+        Log(("sysret: msrSTAR.CS = 0 or SS = 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Commit it.
+     */
+    if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+    {
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        {
+            Log(("sysret: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64 [r11=%#llx]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, pVCpu->cpum.GstCtx.rcx, pVCpu->cpum.GstCtx.r11));
+            /* Note! We disregard intel manual regarding the RCX canonical
+                     check, ask intel+xen why AMD doesn't do it. */
+            pVCpu->cpum.GstCtx.rip       = pVCpu->cpum.GstCtx.rcx;
+            pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_L | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC
+                            | (3 << X86DESCATTR_DPL_SHIFT);
+        }
+        else
+        {
+            Log(("sysret: %04x:%016RX64 [efl=%#llx] -> %04x:%08RX32 [r11=%#llx]\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, uNewCs, pVCpu->cpum.GstCtx.ecx, pVCpu->cpum.GstCtx.r11));
+            pVCpu->cpum.GstCtx.rip       = pVCpu->cpum.GstCtx.ecx;
+            pVCpu->cpum.GstCtx.cs.Attr.u = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC
+                            | (3 << X86DESCATTR_DPL_SHIFT);
+        }
+        /** @todo testcase: See what kind of flags we can make SYSRET restore and
+         *        what it really ignores. RF and VM are hinted at being zero, by AMD.
+         *        Intel says:  RFLAGS := (R11 & 3C7FD7H) | 2; */
+        pVCpu->cpum.GstCtx.rflags.u      = pVCpu->cpum.GstCtx.r11 & (X86_EFL_POPF_BITS | X86_EFL_VIF | X86_EFL_VIP);
+        pVCpu->cpum.GstCtx.rflags.u     |= X86_EFL_1;
+    }
+    else
+    {
+        Log(("sysret: %04x:%08RX32 [efl=%#x] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.eip, pVCpu->cpum.GstCtx.eflags.u, uNewCs, pVCpu->cpum.GstCtx.ecx));
+        pVCpu->cpum.GstCtx.rip           = pVCpu->cpum.GstCtx.rcx;
+        pVCpu->cpum.GstCtx.rflags.u     |= X86_EFL_IF;
+        pVCpu->cpum.GstCtx.cs.Attr.u     = X86DESCATTR_P | X86DESCATTR_G | X86DESCATTR_D | X86DESCATTR_DT | X86_SEL_TYPE_ER_ACC
+                            | (3 << X86DESCATTR_DPL_SHIFT);
+    }
+    pVCpu->cpum.GstCtx.cs.Sel        = uNewCs | 3;
+    pVCpu->cpum.GstCtx.cs.ValidSel   = uNewCs | 3;
+    pVCpu->cpum.GstCtx.cs.u64Base    = 0;
+    pVCpu->cpum.GstCtx.cs.u32Limit   = UINT32_MAX;
+    pVCpu->cpum.GstCtx.cs.fFlags     = CPUMSELREG_FLAGS_VALID;
+
+    pVCpu->cpum.GstCtx.ss.Sel        = uNewSs | 3;
+    pVCpu->cpum.GstCtx.ss.ValidSel   = uNewSs | 3;
+    pVCpu->cpum.GstCtx.ss.fFlags     = CPUMSELREG_FLAGS_VALID;
+    /* The SS hidden bits remains unchanged says AMD. To that I say "Yeah, right!". */
+    pVCpu->cpum.GstCtx.ss.Attr.u    |= (3 << X86DESCATTR_DPL_SHIFT);
+    /** @todo Testcase: verify that SS.u1Long and SS.u1DefBig are left unchanged
+     *        on sysret. */
+
+    pVCpu->iem.s.uCpl       = 3;
+    pVCpu->iem.s.enmCpuMode = iemCalcCpuMode(pVCpu);
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
+
+/** @todo single step   */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements SYSENTER (Intel, 32-bit AMD).
+ */
+IEM_CIMPL_DEF_0(iemCImpl_sysenter)
+{
+    RT_NOREF(cbInstr);
+
+    /*
+     * Check preconditions.
+     *
+     * Note that CPUs described in the documentation may load a few odd values
+     * into CS and SS than we allow here.  This has yet to be checked on real
+     * hardware.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSysEnter)
+    {
+        Log(("sysenter: not supported -=> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
+    {
+        Log(("sysenter: Protected or long mode is required -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    bool fIsLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu));
+    if (IEM_IS_GUEST_CPU_AMD(pVCpu) && fIsLongMode)
+    {
+        Log(("sysenter: Only available in protected mode on AMD -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
+    uint16_t uNewCs = pVCpu->cpum.GstCtx.SysEnter.cs;
+    if ((uNewCs & X86_SEL_MASK_OFF_RPL) == 0)
+    {
+        Log(("sysenter: SYSENTER_CS = %#x -> #GP(0)\n", uNewCs));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* This test isn't in the docs, it's just a safeguard against missing
+       canonical checks when writing the registers. */
+    if (RT_LIKELY(   !fIsLongMode
+                  || (   IEM_IS_CANONICAL(pVCpu->cpum.GstCtx.SysEnter.eip)
+                      && IEM_IS_CANONICAL(pVCpu->cpum.GstCtx.SysEnter.esp))))
+    { /* likely */ }
+    else
+    {
+        Log(("sysenter: SYSENTER_EIP = %#RX64 or/and SYSENTER_ESP = %#RX64 not canonical -> #GP(0)\n",
+             pVCpu->cpum.GstCtx.SysEnter.eip, pVCpu->cpum.GstCtx.SysEnter.esp));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+/** @todo Test: Sysenter from ring-0, ring-1 and ring-2.  */
+
+    /*
+     * Update registers and commit.
+     */
+    if (fIsLongMode)
+    {
+        Log(("sysenter: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip,
+             pVCpu->cpum.GstCtx.rflags.u, uNewCs & X86_SEL_MASK_OFF_RPL, pVCpu->cpum.GstCtx.SysEnter.eip));
+        pVCpu->cpum.GstCtx.rip          = pVCpu->cpum.GstCtx.SysEnter.eip;
+        pVCpu->cpum.GstCtx.rsp          = pVCpu->cpum.GstCtx.SysEnter.esp;
+        pVCpu->cpum.GstCtx.cs.Attr.u    = X86DESCATTR_L | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT
+                                        | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_ER_ACC;
+    }
+    else
+    {
+        Log(("sysenter: %04x:%08RX32 [efl=%#llx] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs, (uint32_t)pVCpu->cpum.GstCtx.rip,
+             pVCpu->cpum.GstCtx.rflags.u, uNewCs & X86_SEL_MASK_OFF_RPL, (uint32_t)pVCpu->cpum.GstCtx.SysEnter.eip));
+        pVCpu->cpum.GstCtx.rip          = (uint32_t)pVCpu->cpum.GstCtx.SysEnter.eip;
+        pVCpu->cpum.GstCtx.rsp          = (uint32_t)pVCpu->cpum.GstCtx.SysEnter.esp;
+        pVCpu->cpum.GstCtx.cs.Attr.u    = X86DESCATTR_D | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT
+                                        | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_ER_ACC;
+    }
+    pVCpu->cpum.GstCtx.cs.Sel           = uNewCs & X86_SEL_MASK_OFF_RPL;
+    pVCpu->cpum.GstCtx.cs.ValidSel      = uNewCs & X86_SEL_MASK_OFF_RPL;
+    pVCpu->cpum.GstCtx.cs.u64Base       = 0;
+    pVCpu->cpum.GstCtx.cs.u32Limit      = UINT32_MAX;
+    pVCpu->cpum.GstCtx.cs.fFlags        = CPUMSELREG_FLAGS_VALID;
+
+    pVCpu->cpum.GstCtx.ss.Sel           = (uNewCs & X86_SEL_MASK_OFF_RPL) + 8;
+    pVCpu->cpum.GstCtx.ss.ValidSel      = (uNewCs & X86_SEL_MASK_OFF_RPL) + 8;
+    pVCpu->cpum.GstCtx.ss.u64Base       = 0;
+    pVCpu->cpum.GstCtx.ss.u32Limit      = UINT32_MAX;
+    pVCpu->cpum.GstCtx.ss.Attr.u        = X86DESCATTR_D | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT
+                                        | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_RW_ACC;
+    pVCpu->cpum.GstCtx.ss.fFlags        = CPUMSELREG_FLAGS_VALID;
+
+    pVCpu->cpum.GstCtx.rflags.Bits.u1IF = 0;
+    pVCpu->cpum.GstCtx.rflags.Bits.u1VM = 0;
+    pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
+
+    pVCpu->iem.s.uCpl                   = 0;
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
+
+/** @todo single stepping   */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements SYSEXIT (Intel, 32-bit AMD).
+ *
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_sysexit, IEMMODE, enmEffOpSize)
+{
+    RT_NOREF(cbInstr);
+
+    /*
+     * Check preconditions.
+     *
+     * Note that CPUs described in the documentation may load a few odd values
+     * into CS and SS than we allow here.  This has yet to be checked on real
+     * hardware.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSysEnter)
+    {
+        Log(("sysexit: not supported -=> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
+    {
+        Log(("sysexit: Protected or long mode is required -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    bool fIsLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu));
+    if (IEM_IS_GUEST_CPU_AMD(pVCpu) && fIsLongMode)
+    {
+        Log(("sysexit: Only available in protected mode on AMD -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("sysexit: CPL(=%u) != 0 -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
+    uint16_t uNewCs = pVCpu->cpum.GstCtx.SysEnter.cs;
+    if ((uNewCs & X86_SEL_MASK_OFF_RPL) == 0)
+    {
+        Log(("sysexit: SYSENTER_CS = %#x -> #GP(0)\n", uNewCs));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Update registers and commit.
+     */
+    if (enmEffOpSize == IEMMODE_64BIT)
+    {
+        Log(("sysexit: %04x:%016RX64 [efl=%#llx] -> %04x:%016RX64\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip,
+             pVCpu->cpum.GstCtx.rflags.u, (uNewCs | 3) + 32, pVCpu->cpum.GstCtx.rcx));
+        pVCpu->cpum.GstCtx.rip          = pVCpu->cpum.GstCtx.rdx;
+        pVCpu->cpum.GstCtx.rsp          = pVCpu->cpum.GstCtx.rcx;
+        pVCpu->cpum.GstCtx.cs.Attr.u    = X86DESCATTR_L | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT
+                                        | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_ER_ACC | (3 << X86DESCATTR_DPL_SHIFT);
+        pVCpu->cpum.GstCtx.cs.Sel       = (uNewCs | 3) + 32;
+        pVCpu->cpum.GstCtx.cs.ValidSel  = (uNewCs | 3) + 32;
+        pVCpu->cpum.GstCtx.ss.Sel       = (uNewCs | 3) + 40;
+        pVCpu->cpum.GstCtx.ss.ValidSel  = (uNewCs | 3) + 40;
+    }
+    else
+    {
+        Log(("sysexit: %04x:%08RX64 [efl=%#llx] -> %04x:%08RX32\n", pVCpu->cpum.GstCtx.cs, pVCpu->cpum.GstCtx.rip,
+             pVCpu->cpum.GstCtx.rflags.u, (uNewCs | 3) + 16, (uint32_t)pVCpu->cpum.GstCtx.edx));
+        pVCpu->cpum.GstCtx.rip          = pVCpu->cpum.GstCtx.edx;
+        pVCpu->cpum.GstCtx.rsp          = pVCpu->cpum.GstCtx.ecx;
+        pVCpu->cpum.GstCtx.cs.Attr.u    = X86DESCATTR_D | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT
+                                        | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_ER_ACC | (3 << X86DESCATTR_DPL_SHIFT);
+        pVCpu->cpum.GstCtx.cs.Sel       = (uNewCs | 3) + 16;
+        pVCpu->cpum.GstCtx.cs.ValidSel  = (uNewCs | 3) + 16;
+        pVCpu->cpum.GstCtx.ss.Sel       = (uNewCs | 3) + 24;
+        pVCpu->cpum.GstCtx.ss.ValidSel  = (uNewCs | 3) + 24;
+    }
+    pVCpu->cpum.GstCtx.cs.u64Base       = 0;
+    pVCpu->cpum.GstCtx.cs.u32Limit      = UINT32_MAX;
+    pVCpu->cpum.GstCtx.cs.fFlags        = CPUMSELREG_FLAGS_VALID;
+
+    pVCpu->cpum.GstCtx.ss.u64Base       = 0;
+    pVCpu->cpum.GstCtx.ss.u32Limit      = UINT32_MAX;
+    pVCpu->cpum.GstCtx.ss.Attr.u        = X86DESCATTR_D | X86DESCATTR_G | X86DESCATTR_P | X86DESCATTR_DT
+                                        | X86DESCATTR_LIMIT_HIGH | X86_SEL_TYPE_RW_ACC | (3 << X86DESCATTR_DPL_SHIFT);
+    pVCpu->cpum.GstCtx.ss.fFlags        = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
+
+    pVCpu->iem.s.uCpl                   = 3;
+/** @todo single stepping   */
+
+    /* Flush the prefetch buffer. */
+    IEM_FLUSH_PREFETCH_HEAVY(pVCpu, cbInstr);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Completes a MOV SReg,XXX or POP SReg instruction.
+ *
+ * When not modifying SS or when we're already in an interrupt shadow we
+ * can update RIP and finish the instruction the normal way.
+ *
+ * Otherwise, the MOV/POP SS interrupt shadow that we now enable will block
+ * both TF and DBx events.  The TF will be ignored while the DBx ones will
+ * be delayed till the next instruction boundrary.  For more details see
+ * @sdmv3{077,200,6.8.3,Masking Exceptions and Interrupts When Switching Stacks}.
+ */
+DECLINLINE(VBOXSTRICTRC) iemCImpl_LoadSRegFinish(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iSegReg)
+{
+    if (iSegReg != X86_SREG_SS || CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+    iemRegAddToRip(pVCpu, cbInstr);
+    pVCpu->cpum.GstCtx.eflags.uBoth &= ~X86_EFL_RF; /* Shadow int isn't set and DRx is delayed, so only clear RF. */
+    CPUMSetInInterruptShadowSs(&pVCpu->cpum.GstCtx);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Common worker for 'pop SReg', 'mov SReg, GReg' and 'lXs GReg, reg/mem'.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling
+ *                      thread.
+ * @param   iSegReg     The segment register number (valid).
+ * @param   uSel        The new selector value.
+ */
+static VBOXSTRICTRC iemCImpl_LoadSRegWorker(PVMCPUCC pVCpu, uint8_t iSegReg, uint16_t uSel)
+{
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iSegReg));
+    uint16_t       *pSel = iemSRegRef(pVCpu, iSegReg);
+    PCPUMSELREGHID  pHid = iemSRegGetHid(pVCpu, iSegReg);
+
+    Assert(iSegReg <= X86_SREG_GS && iSegReg != X86_SREG_CS);
+
+    /*
+     * Real mode and V8086 mode are easy.
+     */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        *pSel           = uSel;
+        pHid->u64Base   = (uint32_t)uSel << 4;
+        pHid->ValidSel  = uSel;
+        pHid->fFlags    = CPUMSELREG_FLAGS_VALID;
+#if 0 /* AMD Volume 2, chapter 4.1 - "real mode segmentation" - states that limit and attributes are untouched. */
+        /** @todo Does the CPU actually load limits and attributes in the
+         *        real/V8086 mode segment load case?  It doesn't for CS in far
+         *        jumps...  Affects unreal mode.  */
+        pHid->u32Limit          = 0xffff;
+        pHid->Attr.u = 0;
+        pHid->Attr.n.u1Present  = 1;
+        pHid->Attr.n.u1DescType = 1;
+        pHid->Attr.n.u4Type     = iSegReg != X86_SREG_CS
+                                ? X86_SEL_TYPE_RW
+                                : X86_SEL_TYPE_READ | X86_SEL_TYPE_CODE;
+#endif
+    }
+    /*
+     * Protected mode.
+     *
+     * Check if it's a null segment selector value first, that's OK for DS, ES,
+     * FS and GS.  If not null, then we have to load and parse the descriptor.
+     */
+    else if (!(uSel & X86_SEL_MASK_OFF_RPL))
+    {
+        Assert(iSegReg != X86_SREG_CS); /** @todo testcase for \#UD on MOV CS, ax! */
+        if (iSegReg == X86_SREG_SS)
+        {
+            /* In 64-bit kernel mode, the stack can be 0 because of the way
+               interrupts are dispatched. AMD seems to have a slighly more
+               relaxed relationship to SS.RPL than intel does. */
+            /** @todo We cannot 'mov ss, 3' in 64-bit kernel mode, can we? There is a testcase (bs-cpu-xcpt-1), but double check this! */
+            if (   pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
+                || pVCpu->iem.s.uCpl > 2
+                || (   uSel != pVCpu->iem.s.uCpl
+                    && !IEM_IS_GUEST_CPU_AMD(pVCpu)) )
+            {
+                Log(("load sreg %#x -> invalid stack selector, #GP(0)\n", uSel));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+        }
+
+        *pSel = uSel;   /* Not RPL, remember :-) */
+        iemHlpLoadNullDataSelectorProt(pVCpu, pHid, uSel);
+        if (iSegReg == X86_SREG_SS)
+            pHid->Attr.u |= pVCpu->iem.s.uCpl << X86DESCATTR_DPL_SHIFT;
+    }
+    else
+    {
+
+        /* Fetch the descriptor. */
+        IEMSELDESC Desc;
+        VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uSel, X86_XCPT_GP); /** @todo Correct exception? */
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        /* Check GPs first. */
+        if (!Desc.Legacy.Gen.u1DescType)
+        {
+            Log(("load sreg %d (=%#x) - system selector (%#x) -> #GP\n", iSegReg, uSel, Desc.Legacy.Gen.u4Type));
+            return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+        }
+        if (iSegReg == X86_SREG_SS) /* SS gets different treatment */
+        {
+            if (    (Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_CODE)
+                || !(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_WRITE) )
+            {
+                Log(("load sreg SS, %#x - code or read only (%#x) -> #GP\n", uSel, Desc.Legacy.Gen.u4Type));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+            }
+            if ((uSel & X86_SEL_RPL) != pVCpu->iem.s.uCpl)
+            {
+                Log(("load sreg SS, %#x - RPL and CPL (%d) differs -> #GP\n", uSel, pVCpu->iem.s.uCpl));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+            }
+            if (Desc.Legacy.Gen.u2Dpl != pVCpu->iem.s.uCpl)
+            {
+                Log(("load sreg SS, %#x - DPL (%d) and CPL (%d) differs -> #GP\n", uSel, Desc.Legacy.Gen.u2Dpl, pVCpu->iem.s.uCpl));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+            }
+        }
+        else
+        {
+            if ((Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ)) == X86_SEL_TYPE_CODE)
+            {
+                Log(("load sreg%u, %#x - execute only segment -> #GP\n", iSegReg, uSel));
+                return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+            }
+            if (   (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+                != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+            {
+#if 0 /* this is what intel says. */
+                if (   (uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl
+                    && pVCpu->iem.s.uCpl        > Desc.Legacy.Gen.u2Dpl)
+                {
+                    Log(("load sreg%u, %#x - both RPL (%d) and CPL (%d) are greater than DPL (%d) -> #GP\n",
+                         iSegReg, uSel, (uSel & X86_SEL_RPL), pVCpu->iem.s.uCpl, Desc.Legacy.Gen.u2Dpl));
+                    return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+                }
+#else /* this is what makes more sense. */
+                if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
+                {
+                    Log(("load sreg%u, %#x - RPL (%d) is greater than DPL (%d) -> #GP\n",
+                         iSegReg, uSel, (uSel & X86_SEL_RPL), Desc.Legacy.Gen.u2Dpl));
+                    return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+                }
+                if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)
+                {
+                    Log(("load sreg%u, %#x - CPL (%d) is greater than DPL (%d) -> #GP\n",
+                         iSegReg, uSel, pVCpu->iem.s.uCpl, Desc.Legacy.Gen.u2Dpl));
+                    return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uSel);
+                }
+#endif
+            }
+        }
+
+        /* Is it there? */
+        if (!Desc.Legacy.Gen.u1Present)
+        {
+            Log(("load sreg%d,%#x - segment not present -> #NP\n", iSegReg, uSel));
+            return iemRaiseSelectorNotPresentBySelector(pVCpu, uSel);
+        }
+
+        /* The base and limit. */
+        uint32_t cbLimit = X86DESC_LIMIT_G(&Desc.Legacy);
+        uint64_t u64Base = X86DESC_BASE(&Desc.Legacy);
+
+        /*
+         * Ok, everything checked out fine.  Now set the accessed bit before
+         * committing the result into the registers.
+         */
+        if (!(Desc.Legacy.Gen.u4Type & X86_SEL_TYPE_ACCESSED))
+        {
+            rcStrict = iemMemMarkSelDescAccessed(pVCpu, uSel);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_ACCESSED;
+        }
+
+        /* commit */
+        *pSel = uSel;
+        pHid->Attr.u   = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+        pHid->u32Limit = cbLimit;
+        pHid->u64Base  = u64Base;
+        pHid->ValidSel = uSel;
+        pHid->fFlags   = CPUMSELREG_FLAGS_VALID;
+
+        /** @todo check if the hidden bits are loaded correctly for 64-bit
+         *        mode.  */
+    }
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pHid));
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_HIDDEN_SEL_REGS);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements 'mov SReg, r/m'.
+ *
+ * @param   iSegReg     The segment register number (valid).
+ * @param   uSel        The new selector value.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_load_SReg, uint8_t, iSegReg, uint16_t, uSel)
+{
+    VBOXSTRICTRC rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemCImpl_LoadSRegFinish(pVCpu, cbInstr, iSegReg);
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'pop SReg'.
+ *
+ * @param   iSegReg         The segment register number (valid).
+ * @param   enmEffOpSize    The efficient operand size (valid).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_pop_Sreg, uint8_t, iSegReg, IEMMODE, enmEffOpSize)
+{
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * Read the selector off the stack and join paths with mov ss, reg.
+     */
+    RTUINT64U TmpRsp;
+    TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t uSel;
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &uSel, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Value;
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Value, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, (uint16_t)u32Value);
+            break;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Value;
+            rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Value, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, (uint16_t)u64Value);
+            break;
+        }
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    /*
+     * If the load succeeded, commit the stack change and finish the instruction.
+     */
+    if (rcStrict == VINF_SUCCESS)
+    {
+        pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+        rcStrict = iemCImpl_LoadSRegFinish(pVCpu, cbInstr, iSegReg);
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Implements lgs, lfs, les, lds & lss.
+ */
+IEM_CIMPL_DEF_5(iemCImpl_load_SReg_Greg, uint16_t, uSel, uint64_t, offSeg, uint8_t, iSegReg, uint8_t, iGReg, IEMMODE, enmEffOpSize)
+{
+    /*
+     * Use iemCImpl_LoadSRegWorker to do the tricky segment register loading.
+     */
+    /** @todo verify and test that mov, pop and lXs works the segment
+     *        register loading in the exact same way. */
+    VBOXSTRICTRC rcStrict = iemCImpl_LoadSRegWorker(pVCpu, iSegReg, uSel);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        switch (enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                *(uint16_t *)iemGRegRef(pVCpu, iGReg) = offSeg;
+                break;
+            case IEMMODE_32BIT:
+            case IEMMODE_64BIT:
+                *(uint64_t *)iemGRegRef(pVCpu, iGReg) = offSeg;
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Helper for VERR, VERW, LAR, and LSL and loads the descriptor into memory.
+ *
+ * @retval VINF_SUCCESS on success.
+ * @retval VINF_IEM_SELECTOR_NOT_OK if the selector isn't ok.
+ * @retval iemMemFetchSysU64 return value.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling thread.
+ * @param   uSel                The selector value.
+ * @param   fAllowSysDesc       Whether system descriptors are OK or not.
+ * @param   pDesc               Where to return the descriptor on success.
+ */
+static VBOXSTRICTRC iemCImpl_LoadDescHelper(PVMCPUCC pVCpu, uint16_t uSel, bool fAllowSysDesc, PIEMSELDESC pDesc)
+{
+    pDesc->Long.au64[0] = 0;
+    pDesc->Long.au64[1] = 0;
+
+    if (!(uSel & X86_SEL_MASK_OFF_RPL)) /** @todo test this on 64-bit. */
+        return VINF_IEM_SELECTOR_NOT_OK;
+
+    /* Within the table limits? */
+    RTGCPTR GCPtrBase;
+    if (uSel & X86_SEL_LDT)
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
+        if (   !pVCpu->cpum.GstCtx.ldtr.Attr.n.u1Present
+            || (uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.ldtr.u32Limit )
+            return VINF_IEM_SELECTOR_NOT_OK;
+        GCPtrBase = pVCpu->cpum.GstCtx.ldtr.u64Base;
+    }
+    else
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR);
+        if ((uSel | X86_SEL_RPL_LDT) > pVCpu->cpum.GstCtx.gdtr.cbGdt)
+            return VINF_IEM_SELECTOR_NOT_OK;
+        GCPtrBase = pVCpu->cpum.GstCtx.gdtr.pGdt;
+    }
+
+    /* Fetch the descriptor. */
+    VBOXSTRICTRC rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Legacy.u, UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK));
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    if (!pDesc->Legacy.Gen.u1DescType)
+    {
+        if (!fAllowSysDesc)
+            return VINF_IEM_SELECTOR_NOT_OK;
+        if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+        {
+            rcStrict = iemMemFetchSysU64(pVCpu, &pDesc->Long.au64[1], UINT8_MAX, GCPtrBase + (uSel & X86_SEL_MASK) + 8);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements verr (fWrite = false) and verw (fWrite = true).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_VerX, uint16_t, uSel, bool, fWrite)
+{
+    Assert(!IEM_IS_REAL_OR_V86_MODE(pVCpu));
+
+    /** @todo figure whether the accessed bit is set or not. */
+
+    bool         fAccessible = true;
+    IEMSELDESC   Desc;
+    VBOXSTRICTRC rcStrict = iemCImpl_LoadDescHelper(pVCpu, uSel, false /*fAllowSysDesc*/, &Desc);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /* Check the descriptor, order doesn't matter much here. */
+        if (   !Desc.Legacy.Gen.u1DescType
+            || !Desc.Legacy.Gen.u1Present)
+            fAccessible = false;
+        else
+        {
+            if (  fWrite
+                ? (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_WRITE)) != X86_SEL_TYPE_WRITE
+                : (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ))  == X86_SEL_TYPE_CODE)
+                fAccessible = false;
+
+            /** @todo testcase for the conforming behavior. */
+            if (   (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+                != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF))
+            {
+                if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
+                    fAccessible = false;
+                else if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)
+                    fAccessible = false;
+            }
+        }
+
+    }
+    else if (rcStrict == VINF_IEM_SELECTOR_NOT_OK)
+        fAccessible = false;
+    else
+        return rcStrict;
+
+    /* commit */
+    pVCpu->cpum.GstCtx.eflags.Bits.u1ZF = fAccessible;
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements LAR and LSL with 64-bit operand size.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pu64Dst         Pointer to the destination register.
+ * @param   uSel            The selector to load details for.
+ * @param   fIsLar          true = LAR, false = LSL.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_LarLsl_u64, uint64_t *, pu64Dst, uint16_t, uSel, bool, fIsLar)
+{
+    Assert(!IEM_IS_REAL_OR_V86_MODE(pVCpu));
+
+    /** @todo figure whether the accessed bit is set or not. */
+
+    bool         fDescOk = true;
+    IEMSELDESC   Desc;
+    VBOXSTRICTRC rcStrict = iemCImpl_LoadDescHelper(pVCpu, uSel, true /*fAllowSysDesc*/, &Desc);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * Check the descriptor type.
+         */
+        if (!Desc.Legacy.Gen.u1DescType)
+        {
+            if (CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+            {
+                if (Desc.Long.Gen.u5Zeros)
+                    fDescOk = false;
+                else
+                    switch (Desc.Long.Gen.u4Type)
+                    {
+                        /** @todo Intel lists 0 as valid for LSL, verify whether that's correct */
+                        case AMD64_SEL_TYPE_SYS_TSS_AVAIL:
+                        case AMD64_SEL_TYPE_SYS_TSS_BUSY:
+                        case AMD64_SEL_TYPE_SYS_LDT: /** @todo Intel lists this as invalid for LAR, AMD and 32-bit does otherwise. */
+                            break;
+                        case AMD64_SEL_TYPE_SYS_CALL_GATE:
+                            fDescOk = fIsLar;
+                            break;
+                        default:
+                            fDescOk = false;
+                            break;
+                    }
+            }
+            else
+            {
+                switch (Desc.Long.Gen.u4Type)
+                {
+                    case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
+                    case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+                    case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
+                    case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+                    case X86_SEL_TYPE_SYS_LDT:
+                        break;
+                    case X86_SEL_TYPE_SYS_286_CALL_GATE:
+                    case X86_SEL_TYPE_SYS_TASK_GATE:
+                    case X86_SEL_TYPE_SYS_386_CALL_GATE:
+                        fDescOk = fIsLar;
+                        break;
+                    default:
+                        fDescOk = false;
+                        break;
+                }
+            }
+        }
+        if (fDescOk)
+        {
+            /*
+             * Check the RPL/DPL/CPL interaction..
+             */
+            /** @todo testcase for the conforming behavior. */
+            if (   (Desc.Legacy.Gen.u4Type & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)) != (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF)
+                || !Desc.Legacy.Gen.u1DescType)
+            {
+                if ((unsigned)(uSel & X86_SEL_RPL) > Desc.Legacy.Gen.u2Dpl)
+                    fDescOk = false;
+                else if (pVCpu->iem.s.uCpl > Desc.Legacy.Gen.u2Dpl)
+                    fDescOk = false;
+            }
+        }
+
+        if (fDescOk)
+        {
+            /*
+             * All fine, start committing the result.
+             */
+            if (fIsLar)
+                *pu64Dst = Desc.Legacy.au32[1] & UINT32_C(0x00ffff00);
+            else
+                *pu64Dst = X86DESC_LIMIT_G(&Desc.Legacy);
+        }
+
+    }
+    else if (rcStrict == VINF_IEM_SELECTOR_NOT_OK)
+        fDescOk = false;
+    else
+        return rcStrict;
+
+    /* commit flags value and advance rip. */
+    pVCpu->cpum.GstCtx.eflags.Bits.u1ZF = fDescOk;
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements LAR and LSL with 16-bit operand size.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pu16Dst         Pointer to the destination register.
+ * @param   uSel            The selector to load details for.
+ * @param   fIsLar          true = LAR, false = LSL.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_LarLsl_u16, uint16_t *, pu16Dst, uint16_t, uSel, bool, fIsLar)
+{
+    uint64_t u64TmpDst = *pu16Dst;
+    IEM_CIMPL_CALL_3(iemCImpl_LarLsl_u64, &u64TmpDst, uSel, fIsLar);
+    *pu16Dst = u64TmpDst;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements lgdt.
+ *
+ * @param   iEffSeg         The segment of the new gdtr contents
+ * @param   GCPtrEffSrc     The address of the new gdtr contents.
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_lgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("lgdt: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_GDTR_IDTR_ACCESS, VMXINSTRID_LGDT, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_GDTR_WRITES))
+    {
+        Log(("lgdt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_GDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Fetch the limit and base address.
+     */
+    uint16_t cbLimit;
+    RTGCPTR  GCPtrBase;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataXdtr(pVCpu, &cbLimit, &GCPtrBase, iEffSeg, GCPtrEffSrc, enmEffOpSize);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (   pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
+            || X86_IS_CANONICAL(GCPtrBase))
+        {
+            rcStrict = CPUMSetGuestGDTR(pVCpu, GCPtrBase, cbLimit);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+        else
+        {
+            Log(("iemCImpl_lgdt: Non-canonical base %04x:%RGv\n", cbLimit, GCPtrBase));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements sgdt.
+ *
+ * @param   iEffSeg         The segment where to store the gdtr content.
+ * @param   GCPtrEffDst     The address where to store the gdtr content.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_sgdt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    /*
+     * Join paths with sidt.
+     * Note! No CPL or V8086 checks here, it's a really sad story, ask Intel if
+     *       you really must know.
+     */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("sgdt: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_GDTR_IDTR_ACCESS, VMXINSTRID_SGDT, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_GDTR_READS))
+    {
+        Log(("sgdt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_GDTR_READ, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_GDTR);
+    VBOXSTRICTRC rcStrict = iemMemStoreDataXdtr(pVCpu, pVCpu->cpum.GstCtx.gdtr.cbGdt, pVCpu->cpum.GstCtx.gdtr.pGdt, iEffSeg, GCPtrEffDst);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements lidt.
+ *
+ * @param   iEffSeg         The segment of the new idtr contents
+ * @param   GCPtrEffSrc     The address of the new idtr contents.
+ * @param   enmEffOpSize    The effective operand size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_lidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc, IEMMODE, enmEffOpSize)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IDTR_WRITES))
+    {
+        Log(("lidt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Fetch the limit and base address.
+     */
+    uint16_t cbLimit;
+    RTGCPTR  GCPtrBase;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataXdtr(pVCpu, &cbLimit, &GCPtrBase, iEffSeg, GCPtrEffSrc, enmEffOpSize);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (   pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
+            || X86_IS_CANONICAL(GCPtrBase))
+        {
+            CPUMSetGuestIDTR(pVCpu, GCPtrBase, cbLimit);
+            rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+        else
+        {
+            Log(("iemCImpl_lidt: Non-canonical base %04x:%RGv\n", cbLimit, GCPtrBase));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements sidt.
+ *
+ * @param   iEffSeg         The segment where to store the idtr content.
+ * @param   GCPtrEffDst     The address where to store the idtr content.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_sidt, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    /*
+     * Join paths with sgdt.
+     * Note! No CPL or V8086 checks here, it's a really sad story, ask Intel if
+     *       you really must know.
+     */
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IDTR_READS))
+    {
+        Log(("sidt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_IDTR_READ, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_IDTR);
+    VBOXSTRICTRC rcStrict = iemMemStoreDataXdtr(pVCpu, pVCpu->cpum.GstCtx.idtr.cbIdt, pVCpu->cpum.GstCtx.idtr.pIdt, iEffSeg, GCPtrEffDst);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements lldt.
+ *
+ * @param   uNewLdt     The new LDT selector value.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_lldt, uint16_t, uNewLdt)
+{
+    /*
+     * Check preconditions.
+     */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        Log(("lldt %04x - real or v8086 mode -> #GP(0)\n", uNewLdt));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("lldt %04x - CPL is %d -> #GP(0)\n", uNewLdt, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    /* Nested-guest VMX intercept. */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("lldt: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_LLDT, cbInstr);
+    }
+    if (uNewLdt & X86_SEL_LDT)
+    {
+        Log(("lldt %04x - LDT selector -> #GP\n", uNewLdt));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewLdt);
+    }
+
+    /*
+     * Now, loading a NULL selector is easy.
+     */
+    if (!(uNewLdt & X86_SEL_MASK_OFF_RPL))
+    {
+        /* Nested-guest SVM intercept. */
+        if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_LDTR_WRITES))
+        {
+            Log(("lldt: Guest intercept -> #VMEXIT\n"));
+            IEM_SVM_UPDATE_NRIP(pVCpu);
+            IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_LDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        Log(("lldt %04x: Loading NULL selector.\n", uNewLdt));
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_LDTR;
+        CPUMSetGuestLDTR(pVCpu, uNewLdt);
+        pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt;
+        pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+        if (IEM_IS_GUEST_CPU_AMD(pVCpu))
+        {
+            /* AMD-V seems to leave the base and limit alone. */
+            pVCpu->cpum.GstCtx.ldtr.Attr.u = X86DESCATTR_UNUSABLE;
+        }
+        else
+        {
+            /* VT-x (Intel 3960x) seems to be doing the following. */
+            pVCpu->cpum.GstCtx.ldtr.Attr.u   = X86DESCATTR_UNUSABLE | X86DESCATTR_G | X86DESCATTR_D;
+            pVCpu->cpum.GstCtx.ldtr.u64Base  = 0;
+            pVCpu->cpum.GstCtx.ldtr.u32Limit = UINT32_MAX;
+        }
+
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /*
+     * Read the descriptor.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR | CPUMCTX_EXTRN_GDTR);
+    IEMSELDESC Desc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uNewLdt, X86_XCPT_GP); /** @todo Correct exception? */
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Check GPs first. */
+    if (Desc.Legacy.Gen.u1DescType)
+    {
+        Log(("lldt %#x - not system selector (type %x) -> #GP\n", uNewLdt, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+    }
+    if (Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_LDT)
+    {
+        Log(("lldt %#x - not LDT selector (type %x) -> #GP\n", uNewLdt, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+    }
+    uint64_t u64Base;
+    if (!IEM_IS_LONG_MODE(pVCpu))
+        u64Base = X86DESC_BASE(&Desc.Legacy);
+    else
+    {
+        if (Desc.Long.Gen.u5Zeros)
+        {
+            Log(("lldt %#x - u5Zeros=%#x -> #GP\n", uNewLdt, Desc.Long.Gen.u5Zeros));
+            return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+        }
+
+        u64Base = X86DESC64_BASE(&Desc.Long);
+        if (!IEM_IS_CANONICAL(u64Base))
+        {
+            Log(("lldt %#x - non-canonical base address %#llx -> #GP\n", uNewLdt, u64Base));
+            return iemRaiseGeneralProtectionFault(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+        }
+    }
+
+    /* NP */
+    if (!Desc.Legacy.Gen.u1Present)
+    {
+        Log(("lldt %#x - segment not present -> #NP\n", uNewLdt));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewLdt);
+    }
+
+    /* Nested-guest SVM intercept. */
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_LDTR_WRITES))
+    {
+        Log(("lldt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_LDTR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * It checks out alright, update the registers.
+     */
+/** @todo check if the actual value is loaded or if the RPL is dropped */
+    CPUMSetGuestLDTR(pVCpu, uNewLdt & X86_SEL_MASK_OFF_RPL);
+    pVCpu->cpum.GstCtx.ldtr.ValidSel = uNewLdt & X86_SEL_MASK_OFF_RPL;
+    pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.ldtr.Attr.u   = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.ldtr.u32Limit = X86DESC_LIMIT_G(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.ldtr.u64Base  = u64Base;
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements sldt GReg
+ *
+ * @param   iGReg           The general register to store the CRx value in.
+ * @param   enmEffOpSize    The operand size.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_sldt_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
+{
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("sldt: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_SLDT, cbInstr);
+    }
+
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_LDTR_READS, SVM_EXIT_LDTR_READ, 0, 0);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT: *(uint16_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break;
+        case IEMMODE_32BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break;
+        case IEMMODE_64BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.ldtr.Sel; break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements sldt mem.
+ *
+ * @param   iEffSeg         The effective segment register to use with @a GCPtrMem.
+ * @param   GCPtrEffDst     Where to store the 16-bit CR0 value.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_sldt_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_LDTR_READS, SVM_EXIT_LDTR_READ, 0, 0);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR);
+    VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, pVCpu->cpum.GstCtx.ldtr.Sel);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements ltr.
+ *
+ * @param   uNewTr      The new TSS selector value.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_ltr, uint16_t, uNewTr)
+{
+    /*
+     * Check preconditions.
+     */
+    if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+    {
+        Log(("ltr %04x - real or v8086 mode -> #GP(0)\n", uNewTr));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("ltr %04x - CPL is %d -> #GP(0)\n", uNewTr, pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("ltr: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_LTR, cbInstr);
+    }
+    if (uNewTr & X86_SEL_LDT)
+    {
+        Log(("ltr %04x - LDT selector -> #GP\n", uNewTr));
+        return iemRaiseGeneralProtectionFaultBySelector(pVCpu, uNewTr);
+    }
+    if (!(uNewTr & X86_SEL_MASK_OFF_RPL))
+    {
+        Log(("ltr %04x - NULL selector -> #GP(0)\n", uNewTr));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_TR_WRITES))
+    {
+        Log(("ltr: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_TR_WRITE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Read the descriptor.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_LDTR | CPUMCTX_EXTRN_GDTR | CPUMCTX_EXTRN_TR);
+    IEMSELDESC Desc;
+    VBOXSTRICTRC rcStrict = iemMemFetchSelDesc(pVCpu, &Desc, uNewTr, X86_XCPT_GP); /** @todo Correct exception? */
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Check GPs first. */
+    if (Desc.Legacy.Gen.u1DescType)
+    {
+        Log(("ltr %#x - not system selector (type %x) -> #GP\n", uNewTr, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
+    }
+    if (   Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_386_TSS_AVAIL /* same as AMD64_SEL_TYPE_SYS_TSS_AVAIL */
+        && (   Desc.Legacy.Gen.u4Type != X86_SEL_TYPE_SYS_286_TSS_AVAIL
+            || IEM_IS_LONG_MODE(pVCpu)) )
+    {
+        Log(("ltr %#x - not an available TSS selector (type %x) -> #GP\n", uNewTr, Desc.Legacy.Gen.u4Type));
+        return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
+    }
+    uint64_t u64Base;
+    if (!IEM_IS_LONG_MODE(pVCpu))
+        u64Base = X86DESC_BASE(&Desc.Legacy);
+    else
+    {
+        if (Desc.Long.Gen.u5Zeros)
+        {
+            Log(("ltr %#x - u5Zeros=%#x -> #GP\n", uNewTr, Desc.Long.Gen.u5Zeros));
+            return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
+        }
+
+        u64Base = X86DESC64_BASE(&Desc.Long);
+        if (!IEM_IS_CANONICAL(u64Base))
+        {
+            Log(("ltr %#x - non-canonical base address %#llx -> #GP\n", uNewTr, u64Base));
+            return iemRaiseGeneralProtectionFault(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
+        }
+    }
+
+    /* NP */
+    if (!Desc.Legacy.Gen.u1Present)
+    {
+        Log(("ltr %#x - segment not present -> #NP\n", uNewTr));
+        return iemRaiseSelectorNotPresentBySelector(pVCpu, uNewTr);
+    }
+
+    /*
+     * Set it busy.
+     * Note! Intel says this should lock down the whole descriptor, but we'll
+     *       restrict our selves to 32-bit for now due to lack of inline
+     *       assembly and such.
+     */
+    void *pvDesc;
+    rcStrict = iemMemMap(pVCpu, &pvDesc, 8, UINT8_MAX, pVCpu->cpum.GstCtx.gdtr.pGdt + (uNewTr & X86_SEL_MASK_OFF_RPL),
+                         IEM_ACCESS_DATA_RW, 0);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    switch ((uintptr_t)pvDesc & 3)
+    {
+        case 0: ASMAtomicBitSet(pvDesc, 40 + 1); break;
+        case 1: ASMAtomicBitSet((uint8_t *)pvDesc + 3, 40 + 1 - 24); break;
+        case 2: ASMAtomicBitSet((uint8_t *)pvDesc + 2, 40 + 1 - 16); break;
+        case 3: ASMAtomicBitSet((uint8_t *)pvDesc + 1, 40 + 1 -  8); break;
+    }
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvDesc, IEM_ACCESS_DATA_RW);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    Desc.Legacy.Gen.u4Type |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
+
+    /*
+     * It checks out alright, update the registers.
+     */
+/** @todo check if the actual value is loaded or if the RPL is dropped */
+    CPUMSetGuestTR(pVCpu, uNewTr & X86_SEL_MASK_OFF_RPL);
+    pVCpu->cpum.GstCtx.tr.ValidSel = uNewTr & X86_SEL_MASK_OFF_RPL;
+    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.tr.Attr.u   = X86DESC_GET_HID_ATTR(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.tr.u32Limit = X86DESC_LIMIT_G(&Desc.Legacy);
+    pVCpu->cpum.GstCtx.tr.u64Base  = u64Base;
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements str GReg
+ *
+ * @param   iGReg           The general register to store the CRx value in.
+ * @param   enmEffOpSize    The operand size.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_str_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
+{
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("str_reg: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_STR, cbInstr);
+    }
+
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_TR_READS, SVM_EXIT_TR_READ, 0, 0);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT: *(uint16_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break;
+        case IEMMODE_32BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break;
+        case IEMMODE_64BIT: *(uint64_t *)iemGRegRef(pVCpu, iGReg) = pVCpu->cpum.GstCtx.tr.Sel; break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements str mem.
+ *
+ * @param   iEffSeg         The effective segment register to use with @a GCPtrMem.
+ * @param   GCPtrEffDst     Where to store the 16-bit CR0 value.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_str_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_DESC_TABLE_EXIT))
+    {
+        Log(("str_mem: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_LDTR_TR_ACCESS, VMXINSTRID_STR, cbInstr);
+    }
+
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_TR_READS, SVM_EXIT_TR_READ, 0, 0);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TR);
+    VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, pVCpu->cpum.GstCtx.tr.Sel);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements mov GReg,CRx.
+ *
+ * @param   iGReg           The general register to store the CRx value in.
+ * @param   iCrReg          The CRx register to read (valid).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Cd, uint8_t, iGReg, uint8_t, iCrReg)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+
+    if (IEM_SVM_IS_READ_CR_INTERCEPT_SET(pVCpu, iCrReg))
+    {
+        Log(("iemCImpl_mov_Rd_Cd: Guest intercept CR%u -> #VMEXIT\n", iCrReg));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_READ_CR0 + iCrReg, IEMACCESSCRX_MOV_CRX, iGReg);
+    }
+
+    /* Read it. */
+    uint64_t crX;
+    switch (iCrReg)
+    {
+        case 0:
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+            crX = pVCpu->cpum.GstCtx.cr0;
+            if (IEM_GET_TARGET_CPU(pVCpu) <= IEMTARGETCPU_386)
+                crX |= UINT32_C(0x7fffffe0); /* All reserved CR0 flags are set on a 386, just like MSW on 286. */
+            break;
+        case 2:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR2);
+            crX = pVCpu->cpum.GstCtx.cr2;
+            break;
+        case 3:
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+            crX = pVCpu->cpum.GstCtx.cr3;
+            break;
+        case 4:
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+            crX = pVCpu->cpum.GstCtx.cr4;
+            break;
+        case 8:
+        {
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+            {
+                VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovFromCr8(pVCpu, iGReg, cbInstr);
+                if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+                    return rcStrict;
+
+                /*
+                 * If the Mov-from-CR8 doesn't cause a VM-exit, bits 7:4 of the VTPR is copied
+                 * to bits 0:3 of the destination operand. Bits 63:4 of the destination operand
+                 * are cleared.
+                 *
+                 * See Intel Spec. 29.3 "Virtualizing CR8-based TPR Accesses"
+                 */
+                if (IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_USE_TPR_SHADOW))
+                {
+                    uint32_t const uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
+                    crX = (uTpr >> 4) & 0xf;
+                    break;
+                }
+            }
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+            if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
+            {
+                PCSVMVMCBCTRL pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl;
+                if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, IEM_GET_CTX(pVCpu)))
+                {
+                    crX = pVmcbCtrl->IntCtrl.n.u8VTPR & 0xf;
+                    break;
+                }
+            }
+#endif
+            uint8_t uTpr;
+            int rc = APICGetTpr(pVCpu, &uTpr, NULL, NULL);
+            if (RT_SUCCESS(rc))
+                crX = uTpr >> 4;
+            else
+                crX = 0;
+            break;
+        }
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        switch (iCrReg)
+        {
+            /* CR0/CR4 reads are subject to masking when in VMX non-root mode. */
+            case 0: crX = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u); break;
+            case 4: crX = CPUMGetGuestVmxMaskedCr4(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr4Mask.u); break;
+
+            case 3:
+            {
+                VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovFromCr3(pVCpu, iGReg, cbInstr);
+                if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+                    return rcStrict;
+                break;
+            }
+        }
+    }
+#endif
+
+    /* Store it. */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        *(uint64_t *)iemGRegRef(pVCpu, iGReg) = crX;
+    else
+        *(uint64_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)crX;
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements smsw GReg.
+ *
+ * @param   iGReg           The general register to store the CRx value in.
+ * @param   enmEffOpSize    The operand size.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_smsw_reg, uint8_t, iGReg, uint8_t, enmEffOpSize)
+{
+    IEM_SVM_CHECK_READ_CR0_INTERCEPT(pVCpu, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    uint64_t u64MaskedCr0;
+    if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        u64MaskedCr0 = pVCpu->cpum.GstCtx.cr0;
+    else
+        u64MaskedCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u);
+    uint64_t const u64GuestCr0 = u64MaskedCr0;
+#else
+    uint64_t const u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
+#endif
+
+    switch (enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
+                *(uint16_t *)iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0;
+            else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
+                *(uint16_t *)iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0 | 0xffe0;
+            else
+                *(uint16_t *)iemGRegRef(pVCpu, iGReg) = (uint16_t)u64GuestCr0 | 0xfff0;
+            break;
+
+        case IEMMODE_32BIT:
+            *(uint32_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)u64GuestCr0;
+            break;
+
+        case IEMMODE_64BIT:
+            *(uint64_t *)iemGRegRef(pVCpu, iGReg) = u64GuestCr0;
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements smsw mem.
+ *
+ * @param   iEffSeg         The effective segment register to use with @a GCPtrMem.
+ * @param   GCPtrEffDst     Where to store the 16-bit CR0 value.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_smsw_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    IEM_SVM_CHECK_READ_CR0_INTERCEPT(pVCpu, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    uint64_t u64MaskedCr0;
+    if (!IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        u64MaskedCr0 = pVCpu->cpum.GstCtx.cr0;
+    else
+        u64MaskedCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u);
+    uint64_t const u64GuestCr0 = u64MaskedCr0;
+#else
+    uint64_t const u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
+#endif
+
+    uint16_t u16Value;
+    if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
+        u16Value = (uint16_t)u64GuestCr0;
+    else if (IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_386)
+        u16Value = (uint16_t)u64GuestCr0 | 0xffe0;
+    else
+        u16Value = (uint16_t)u64GuestCr0 | 0xfff0;
+
+    VBOXSTRICTRC rcStrict = iemMemStoreDataU16(pVCpu, iEffSeg, GCPtrEffDst, u16Value);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Helper for mapping CR3 and PAE PDPEs for 'mov CRx,GReg'.
+ */
+#define IEM_MAP_PAE_PDPES_AT_CR3_RET(a_pVCpu, a_iCrReg, a_uCr3) \
+    do \
+    { \
+        int const rcX = PGMGstMapPaePdpesAtCr3(a_pVCpu, a_uCr3); \
+        if (RT_SUCCESS(rcX)) \
+        { /* likely */ } \
+        else \
+        { \
+            /* Either invalid PDPTEs or CR3 second-level translation failed. Raise #GP(0) either way. */ \
+            Log(("iemCImpl_load_Cr%#x: Trying to load invalid PAE PDPEs\n", a_iCrReg)); \
+            return iemRaiseGeneralProtectionFault0(a_pVCpu); \
+        } \
+    } while (0)
+
+
+/**
+ * Used to implemented 'mov CRx,GReg' and 'lmsw r/m16'.
+ *
+ * @param   iCrReg          The CRx register to write (valid).
+ * @param   uNewCrX         The new value.
+ * @param   enmAccessCrX    The instruction that caused the CrX load.
+ * @param   iGReg           The general register in case of a 'mov CRx,GReg'
+ *                          instruction.
+ */
+IEM_CIMPL_DEF_4(iemCImpl_load_CrX, uint8_t, iCrReg, uint64_t, uNewCrX, IEMACCESSCRX, enmAccessCrX, uint8_t, iGReg)
+{
+    VBOXSTRICTRC    rcStrict;
+    int             rc;
+#ifndef VBOX_WITH_NESTED_HWVIRT_SVM
+    RT_NOREF2(iGReg, enmAccessCrX);
+#endif
+
+    /*
+     * Try store it.
+     * Unfortunately, CPUM only does a tiny bit of the work.
+     */
+    switch (iCrReg)
+    {
+        case 0:
+        {
+            /*
+             * Perform checks.
+             */
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+
+            uint64_t const uOldCrX = pVCpu->cpum.GstCtx.cr0;
+            uint32_t const fValid  = CPUMGetGuestCR0ValidMask();
+
+            /* ET is hardcoded on 486 and later. */
+            if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_486)
+                uNewCrX |= X86_CR0_ET;
+            /* The 386 and 486 didn't #GP(0) on attempting to set reserved CR0 bits. ET was settable on 386. */
+            else if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_486)
+            {
+                uNewCrX &= fValid;
+                uNewCrX |= X86_CR0_ET;
+            }
+            else
+                uNewCrX &= X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG | X86_CR0_ET;
+
+            /* Check for reserved bits. */
+            if (uNewCrX & ~(uint64_t)fValid)
+            {
+                Log(("Trying to set reserved CR0 bits: NewCR0=%#llx InvalidBits=%#llx\n", uNewCrX, uNewCrX & ~(uint64_t)fValid));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            /* Check for invalid combinations. */
+            if (    (uNewCrX & X86_CR0_PG)
+                && !(uNewCrX & X86_CR0_PE) )
+            {
+                Log(("Trying to set CR0.PG without CR0.PE\n"));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            if (   !(uNewCrX & X86_CR0_CD)
+                && (uNewCrX & X86_CR0_NW) )
+            {
+                Log(("Trying to clear CR0.CD while leaving CR0.NW set\n"));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            if (   !(uNewCrX & X86_CR0_PG)
+                && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCIDE))
+            {
+                Log(("Trying to clear CR0.PG while leaving CR4.PCID set\n"));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            /* Long mode consistency checks. */
+            if (    (uNewCrX & X86_CR0_PG)
+                && !(uOldCrX & X86_CR0_PG)
+                &&  (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME) )
+            {
+                if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE))
+                {
+                    Log(("Trying to enabled long mode paging without CR4.PAE set\n"));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+                if (pVCpu->cpum.GstCtx.cs.Attr.n.u1Long)
+                {
+                    Log(("Trying to enabled long mode paging with a long CS descriptor loaded.\n"));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+            }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            /* Check for bits that must remain set or cleared in VMX operation,
+               see Intel spec. 23.8 "Restrictions on VMX operation". */
+            if (IEM_VMX_IS_ROOT_MODE(pVCpu))
+            {
+                uint64_t const uCr0Fixed0 = iemVmxGetCr0Fixed0(pVCpu, IEM_VMX_IS_NON_ROOT_MODE(pVCpu));
+                if ((uNewCrX & uCr0Fixed0) != uCr0Fixed0)
+                {
+                    Log(("Trying to clear reserved CR0 bits in VMX operation: NewCr0=%#llx MB1=%#llx\n", uNewCrX, uCr0Fixed0));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+
+                uint64_t const uCr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+                if (uNewCrX & ~uCr0Fixed1)
+                {
+                    Log(("Trying to set reserved CR0 bits in VMX operation: NewCr0=%#llx MB0=%#llx\n", uNewCrX, uCr0Fixed1));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+            }
+#endif
+
+            /*
+             * SVM nested-guest CR0 write intercepts.
+             */
+            if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, iCrReg))
+            {
+                Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
+                IEM_SVM_UPDATE_NRIP(pVCpu);
+                IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR0, enmAccessCrX, iGReg);
+            }
+            if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
+            {
+                /* 'lmsw' intercepts regardless of whether the TS/MP bits are actually toggled. */
+                if (   enmAccessCrX == IEMACCESSCRX_LMSW
+                    || (uNewCrX & ~(X86_CR0_TS | X86_CR0_MP)) != (uOldCrX & ~(X86_CR0_TS | X86_CR0_MP)))
+                {
+                    Assert(enmAccessCrX != IEMACCESSCRX_CLTS);
+                    Log(("iemCImpl_load_Cr%#x: lmsw or bits other than TS/MP changed: Guest intercept -> #VMEXIT\n", iCrReg));
+                    IEM_SVM_UPDATE_NRIP(pVCpu);
+                    IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_CR0_SEL_WRITE, enmAccessCrX, iGReg);
+                }
+            }
+
+            /*
+             * Change EFER.LMA if entering or leaving long mode.
+             */
+            uint64_t NewEFER = pVCpu->cpum.GstCtx.msrEFER;
+            if (   (uNewCrX & X86_CR0_PG) != (uOldCrX & X86_CR0_PG)
+                && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME) )
+            {
+                if (uNewCrX & X86_CR0_PG)
+                    NewEFER |= MSR_K6_EFER_LMA;
+                else
+                    NewEFER &= ~MSR_K6_EFER_LMA;
+
+                CPUMSetGuestEFER(pVCpu, NewEFER);
+                Assert(pVCpu->cpum.GstCtx.msrEFER == NewEFER);
+            }
+
+            /*
+             * Inform PGM.
+             */
+            if (    (uNewCrX & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE | X86_CR0_CD | X86_CR0_NW))
+                !=  (uOldCrX & (X86_CR0_PG | X86_CR0_WP | X86_CR0_PE | X86_CR0_CD | X86_CR0_NW)) )
+            {
+                if (    enmAccessCrX != IEMACCESSCRX_MOV_CRX
+                    || !CPUMIsPaePagingEnabled(uNewCrX, pVCpu->cpum.GstCtx.cr4, NewEFER)
+                    ||  CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
+                { /* likely */ }
+                else
+                    IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, pVCpu->cpum.GstCtx.cr3);
+                rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
+                AssertRCReturn(rc, rc);
+                /* ignore informational status codes */
+            }
+
+            /*
+             * Change CR0.
+             */
+            CPUMSetGuestCR0(pVCpu, uNewCrX);
+            Assert(pVCpu->cpum.GstCtx.cr0 == uNewCrX);
+
+            rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
+                                     false /* fForce */);
+            break;
+        }
+
+        /*
+         * CR2 can be changed without any restrictions.
+         */
+        case 2:
+        {
+            if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 2))
+            {
+                Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
+                IEM_SVM_UPDATE_NRIP(pVCpu);
+                IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR2, enmAccessCrX, iGReg);
+            }
+            pVCpu->cpum.GstCtx.cr2 = uNewCrX;
+            pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_CR2;
+            rcStrict  = VINF_SUCCESS;
+            break;
+        }
+
+        /*
+         * CR3 is relatively simple, although AMD and Intel have different
+         * accounts of how setting reserved bits are handled.  We take intel's
+         * word for the lower bits and AMD's for the high bits (63:52).  The
+         * lower reserved bits are ignored and left alone; OpenBSD 5.8 relies
+         * on this.
+         */
+        /** @todo Testcase: Setting reserved bits in CR3, especially before
+         *        enabling paging. */
+        case 3:
+        {
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+
+            /* Bit 63 being clear in the source operand with PCIDE indicates no invalidations are required. */
+            if (   (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCIDE)
+                && (uNewCrX & RT_BIT_64(63)))
+            {
+                /** @todo r=ramshankar: avoiding a TLB flush altogether here causes Windows 10
+                 *        SMP(w/o nested-paging) to hang during bootup on Skylake systems, see
+                 *        Intel spec. 4.10.4.1 "Operations that Invalidate TLBs and
+                 *        Paging-Structure Caches". */
+                uNewCrX &= ~RT_BIT_64(63);
+            }
+
+            /* Check / mask the value. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            /* See Intel spec. 27.2.2 "EPT Translation Mechanism" footnote. */
+            uint64_t const fInvPhysMask = !CPUMIsGuestVmxEptPagingEnabledEx(IEM_GET_CTX(pVCpu))
+                                        ? (UINT64_MAX << IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth)
+                                        : (~X86_CR3_EPT_PAGE_MASK & X86_PAGE_4K_BASE_MASK);
+#else
+            uint64_t const fInvPhysMask = UINT64_C(0xfff0000000000000);
+#endif
+            if (uNewCrX & fInvPhysMask)
+            {
+                /** @todo Should we raise this only for 64-bit mode like Intel claims? AMD is
+                 *        very vague in this area. As mentioned above, need testcase on real
+                 *        hardware... Sigh. */
+                Log(("Trying to load CR3 with invalid high bits set: %#llx\n", uNewCrX));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            uint64_t fValid;
+            if (   (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PAE)
+                && (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME))
+            {
+                /** @todo Redundant? This value has already been validated above. */
+                fValid = UINT64_C(0x000fffffffffffff);
+            }
+            else
+                fValid = UINT64_C(0xffffffff);
+            if (uNewCrX & ~fValid)
+            {
+                Log(("Automatically clearing reserved MBZ bits in CR3 load: NewCR3=%#llx ClearedBits=%#llx\n",
+                     uNewCrX, uNewCrX & ~fValid));
+                uNewCrX &= fValid;
+            }
+
+            if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 3))
+            {
+                Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
+                IEM_SVM_UPDATE_NRIP(pVCpu);
+                IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR3, enmAccessCrX, iGReg);
+            }
+
+            /* Inform PGM. */
+            if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PG)
+            {
+                if (   !CPUMIsGuestInPAEModeEx(IEM_GET_CTX(pVCpu))
+                    ||  CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
+                { /* likely */ }
+                else
+                {
+                    Assert(enmAccessCrX == IEMACCESSCRX_MOV_CRX);
+                    IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, uNewCrX);
+                }
+                rc = PGMFlushTLB(pVCpu, uNewCrX, !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PGE));
+                AssertRCReturn(rc, rc);
+                /* ignore informational status codes */
+            }
+
+            /* Make the change. */
+            rc = CPUMSetGuestCR3(pVCpu, uNewCrX);
+            AssertRCSuccessReturn(rc, rc);
+
+            rcStrict = VINF_SUCCESS;
+            break;
+        }
+
+        /*
+         * CR4 is a bit more tedious as there are bits which cannot be cleared
+         * under some circumstances and such.
+         */
+        case 4:
+        {
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+            uint64_t const uOldCrX = pVCpu->cpum.GstCtx.cr4;
+
+            /* Reserved bits. */
+            uint32_t const fValid = CPUMGetGuestCR4ValidMask(pVCpu->CTX_SUFF(pVM));
+            if (uNewCrX & ~(uint64_t)fValid)
+            {
+                Log(("Trying to set reserved CR4 bits: NewCR4=%#llx InvalidBits=%#llx\n", uNewCrX, uNewCrX & ~(uint64_t)fValid));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            bool const fPcide    = !(uOldCrX & X86_CR4_PCIDE) && (uNewCrX & X86_CR4_PCIDE);
+            bool const fLongMode = CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu));
+
+            /* PCIDE check. */
+            if (   fPcide
+                && (   !fLongMode
+                    || (pVCpu->cpum.GstCtx.cr3 & UINT64_C(0xfff))))
+            {
+                Log(("Trying to set PCIDE with invalid PCID or outside long mode. Pcid=%#x\n", (pVCpu->cpum.GstCtx.cr3 & UINT64_C(0xfff))));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            /* PAE check. */
+            if (   fLongMode
+                && (uOldCrX & X86_CR4_PAE)
+                && !(uNewCrX & X86_CR4_PAE))
+            {
+                Log(("Trying to set clear CR4.PAE while long mode is active\n"));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 4))
+            {
+                Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
+                IEM_SVM_UPDATE_NRIP(pVCpu);
+                IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR4, enmAccessCrX, iGReg);
+            }
+
+            /* Check for bits that must remain set or cleared in VMX operation,
+               see Intel spec. 23.8 "Restrictions on VMX operation". */
+            if (IEM_VMX_IS_ROOT_MODE(pVCpu))
+            {
+                uint64_t const uCr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+                if ((uNewCrX & uCr4Fixed0) != uCr4Fixed0)
+                {
+                    Log(("Trying to clear reserved CR4 bits in VMX operation: NewCr4=%#llx MB1=%#llx\n", uNewCrX, uCr4Fixed0));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+
+                uint64_t const uCr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+                if (uNewCrX & ~uCr4Fixed1)
+                {
+                    Log(("Trying to set reserved CR4 bits in VMX operation: NewCr4=%#llx MB0=%#llx\n", uNewCrX, uCr4Fixed1));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+            }
+
+            /*
+             * Notify PGM.
+             */
+            if ((uNewCrX ^ uOldCrX) & (X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_PCIDE /* | X86_CR4_SMEP */))
+            {
+                if (   !CPUMIsPaePagingEnabled(pVCpu->cpum.GstCtx.cr0, uNewCrX, pVCpu->cpum.GstCtx.msrEFER)
+                    || CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
+                { /* likely */ }
+                else
+                {
+                    Assert(enmAccessCrX == IEMACCESSCRX_MOV_CRX);
+                    IEM_MAP_PAE_PDPES_AT_CR3_RET(pVCpu, iCrReg, pVCpu->cpum.GstCtx.cr3);
+                }
+                rc = PGMFlushTLB(pVCpu, pVCpu->cpum.GstCtx.cr3, true /* global */);
+                AssertRCReturn(rc, rc);
+                /* ignore informational status codes */
+            }
+
+            /*
+             * Change it.
+             */
+            rc = CPUMSetGuestCR4(pVCpu, uNewCrX);
+            AssertRCSuccessReturn(rc, rc);
+            Assert(pVCpu->cpum.GstCtx.cr4 == uNewCrX);
+
+            rcStrict = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
+                                     false /* fForce */);
+            break;
+        }
+
+        /*
+         * CR8 maps to the APIC TPR.
+         */
+        case 8:
+        {
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
+            if (uNewCrX & ~(uint64_t)0xf)
+            {
+                Log(("Trying to set reserved CR8 bits (%#RX64)\n", uNewCrX));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+                && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_USE_TPR_SHADOW))
+            {
+                /*
+                 * If the Mov-to-CR8 doesn't cause a VM-exit, bits 0:3 of the source operand
+                 * is copied to bits 7:4 of the VTPR. Bits 0:3 and bits 31:8 of the VTPR are
+                 * cleared. Following this the processor performs TPR virtualization.
+                 *
+                 * However, we should not perform TPR virtualization immediately here but
+                 * after this instruction has completed.
+                 *
+                 * See Intel spec. 29.3 "Virtualizing CR8-based TPR Accesses"
+                 * See Intel spec. 27.1 "Architectural State Before A VM-exit"
+                 */
+                uint32_t const uTpr = (uNewCrX & 0xf) << 4;
+                Log(("iemCImpl_load_Cr%#x: Virtualizing TPR (%#x) write\n", iCrReg, uTpr));
+                iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_TPR, uTpr);
+                iemVmxVirtApicSetPendingWrite(pVCpu, XAPIC_OFF_TPR);
+                rcStrict = VINF_SUCCESS;
+                break;
+            }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+            if (CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)))
+            {
+                if (IEM_SVM_IS_WRITE_CR_INTERCEPT_SET(pVCpu, /*cr*/ 8))
+                {
+                    Log(("iemCImpl_load_Cr%#x: Guest intercept -> #VMEXIT\n", iCrReg));
+                    IEM_SVM_UPDATE_NRIP(pVCpu);
+                    IEM_SVM_CRX_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_CR8, enmAccessCrX, iGReg);
+                }
+
+                pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.IntCtrl.n.u8VTPR = uNewCrX;
+                if (CPUMIsGuestSvmVirtIntrMasking(pVCpu, IEM_GET_CTX(pVCpu)))
+                {
+                    rcStrict = VINF_SUCCESS;
+                    break;
+                }
+            }
+#endif
+            uint8_t const u8Tpr = (uint8_t)uNewCrX << 4;
+            APICSetTpr(pVCpu, u8Tpr);
+            rcStrict = VINF_SUCCESS;
+            break;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
+    }
+
+    /*
+     * Advance the RIP on success.
+     */
+    if (RT_SUCCESS(rcStrict))
+    {
+        if (rcStrict != VINF_SUCCESS)
+            iemSetPassUpStatus(pVCpu, rcStrict);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Implements mov CRx,GReg.
+ *
+ * @param   iCrReg          The CRx register to write (valid).
+ * @param   iGReg           The general register to load the CRx value from.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_mov_Cd_Rd, uint8_t, iCrReg, uint8_t, iGReg)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+
+    /*
+     * Read the new value from the source register and call common worker.
+     */
+    uint64_t uNewCrX;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        uNewCrX = iemGRegFetchU64(pVCpu, iGReg);
+    else
+        uNewCrX = iemGRegFetchU32(pVCpu, iGReg);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = VINF_VMX_INTERCEPT_NOT_ACTIVE;
+        switch (iCrReg)
+        {
+            case 0:
+            case 4: rcStrict = iemVmxVmexitInstrMovToCr0Cr4(pVCpu, iCrReg, &uNewCrX, iGReg, cbInstr);   break;
+            case 3: rcStrict = iemVmxVmexitInstrMovToCr3(pVCpu, uNewCrX, iGReg, cbInstr);               break;
+            case 8: rcStrict = iemVmxVmexitInstrMovToCr8(pVCpu, iGReg, cbInstr);                        break;
+        }
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+    return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, iCrReg, uNewCrX, IEMACCESSCRX_MOV_CRX, iGReg);
+}
+
+
+/**
+ * Implements 'LMSW r/m16'
+ *
+ * @param   u16NewMsw       The new value.
+ * @param   GCPtrEffDst     The guest-linear address of the source operand in case
+ *                          of a memory operand. For register operand, pass
+ *                          NIL_RTGCPTR.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_lmsw, uint16_t, u16NewMsw, RTGCPTR, GCPtrEffDst)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* Check nested-guest VMX intercept and get updated MSW if there's no VM-exit. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = iemVmxVmexitInstrLmsw(pVCpu, pVCpu->cpum.GstCtx.cr0, &u16NewMsw, GCPtrEffDst, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#else
+    RT_NOREF_PV(GCPtrEffDst);
+#endif
+
+    /*
+     * Compose the new CR0 value and call common worker.
+     */
+    uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0  & ~(X86_CR0_MP | X86_CR0_EM | X86_CR0_TS);
+    uNewCr0 |= u16NewMsw & (X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS);
+    return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, /*cr*/ 0, uNewCr0, IEMACCESSCRX_LMSW, UINT8_MAX /* iGReg */);
+}
+
+
+/**
+ * Implements 'CLTS'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_clts)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+    uint64_t uNewCr0 = pVCpu->cpum.GstCtx.cr0;
+    uNewCr0 &= ~X86_CR0_TS;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = iemVmxVmexitInstrClts(pVCpu, cbInstr);
+        if (rcStrict == VINF_VMX_MODIFIES_BEHAVIOR)
+            uNewCr0 |= (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS);
+        else if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+    return IEM_CIMPL_CALL_4(iemCImpl_load_CrX, /*cr*/ 0, uNewCr0, IEMACCESSCRX_CLTS, UINT8_MAX /* iGReg */);
+}
+
+
+/**
+ * Implements mov GReg,DRx.
+ *
+ * @param   iGReg           The general register to store the DRx value in.
+ * @param   iDrReg          The DRx register to read (0-7).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Dd, uint8_t, iGReg, uint8_t, iDrReg)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Check nested-guest VMX intercept.
+     * Unlike most other intercepts, the Mov DRx intercept takes preceedence
+     * over CPL and CR4.DE and even DR4/DR5 checks.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovDrX(pVCpu, VMXINSTRID_MOV_FROM_DRX, iDrReg, iGReg, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+    /*
+     * Check preconditions.
+     */
+    /* Raise GPs. */
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
+
+    /** @todo \#UD in outside ring-0 too? */
+    if (iDrReg == 4 || iDrReg == 5)
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR4);
+        if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE)
+        {
+            Log(("mov r%u,dr%u: CR4.DE=1 -> #GP(0)\n", iGReg, iDrReg));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        iDrReg += 2;
+    }
+
+    /* Raise #DB if general access detect is enabled. */
+    if (pVCpu->cpum.GstCtx.dr[7] & X86_DR7_GD)
+    {
+        Log(("mov r%u,dr%u: DR7.GD=1 -> #DB\n", iGReg, iDrReg));
+        return iemRaiseDebugException(pVCpu);
+    }
+
+    /*
+     * Read the debug register and store it in the specified general register.
+     */
+    uint64_t drX;
+    switch (iDrReg)
+    {
+        case 0:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+            drX = pVCpu->cpum.GstCtx.dr[0];
+            break;
+        case 1:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+            drX = pVCpu->cpum.GstCtx.dr[1];
+            break;
+        case 2:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+            drX = pVCpu->cpum.GstCtx.dr[2];
+            break;
+        case 3:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+            drX = pVCpu->cpum.GstCtx.dr[3];
+            break;
+        case 6:
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
+            drX = pVCpu->cpum.GstCtx.dr[6];
+            drX |= X86_DR6_RA1_MASK;
+            drX &= ~X86_DR6_RAZ_MASK;
+            break;
+        case 7:
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
+            drX = pVCpu->cpum.GstCtx.dr[7];
+            drX |=X86_DR7_RA1_MASK;
+            drX &= ~X86_DR7_RAZ_MASK;
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* caller checks */
+    }
+
+    /** @todo SVM nested-guest intercept for DR8-DR15? */
+    /*
+     * Check for any SVM nested-guest intercepts for the DRx read.
+     */
+    if (IEM_SVM_IS_READ_DR_INTERCEPT_SET(pVCpu, iDrReg))
+    {
+        Log(("mov r%u,dr%u: Guest intercept -> #VMEXIT\n", iGReg, iDrReg));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_READ_DR0 + (iDrReg & 0xf),
+                           IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? (iGReg & 7) : 0, 0 /* uExitInfo2 */);
+    }
+
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        *(uint64_t *)iemGRegRef(pVCpu, iGReg) = drX;
+    else
+        *(uint64_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)drX;
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements mov DRx,GReg.
+ *
+ * @param   iDrReg          The DRx register to write (valid).
+ * @param   iGReg           The general register to load the DRx value from.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_mov_Dd_Rd, uint8_t, iDrReg, uint8_t, iGReg)
+{
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Check nested-guest VMX intercept.
+     * Unlike most other intercepts, the Mov DRx intercept takes preceedence
+     * over CPL and CR4.DE and even DR4/DR5 checks.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = iemVmxVmexitInstrMovDrX(pVCpu, VMXINSTRID_MOV_TO_DRX, iDrReg, iGReg, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+    /*
+     * Check preconditions.
+     */
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DR7);
+
+    if (iDrReg == 4 || iDrReg == 5)
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_CR4);
+        if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE)
+        {
+            Log(("mov dr%u,r%u: CR4.DE=1 -> #GP(0)\n", iDrReg, iGReg));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        iDrReg += 2;
+    }
+
+    /* Raise #DB if general access detect is enabled. */
+    /** @todo is \#DB/DR7.GD raised before any reserved high bits in DR7/DR6
+     *        \#GP? */
+    if (pVCpu->cpum.GstCtx.dr[7] & X86_DR7_GD)
+    {
+        Log(("mov dr%u,r%u: DR7.GD=1 -> #DB\n", iDrReg, iGReg));
+        return iemRaiseDebugException(pVCpu);
+    }
+
+    /*
+     * Read the new value from the source register.
+     */
+    uint64_t uNewDrX;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        uNewDrX = iemGRegFetchU64(pVCpu, iGReg);
+    else
+        uNewDrX = iemGRegFetchU32(pVCpu, iGReg);
+
+    /*
+     * Adjust it.
+     */
+    switch (iDrReg)
+    {
+        case 0:
+        case 1:
+        case 2:
+        case 3:
+            /* nothing to adjust */
+            break;
+
+        case 6:
+            if (uNewDrX & X86_DR6_MBZ_MASK)
+            {
+                Log(("mov dr%u,%#llx: DR6 high bits are not zero -> #GP(0)\n", iDrReg, uNewDrX));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+            uNewDrX |= X86_DR6_RA1_MASK;
+            uNewDrX &= ~X86_DR6_RAZ_MASK;
+            break;
+
+        case 7:
+            if (uNewDrX & X86_DR7_MBZ_MASK)
+            {
+                Log(("mov dr%u,%#llx: DR7 high bits are not zero -> #GP(0)\n", iDrReg, uNewDrX));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+            uNewDrX |= X86_DR7_RA1_MASK;
+            uNewDrX &= ~X86_DR7_RAZ_MASK;
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    /** @todo SVM nested-guest intercept for DR8-DR15? */
+    /*
+     * Check for any SVM nested-guest intercepts for the DRx write.
+     */
+    if (IEM_SVM_IS_WRITE_DR_INTERCEPT_SET(pVCpu, iDrReg))
+    {
+        Log2(("mov dr%u,r%u: Guest intercept -> #VMEXIT\n", iDrReg, iGReg));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_WRITE_DR0 + (iDrReg & 0xf),
+                              IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? (iGReg & 7) : 0, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Do the actual setting.
+     */
+    if (iDrReg < 4)
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3);
+    else if (iDrReg == 6)
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
+
+    int rc = CPUMSetGuestDRx(pVCpu, iDrReg, uNewDrX);
+    AssertRCSuccessReturn(rc, RT_SUCCESS_NP(rc) ? VERR_IEM_IPE_1 : rc);
+
+    /*
+     * Re-init hardware breakpoint summary if it was DR7 that got changed.
+     */
+    if (iDrReg == 7)
+    {
+        pVCpu->iem.s.fPendingInstructionBreakpoints = false;
+        pVCpu->iem.s.fPendingDataBreakpoints        = false;
+        pVCpu->iem.s.fPendingIoBreakpoints          = false;
+        iemInitPendingBreakpointsSlow(pVCpu);
+    }
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements mov GReg,TRx.
+ *
+ * @param   iGReg           The general register to store the
+ *                          TRx value in.
+ * @param   iTrReg          The TRx register to read (6/7).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_mov_Rd_Td, uint8_t, iGReg, uint8_t, iTrReg)
+{
+    /*
+     * Check preconditions. NB: This instruction is 386/486 only.
+     */
+
+    /* Raise GPs. */
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+
+    if (iTrReg < 6 || iTrReg > 7)
+    {
+        /** @todo Do Intel CPUs reject this or are the TRs aliased? */
+        Log(("mov r%u,tr%u: invalid register -> #GP(0)\n", iGReg, iTrReg));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Read the test register and store it in the specified general register.
+     * This is currently a dummy implementation that only exists to satisfy
+     * old debuggers like WDEB386 or OS/2 KDB which unconditionally read the
+     * TR6/TR7 registers. Software which actually depends on the TR values
+     * (different on 386/486) is exceedingly rare.
+     */
+    uint64_t trX;
+    switch (iTrReg)
+    {
+        case 6:
+            trX = 0;    /* Currently a dummy. */
+            break;
+        case 7:
+            trX = 0;    /* Currently a dummy. */
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* call checks */
+    }
+
+    *(uint64_t *)iemGRegRef(pVCpu, iGReg) = (uint32_t)trX;
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements mov TRx,GReg.
+ *
+ * @param   iTrReg          The TRx register to write (valid).
+ * @param   iGReg           The general register to load the TRx
+ *                          value from.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_mov_Td_Rd, uint8_t, iTrReg, uint8_t, iGReg)
+{
+    /*
+     * Check preconditions. NB: This instruction is 386/486 only.
+     */
+
+    /* Raise GPs. */
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+
+    if (iTrReg < 6 || iTrReg > 7)
+    {
+        /** @todo Do Intel CPUs reject this or are the TRs aliased? */
+        Log(("mov r%u,tr%u: invalid register -> #GP(0)\n", iGReg, iTrReg));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Read the new value from the source register.
+     */
+    uint64_t uNewTrX;
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        uNewTrX = iemGRegFetchU64(pVCpu, iGReg);
+    else
+        uNewTrX = iemGRegFetchU32(pVCpu, iGReg);
+
+    /*
+     * Here we would do the actual setting if this weren't a dummy implementation.
+     * This is currently a dummy implementation that only exists to prevent
+     * old debuggers like WDEB386 or OS/2 KDB from crashing.
+     */
+    RT_NOREF(uNewTrX);
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'INVLPG m'.
+ *
+ * @param   GCPtrPage       The effective address of the page to invalidate.
+ * @remarks Updates the RIP.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_invlpg, RTGCPTR, GCPtrPage)
+{
+    /* ring-0 only. */
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    Assert(!pVCpu->cpum.GstCtx.eflags.Bits.u1VM);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INVLPG_EXIT))
+    {
+        Log(("invlpg: Guest intercept (%RGp) -> VM-exit\n", GCPtrPage));
+        return iemVmxVmexitInstrInvlpg(pVCpu, GCPtrPage, cbInstr);
+    }
+#endif
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INVLPG))
+    {
+        Log(("invlpg: Guest intercept (%RGp) -> #VMEXIT\n", GCPtrPage));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_INVLPG,
+                              IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? GCPtrPage : 0, 0 /* uExitInfo2 */);
+    }
+
+    int rc = PGMInvalidatePage(pVCpu, GCPtrPage);
+    if (rc == VINF_SUCCESS)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    if (rc == VINF_PGM_SYNC_CR3)
+    {
+        iemSetPassUpStatus(pVCpu, rc);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
+    Log(("PGMInvalidatePage(%RGv) -> %Rrc\n", GCPtrPage, rc));
+    return rc;
+}
+
+
+/**
+ * Implements INVPCID.
+ *
+ * @param   iEffSeg              The segment of the invpcid descriptor.
+ * @param   GCPtrInvpcidDesc     The address of invpcid descriptor.
+ * @param   uInvpcidType         The invalidation type.
+ * @remarks Updates the RIP.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_invpcid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvpcidDesc, uint64_t, uInvpcidType)
+{
+    /*
+     * Check preconditions.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fInvpcid)
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    /* When in VMX non-root mode and INVPCID is not enabled, it results in #UD. */
+    if (    IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_INVPCID))
+    {
+        Log(("invpcid: Not enabled for nested-guest execution -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("invpcid: CPL != 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_IS_V86_MODE(pVCpu))
+    {
+        Log(("invpcid: v8086 mode -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Check nested-guest intercept.
+     *
+     * INVPCID causes a VM-exit if "enable INVPCID" and "INVLPG exiting" are
+     * both set. We have already checked the former earlier in this function.
+     *
+     * CPL and virtual-8086 mode checks take priority over this VM-exit.
+     * See Intel spec. "25.1.1 Relative Priority of Faults and VM Exits".
+     */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INVLPG_EXIT))
+    {
+        Log(("invpcid: Guest intercept -> #VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_NEEDS_INFO_RET(pVCpu, VMX_EXIT_INVPCID, VMXINSTRID_NONE, cbInstr);
+    }
+
+    if (uInvpcidType > X86_INVPCID_TYPE_MAX_VALID)
+    {
+        Log(("invpcid: invalid/unrecognized invpcid type %#RX64 -> #GP(0)\n", uInvpcidType));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
+
+    /*
+     * Fetch the invpcid descriptor from guest memory.
+     */
+    RTUINT128U uDesc;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU128(pVCpu, &uDesc, iEffSeg, GCPtrInvpcidDesc);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * Validate the descriptor.
+         */
+        if (uDesc.s.Lo > 0xfff)
+        {
+            Log(("invpcid: reserved bits set in invpcid descriptor %#RX64 -> #GP(0)\n", uDesc.s.Lo));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        RTGCUINTPTR64 const GCPtrInvAddr = uDesc.s.Hi;
+        uint8_t       const uPcid        = uDesc.s.Lo & UINT64_C(0xfff);
+        uint32_t      const uCr4         = pVCpu->cpum.GstCtx.cr4;
+        uint64_t      const uCr3         = pVCpu->cpum.GstCtx.cr3;
+        switch (uInvpcidType)
+        {
+            case X86_INVPCID_TYPE_INDV_ADDR:
+            {
+                if (!IEM_IS_CANONICAL(GCPtrInvAddr))
+                {
+                    Log(("invpcid: invalidation address %#RGP is not canonical -> #GP(0)\n", GCPtrInvAddr));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+                if (  !(uCr4 & X86_CR4_PCIDE)
+                    && uPcid != 0)
+                {
+                    Log(("invpcid: invalid pcid %#x\n", uPcid));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+
+                /* Invalidate mappings for the linear address tagged with PCID except global translations. */
+                PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
+                break;
+            }
+
+            case X86_INVPCID_TYPE_SINGLE_CONTEXT:
+            {
+                if (  !(uCr4 & X86_CR4_PCIDE)
+                    && uPcid != 0)
+                {
+                    Log(("invpcid: invalid pcid %#x\n", uPcid));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+                /* Invalidate all mappings associated with PCID except global translations. */
+                PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
+                break;
+            }
+
+            case X86_INVPCID_TYPE_ALL_CONTEXT_INCL_GLOBAL:
+            {
+                PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+                break;
+            }
+
+            case X86_INVPCID_TYPE_ALL_CONTEXT_EXCL_GLOBAL:
+            {
+                PGMFlushTLB(pVCpu, uCr3, false /* fGlobal */);
+                break;
+            }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements INVD.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_invd)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("invd: CPL != 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_INVD, cbInstr);
+
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_INVD, SVM_EXIT_INVD, 0, 0);
+
+    /* We currently take no action here. */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements WBINVD.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_wbinvd)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("wbinvd: CPL != 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_WBINVD, cbInstr);
+
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_WBINVD, SVM_EXIT_WBINVD, 0, 0);
+
+    /* We currently take no action here. */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/** Opcode 0x0f 0xaa. */
+IEM_CIMPL_DEF_0(iemCImpl_rsm)
+{
+    IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_RSM, SVM_EXIT_RSM, 0, 0);
+    NOREF(cbInstr);
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Implements RDTSC.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_rdtsc)
+{
+    /*
+     * Check preconditions.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fTsc)
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+        if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_TSD)
+        {
+            Log(("rdtsc: CR4.TSD and CPL=%u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+    }
+
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        Log(("rdtsc: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDTSC, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDTSC))
+    {
+        Log(("rdtsc: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDTSC, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Do the job.
+     */
+    uint64_t uTicks = TMCpuTickGet(pVCpu);
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+    uTicks = CPUMApplyNestedGuestTscOffset(pVCpu, uTicks);
+#endif
+    pVCpu->cpum.GstCtx.rax = RT_LO_U32(uTicks);
+    pVCpu->cpum.GstCtx.rdx = RT_HI_U32(uTicks);
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX); /* For IEMExecDecodedRdtsc. */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements RDTSC.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_rdtscp)
+{
+    /*
+     * Check preconditions.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fRdTscP)
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    if (    IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_RDTSCP))
+    {
+        Log(("rdtscp: Not enabled for VMX non-root mode -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+        if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_TSD)
+        {
+            Log(("rdtscp: CR4.TSD and CPL=%u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+    }
+
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        Log(("rdtscp: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDTSCP, cbInstr);
+    }
+    else if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDTSCP))
+    {
+        Log(("rdtscp: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDTSCP, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Do the job.
+     * Query the MSR first in case of trips to ring-3.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
+    VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pVCpu->cpum.GstCtx.rcx);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /* Low dword of the TSC_AUX msr only. */
+        pVCpu->cpum.GstCtx.rcx &= UINT32_C(0xffffffff);
+
+        uint64_t uTicks = TMCpuTickGet(pVCpu);
+#if defined(VBOX_WITH_NESTED_HWVIRT_SVM) || defined(VBOX_WITH_NESTED_HWVIRT_VMX)
+        uTicks = CPUMApplyNestedGuestTscOffset(pVCpu, uTicks);
+#endif
+        pVCpu->cpum.GstCtx.rax = RT_LO_U32(uTicks);
+        pVCpu->cpum.GstCtx.rdx = RT_HI_U32(uTicks);
+        pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX); /* For IEMExecDecodedRdtscp. */
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements RDPMC.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_rdpmc)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+
+    if (   pVCpu->iem.s.uCpl != 0
+        && !(pVCpu->cpum.GstCtx.cr4 & X86_CR4_PCE))
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_RDPMC_EXIT))
+    {
+        Log(("rdpmc: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDPMC, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_RDPMC))
+    {
+        Log(("rdpmc: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_RDPMC, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /** @todo Emulate performance counters, for now just return 0. */
+    pVCpu->cpum.GstCtx.rax = 0;
+    pVCpu->cpum.GstCtx.rdx = 0;
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX);
+    /** @todo We should trigger a \#GP here if the CPU doesn't support the index in
+     *        ecx but see @bugref{3472}! */
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements RDMSR.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_rdmsr)
+{
+    /*
+     * Check preconditions.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMsr)
+        return iemRaiseUndefinedOpcode(pVCpu);
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    /*
+     * Check nested-guest intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (iemVmxIsRdmsrWrmsrInterceptSet(pVCpu, VMX_EXIT_RDMSR, pVCpu->cpum.GstCtx.ecx))
+            IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_RDMSR, cbInstr);
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT))
+    {
+        VBOXSTRICTRC rcStrict = iemSvmHandleMsrIntercept(pVCpu, pVCpu->cpum.GstCtx.ecx, false /* fWrite */);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("IEM: SVM intercepted rdmsr(%#x) failed. rc=%Rrc\n", pVCpu->cpum.GstCtx.ecx, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    /*
+     * Do the job.
+     */
+    RTUINT64U uValue;
+    /** @todo make CPUMAllMsrs.cpp import the necessary MSR state. */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
+
+    VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pVCpu->cpum.GstCtx.ecx, &uValue.u);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        pVCpu->cpum.GstCtx.rax = uValue.s.Lo;
+        pVCpu->cpum.GstCtx.rdx = uValue.s.Hi;
+        pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX);
+
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+#ifndef IN_RING3
+    /* Deferred to ring-3. */
+    if (rcStrict == VINF_CPUM_R3_MSR_READ)
+    {
+        Log(("IEM: rdmsr(%#x) -> ring-3\n", pVCpu->cpum.GstCtx.ecx));
+        return rcStrict;
+    }
+#endif
+
+    /* Often a unimplemented MSR or MSR bit, so worth logging. */
+    if (pVCpu->iem.s.cLogRelRdMsr < 32)
+    {
+        pVCpu->iem.s.cLogRelRdMsr++;
+        LogRel(("IEM: rdmsr(%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.ecx));
+    }
+    else
+        Log((   "IEM: rdmsr(%#x) -> #GP(0)\n", pVCpu->cpum.GstCtx.ecx));
+    AssertMsgReturn(rcStrict == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IPE_UNEXPECTED_STATUS);
+    return iemRaiseGeneralProtectionFault0(pVCpu);
+}
+
+
+/**
+ * Implements WRMSR.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_wrmsr)
+{
+    /*
+     * Check preconditions.
+     */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMsr)
+        return iemRaiseUndefinedOpcode(pVCpu);
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    RTUINT64U uValue;
+    uValue.s.Lo = pVCpu->cpum.GstCtx.eax;
+    uValue.s.Hi = pVCpu->cpum.GstCtx.edx;
+
+    uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
+
+    /** @todo make CPUMAllMsrs.cpp import the necessary MSR state. */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
+
+    /*
+     * Check nested-guest intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (iemVmxIsRdmsrWrmsrInterceptSet(pVCpu, VMX_EXIT_WRMSR, idMsr))
+            IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_WRMSR, cbInstr);
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MSR_PROT))
+    {
+        VBOXSTRICTRC rcStrict = iemSvmHandleMsrIntercept(pVCpu, idMsr, true /* fWrite */);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("IEM: SVM intercepted rdmsr(%#x) failed. rc=%Rrc\n", idMsr, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    /*
+     * Do the job.
+     */
+    VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, idMsr, uValue.u);
+    if (rcStrict == VINF_SUCCESS)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+#ifndef IN_RING3
+    /* Deferred to ring-3. */
+    if (rcStrict == VINF_CPUM_R3_MSR_WRITE)
+    {
+        Log(("IEM: wrmsr(%#x) -> ring-3\n", idMsr));
+        return rcStrict;
+    }
+#endif
+
+    /* Often a unimplemented MSR or MSR bit, so worth logging. */
+    if (pVCpu->iem.s.cLogRelWrMsr < 32)
+    {
+        pVCpu->iem.s.cLogRelWrMsr++;
+        LogRel(("IEM: wrmsr(%#x,%#x`%08x) -> #GP(0)\n", idMsr, uValue.s.Hi, uValue.s.Lo));
+    }
+    else
+        Log((   "IEM: wrmsr(%#x,%#x`%08x) -> #GP(0)\n", idMsr, uValue.s.Hi, uValue.s.Lo));
+    AssertMsgReturn(rcStrict == VERR_CPUM_RAISE_GP_0, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IPE_UNEXPECTED_STATUS);
+    return iemRaiseGeneralProtectionFault0(pVCpu);
+}
+
+
+/**
+ * Implements 'IN eAX, port'.
+ *
+ * @param   u16Port     The source port.
+ * @param   fImm        Whether the port was specified through an immediate operand
+ *                      or the implicit DX register.
+ * @param   cbReg       The register size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_in, uint16_t, u16Port, bool, fImm, uint8_t, cbReg)
+{
+    /*
+     * CPL check
+     */
+    VBOXSTRICTRC rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, cbReg);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Check VMX nested-guest IO intercept.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        rcStrict = iemVmxVmexitInstrIo(pVCpu, VMXINSTRID_IO_IN, u16Port, fImm, cbReg, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#else
+    RT_NOREF(fImm);
+#endif
+
+    /*
+     * Check SVM nested-guest IO intercept.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        uint8_t cAddrSizeBits;
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: cAddrSizeBits = 16; break;
+            case IEMMODE_32BIT: cAddrSizeBits = 32; break;
+            case IEMMODE_64BIT: cAddrSizeBits = 64; break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_IN, cbReg, cAddrSizeBits, 0 /* N/A - iEffSeg */,
+                                           false /* fRep */, false /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_in: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, cbReg,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    /*
+     * Perform the I/O.
+     */
+    PVMCC const pVM      = pVCpu->CTX_SUFF(pVM);
+    uint32_t    u32Value = 0;
+    rcStrict = IOMIOPortRead(pVM, pVCpu, u16Port, &u32Value, cbReg);
+    if (IOM_SUCCESS(rcStrict))
+    {
+        switch (cbReg)
+        {
+            case 1: pVCpu->cpum.GstCtx.al  = (uint8_t)u32Value;  break;
+            case 2: pVCpu->cpum.GstCtx.ax  = (uint16_t)u32Value; break;
+            case 4: pVCpu->cpum.GstCtx.rax = u32Value;           break;
+            default: AssertFailedReturn(VERR_IEM_IPE_3);
+        }
+
+        pVCpu->iem.s.cPotentialExits++;
+        if (rcStrict != VINF_SUCCESS)
+            iemSetPassUpStatus(pVCpu, rcStrict);
+
+        /*
+         * Check for I/O breakpoints before we complete the instruction.
+         */
+        uint32_t const fDr7 = pVCpu->cpum.GstCtx.dr[7];
+        if (RT_UNLIKELY(   (   (   (fDr7 & X86_DR7_ENABLED_MASK)
+                                && X86_DR7_ANY_RW_IO(fDr7)
+                                && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE))
+                            || pVM->dbgf.ro.cEnabledHwIoBreakpoints > 0)
+                        && rcStrict == VINF_SUCCESS))
+        {
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3 | CPUMCTX_EXTRN_DR6);
+            pVCpu->cpum.GstCtx.eflags.uBoth |= DBGFBpCheckIo2(pVM, pVCpu, u16Port, cbReg);
+        }
+
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'IN eAX, DX'.
+ *
+ * @param   cbReg       The register size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_in_eAX_DX, uint8_t, cbReg)
+{
+    return IEM_CIMPL_CALL_3(iemCImpl_in, pVCpu->cpum.GstCtx.dx, false /* fImm */, cbReg);
+}
+
+
+/**
+ * Implements 'OUT port, eAX'.
+ *
+ * @param   u16Port     The destination port.
+ * @param   fImm        Whether the port was specified through an immediate operand
+ *                      or the implicit DX register.
+ * @param   cbReg       The register size.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_out, uint16_t, u16Port, bool, fImm, uint8_t, cbReg)
+{
+    /*
+     * CPL check
+     */
+    VBOXSTRICTRC rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, cbReg);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Check VMX nested-guest I/O intercept.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        rcStrict = iemVmxVmexitInstrIo(pVCpu, VMXINSTRID_IO_OUT, u16Port, fImm, cbReg, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#else
+    RT_NOREF(fImm);
+#endif
+
+    /*
+     * Check SVM nested-guest I/O intercept.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        uint8_t cAddrSizeBits;
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: cAddrSizeBits = 16; break;
+            case IEMMODE_32BIT: cAddrSizeBits = 32; break;
+            case IEMMODE_64BIT: cAddrSizeBits = 64; break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_OUT, cbReg, cAddrSizeBits, 0 /* N/A - iEffSeg */,
+                                           false /* fRep */, false /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_out: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, cbReg,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    /*
+     * Perform the I/O.
+     */
+    PVMCC const pVM      = pVCpu->CTX_SUFF(pVM);
+    uint32_t u32Value;
+    switch (cbReg)
+    {
+        case 1: u32Value = pVCpu->cpum.GstCtx.al;  break;
+        case 2: u32Value = pVCpu->cpum.GstCtx.ax;  break;
+        case 4: u32Value = pVCpu->cpum.GstCtx.eax; break;
+        default: AssertFailedReturn(VERR_IEM_IPE_4);
+    }
+    rcStrict = IOMIOPortWrite(pVM, pVCpu, u16Port, u32Value, cbReg);
+    if (IOM_SUCCESS(rcStrict))
+    {
+        pVCpu->iem.s.cPotentialExits++;
+        if (rcStrict != VINF_SUCCESS)
+            iemSetPassUpStatus(pVCpu, rcStrict);
+
+        /*
+         * Check for I/O breakpoints before we complete the instruction.
+         */
+        uint32_t const fDr7 = pVCpu->cpum.GstCtx.dr[7];
+        if (RT_UNLIKELY(   (   (   (fDr7 & X86_DR7_ENABLED_MASK)
+                                && X86_DR7_ANY_RW_IO(fDr7)
+                                && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_DE))
+                            || pVM->dbgf.ro.cEnabledHwIoBreakpoints > 0)
+                        && rcStrict == VINF_SUCCESS))
+        {
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR0_DR3 | CPUMCTX_EXTRN_DR6);
+            pVCpu->cpum.GstCtx.eflags.uBoth |= DBGFBpCheckIo2(pVM, pVCpu, u16Port, cbReg);
+        }
+
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'OUT DX, eAX'.
+ *
+ * @param   cbReg       The register size.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_out_DX_eAX, uint8_t, cbReg)
+{
+    return IEM_CIMPL_CALL_3(iemCImpl_out, pVCpu->cpum.GstCtx.dx, false /* fImm */, cbReg);
+}
+
+
+/**
+ * Implements 'CLI'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_cli)
+{
+    uint32_t        fEfl    = IEMMISC_GET_EFL(pVCpu);
+#ifdef LOG_ENABLED
+    uint32_t const  fEflOld = fEfl;
+#endif
+
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4);
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
+    {
+        uint8_t const uIopl = X86_EFL_GET_IOPL(fEfl);
+        if (!(fEfl & X86_EFL_VM))
+        {
+            if (pVCpu->iem.s.uCpl <= uIopl)
+                fEfl &= ~X86_EFL_IF;
+            else if (   pVCpu->iem.s.uCpl == 3
+                     && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PVI) )
+                fEfl &= ~X86_EFL_VIF;
+            else
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        /* V8086 */
+        else if (uIopl == 3)
+            fEfl &= ~X86_EFL_IF;
+        else if (   uIopl < 3
+                 && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME) )
+            fEfl &= ~X86_EFL_VIF;
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    /* real mode */
+    else
+        fEfl &= ~X86_EFL_IF;
+
+    /* Commit. */
+    IEMMISC_SET_EFL(pVCpu, fEfl);
+    VBOXSTRICTRC const rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    Log2(("CLI: %#x -> %#x\n", fEflOld, fEfl));
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'STI'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_sti)
+{
+    uint32_t        fEfl    = IEMMISC_GET_EFL(pVCpu);
+    uint32_t const  fEflOld = fEfl;
+
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4);
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE)
+    {
+        uint8_t const uIopl = X86_EFL_GET_IOPL(fEfl);
+        if (!(fEfl & X86_EFL_VM))
+        {
+            if (pVCpu->iem.s.uCpl <= uIopl)
+                fEfl |= X86_EFL_IF;
+            else if (   pVCpu->iem.s.uCpl == 3
+                     && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_PVI)
+                     && !(fEfl & X86_EFL_VIP) )
+                fEfl |= X86_EFL_VIF;
+            else
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        /* V8086 */
+        else if (uIopl == 3)
+            fEfl |= X86_EFL_IF;
+        else if (   uIopl < 3
+                 && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_VME)
+                 && !(fEfl & X86_EFL_VIP) )
+            fEfl |= X86_EFL_VIF;
+        else
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    /* real mode */
+    else
+        fEfl |= X86_EFL_IF;
+
+    /*
+     * Commit.
+     *
+     * Note! Setting the shadow interrupt flag must be done after RIP updating.
+     */
+    IEMMISC_SET_EFL(pVCpu, fEfl);
+    VBOXSTRICTRC const rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    if (!(fEflOld & X86_EFL_IF) && (fEfl & X86_EFL_IF))
+    {
+        /** @todo only set it the shadow flag if it was clear before? */
+        CPUMSetInInterruptShadowSti(&pVCpu->cpum.GstCtx);
+    }
+    Log2(("STI: %#x -> %#x\n", fEflOld, fEfl));
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'HLT'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_hlt)
+{
+    if (pVCpu->iem.s.uCpl != 0)
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_HLT_EXIT))
+    {
+        Log2(("hlt: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_HLT, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_HLT))
+    {
+        Log2(("hlt: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_HLT, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /** @todo finish: This ASSUMES that iemRegAddToRipAndFinishingClearingRF won't
+     * be returning any status codes relating to non-guest events being raised, as
+     * we'll mess up the guest HALT otherwise.  */
+    VBOXSTRICTRC rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = VINF_EM_HALT;
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'MONITOR'.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_monitor, uint8_t, iEffSeg)
+{
+    /*
+     * Permission checks.
+     */
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log2(("monitor: CPL != 0\n"));
+        return iemRaiseUndefinedOpcode(pVCpu); /** @todo MSR[0xC0010015].MonMwaitUserEn if we care. */
+    }
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMonitorMWait)
+    {
+        Log2(("monitor: Not in CPUID\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    /*
+     * Check VMX guest-intercept.
+     * This should be considered a fault-like VM-exit.
+     * See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
+     */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_MONITOR_EXIT))
+    {
+        Log2(("monitor: Guest intercept -> #VMEXIT\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_MONITOR, cbInstr);
+    }
+
+    /*
+     * Gather the operands and validate them.
+     */
+    RTGCPTR  GCPtrMem   = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
+    uint32_t uEcx       = pVCpu->cpum.GstCtx.ecx;
+    uint32_t uEdx       = pVCpu->cpum.GstCtx.edx;
+/** @todo Test whether EAX or ECX is processed first, i.e. do we get \#PF or
+ *        \#GP first. */
+    if (uEcx != 0)
+    {
+        Log2(("monitor rax=%RX64, ecx=%RX32, edx=%RX32; ECX != 0 -> #GP(0)\n", GCPtrMem, uEcx, uEdx)); NOREF(uEdx);
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, iEffSeg, 1, &GCPtrMem);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    RTGCPHYS GCPhysMem;
+    /** @todo access size   */
+    rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrMem, 1, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, &GCPhysMem);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
+    {
+        /*
+         * MONITOR does not access the memory, just monitors the address. However,
+         * if the address falls in the APIC-access page, the address monitored must
+         * instead be the corresponding address in the virtual-APIC page.
+         *
+         * See Intel spec. 29.4.4 "Instruction-Specific Considerations".
+         */
+        rcStrict = iemVmxVirtApicAccessUnused(pVCpu, &GCPhysMem, 1, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA);
+        if (   rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE
+            && rcStrict != VINF_VMX_MODIFIES_BEHAVIOR)
+                return rcStrict;
+    }
+#endif
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MONITOR))
+    {
+        Log2(("monitor: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MONITOR, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Call EM to prepare the monitor/wait.
+     */
+    rcStrict = EMMonitorWaitPrepare(pVCpu, pVCpu->cpum.GstCtx.rax, pVCpu->cpum.GstCtx.rcx, pVCpu->cpum.GstCtx.rdx, GCPhysMem);
+    Assert(rcStrict == VINF_SUCCESS);
+    if (rcStrict == VINF_SUCCESS)
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'MWAIT'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_mwait)
+{
+    /*
+     * Permission checks.
+     */
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log2(("mwait: CPL != 0\n"));
+        /** @todo MSR[0xC0010015].MonMwaitUserEn if we care. (Remember to check
+         *        EFLAGS.VM then.) */
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMonitorMWait)
+    {
+        Log2(("mwait: Not in CPUID\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    /* Check VMX nested-guest intercept. */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_MWAIT_EXIT))
+        IEM_VMX_VMEXIT_MWAIT_RET(pVCpu, EMMonitorIsArmed(pVCpu), cbInstr);
+
+    /*
+     * Gather the operands and validate them.
+     */
+    uint32_t const uEax = pVCpu->cpum.GstCtx.eax;
+    uint32_t const uEcx = pVCpu->cpum.GstCtx.ecx;
+    if (uEcx != 0)
+    {
+        /* Only supported extension is break on IRQ when IF=0. */
+        if (uEcx > 1)
+        {
+            Log2(("mwait eax=%RX32, ecx=%RX32; ECX > 1 -> #GP(0)\n", uEax, uEcx));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+        uint32_t fMWaitFeatures = 0;
+        uint32_t uIgnore = 0;
+        CPUMGetGuestCpuId(pVCpu, 5, 0, -1 /*f64BitMode*/, &uIgnore, &uIgnore, &fMWaitFeatures, &uIgnore);
+        if (    (fMWaitFeatures & (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0))
+            !=                    (X86_CPUID_MWAIT_ECX_EXT | X86_CPUID_MWAIT_ECX_BREAKIRQIF0))
+        {
+            Log2(("mwait eax=%RX32, ecx=%RX32; break-on-IRQ-IF=0 extension not enabled -> #GP(0)\n", uEax, uEcx));
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        /*
+         * If the interrupt-window exiting control is set or a virtual-interrupt is pending
+         * for delivery; and interrupts are disabled the processor does not enter its
+         * mwait state but rather passes control to the next instruction.
+         *
+         * See Intel spec. 25.3 "Changes to Instruction Behavior In VMX Non-root Operation".
+         */
+        if (    IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+            && !pVCpu->cpum.GstCtx.eflags.Bits.u1IF)
+        {
+            if (   IEM_VMX_IS_PROCCTLS_SET(pVCpu, VMX_PROC_CTLS_INT_WINDOW_EXIT)
+                || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
+                /** @todo finish: check up this out after we move int window stuff out of the
+                 *        run loop and into the instruction finishing logic here. */
+                return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+#endif
+    }
+
+    /*
+     * Check SVM nested-guest mwait intercepts.
+     */
+    if (   IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MWAIT_ARMED)
+        && EMMonitorIsArmed(pVCpu))
+    {
+        Log2(("mwait: Guest intercept (monitor hardware armed) -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MWAIT_ARMED, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_MWAIT))
+    {
+        Log2(("mwait: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_MWAIT, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /*
+     * Call EM to prepare the monitor/wait.
+     *
+     * This will return VINF_EM_HALT. If there the trap flag is set, we may
+     * override it when executing iemRegAddToRipAndFinishingClearingRF ASSUMING
+     * that will only return guest related events.
+     */
+    VBOXSTRICTRC rcStrict = EMMonitorWaitPerform(pVCpu, uEax, uEcx);
+
+    /** @todo finish: This needs more thinking as we should suppress internal
+     * debugger events here, or we'll bugger up the guest state even more than we
+     * alread do around VINF_EM_HALT. */
+    VBOXSTRICTRC rcStrict2 = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    if (rcStrict2 != VINF_SUCCESS)
+    {
+        Log2(("mwait: %Rrc (perform) -> %Rrc (finish)!\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2) ));
+        rcStrict = rcStrict2;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'SWAPGS'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_swapgs)
+{
+    Assert(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT); /* Caller checks this. */
+
+    /*
+     * Permission checks.
+     */
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log2(("swapgs: CPL != 0\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+
+    /*
+     * Do the job.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_GS);
+    uint64_t uOtherGsBase = pVCpu->cpum.GstCtx.msrKERNELGSBASE;
+    pVCpu->cpum.GstCtx.msrKERNELGSBASE = pVCpu->cpum.GstCtx.gs.u64Base;
+    pVCpu->cpum.GstCtx.gs.u64Base = uOtherGsBase;
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
+/**
+ * Handles a CPUID call.
+ */
+static VBOXSTRICTRC iemCpuIdVBoxCall(PVMCPUCC pVCpu, uint32_t iFunction,
+                                     uint32_t *pEax, uint32_t *pEbx, uint32_t *pEcx, uint32_t *pEdx)
+{
+    switch (iFunction)
+    {
+        case VBOX_CPUID_FN_ID:
+            LogFlow(("iemCpuIdVBoxCall: VBOX_CPUID_FN_ID\n"));
+            *pEax = VBOX_CPUID_RESP_ID_EAX;
+            *pEbx = VBOX_CPUID_RESP_ID_EBX;
+            *pEcx = VBOX_CPUID_RESP_ID_ECX;
+            *pEdx = VBOX_CPUID_RESP_ID_EDX;
+            break;
+
+        case VBOX_CPUID_FN_LOG:
+        {
+            CPUM_IMPORT_EXTRN_RET(pVCpu, CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX | CPUMCTX_EXTRN_RSI
+                                       | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+
+            /* Validate input. */
+            uint32_t cchToLog = *pEdx;
+            if (cchToLog <= _2M)
+            {
+                uint32_t const uLogPicker = *pEbx;
+                if (uLogPicker <= 1)
+                {
+                    /* Resolve the logger. */
+                    PRTLOGGER const pLogger = !uLogPicker
+                                            ? RTLogDefaultInstanceEx(UINT32_MAX) : RTLogRelGetDefaultInstanceEx(UINT32_MAX);
+                    if (pLogger)
+                    {
+                        /* Copy over the data: */
+                        RTGCPTR GCPtrSrc = pVCpu->cpum.GstCtx.rsi;
+                        while (cchToLog > 0)
+                        {
+                            uint32_t cbToMap = GUEST_PAGE_SIZE - (GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
+                            if (cbToMap > cchToLog)
+                                cbToMap = cchToLog;
+                            /** @todo Extend iemMemMap to allowing page size accessing and avoid 7
+                             *        unnecessary calls & iterations per pages. */
+                            if (cbToMap > 512)
+                                cbToMap = 512;
+                            void        *pvSrc    = NULL;
+                            VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvSrc, cbToMap, UINT8_MAX, GCPtrSrc, IEM_ACCESS_DATA_R, 0);
+                            if (rcStrict == VINF_SUCCESS)
+                            {
+                                RTLogBulkNestedWrite(pLogger, (const char *)pvSrc, cbToMap, "Gst:");
+                                rcStrict = iemMemCommitAndUnmap(pVCpu, pvSrc, IEM_ACCESS_DATA_R);
+                                AssertRCSuccessReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
+                            }
+                            else
+                            {
+                                Log(("iemCpuIdVBoxCall: %Rrc at %RGp LB %#x\n", VBOXSTRICTRC_VAL(rcStrict), GCPtrSrc, cbToMap));
+                                return rcStrict;
+                            }
+
+                            /* Advance. */
+                            pVCpu->cpum.GstCtx.rsi = GCPtrSrc += cbToMap;
+                            *pEdx                  = cchToLog -= cbToMap;
+                        }
+                        *pEax = VINF_SUCCESS;
+                    }
+                    else
+                        *pEax = (uint32_t)VERR_NOT_FOUND;
+                }
+                else
+                    *pEax = (uint32_t)VERR_NOT_FOUND;
+            }
+            else
+                *pEax = (uint32_t)VERR_TOO_MUCH_DATA;
+            *pEdx = VBOX_CPUID_RESP_GEN_EDX;
+            *pEcx = VBOX_CPUID_RESP_GEN_ECX;
+            *pEbx = VBOX_CPUID_RESP_GEN_EBX;
+            break;
+        }
+
+        default:
+            LogFlow(("iemCpuIdVBoxCall: Invalid function %#x (%#x, %#x)\n", iFunction, *pEbx, *pEdx));
+            *pEax = (uint32_t)VERR_INVALID_FUNCTION;
+            *pEbx = (uint32_t)VERR_INVALID_FUNCTION;
+            *pEcx = (uint32_t)VERR_INVALID_FUNCTION;
+            *pEdx = (uint32_t)VERR_INVALID_FUNCTION;
+            break;
+    }
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_WITHOUT_CPUID_HOST_CALL */
+
+/**
+ * Implements 'CPUID'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_cpuid)
+{
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        Log2(("cpuid: Guest intercept -> VM-exit\n"));
+        IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_CPUID, cbInstr);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CPUID))
+    {
+        Log2(("cpuid: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_CPUID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+
+    uint32_t const uEax = pVCpu->cpum.GstCtx.eax;
+    uint32_t const uEcx = pVCpu->cpum.GstCtx.ecx;
+
+#ifndef VBOX_WITHOUT_CPUID_HOST_CALL
+    /*
+     * CPUID host call backdoor.
+     */
+    if (   uEax == VBOX_CPUID_REQ_EAX_FIXED
+        && (uEcx & VBOX_CPUID_REQ_ECX_FIXED_MASK) == VBOX_CPUID_REQ_ECX_FIXED
+        && pVCpu->CTX_SUFF(pVM)->iem.s.fCpuIdHostCall)
+    {
+        VBOXSTRICTRC rcStrict = iemCpuIdVBoxCall(pVCpu, uEcx & VBOX_CPUID_REQ_ECX_FN_MASK,
+                                                 &pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx,
+                                                 &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+    /*
+     * Regular CPUID.
+     */
+    else
+#endif
+        CPUMGetGuestCpuId(pVCpu, uEax, uEcx, pVCpu->cpum.GstCtx.cs.Attr.n.u1Long,
+                          &pVCpu->cpum.GstCtx.eax, &pVCpu->cpum.GstCtx.ebx, &pVCpu->cpum.GstCtx.ecx, &pVCpu->cpum.GstCtx.edx);
+
+    pVCpu->cpum.GstCtx.rax &= UINT32_C(0xffffffff);
+    pVCpu->cpum.GstCtx.rbx &= UINT32_C(0xffffffff);
+    pVCpu->cpum.GstCtx.rcx &= UINT32_C(0xffffffff);
+    pVCpu->cpum.GstCtx.rdx &= UINT32_C(0xffffffff);
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX);
+
+    pVCpu->iem.s.cPotentialExits++;
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'AAD'.
+ *
+ * @param   bImm            The immediate operand.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_aad, uint8_t, bImm)
+{
+    uint16_t const ax = pVCpu->cpum.GstCtx.ax;
+    uint8_t const  al = (uint8_t)ax + (uint8_t)(ax >> 8) * bImm;
+    pVCpu->cpum.GstCtx.ax = al;
+    iemHlpUpdateArithEFlagsU8(pVCpu, al,
+                              X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF,
+                              X86_EFL_OF | X86_EFL_AF | X86_EFL_CF);
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'AAM'.
+ *
+ * @param   bImm            The immediate operand. Cannot be 0.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_aam, uint8_t, bImm)
+{
+    Assert(bImm != 0); /* #DE on 0 is handled in the decoder. */
+
+    uint16_t const ax = pVCpu->cpum.GstCtx.ax;
+    uint8_t const  al = (uint8_t)ax % bImm;
+    uint8_t const  ah = (uint8_t)ax / bImm;
+    pVCpu->cpum.GstCtx.ax = (ah << 8) + al;
+    iemHlpUpdateArithEFlagsU8(pVCpu, al,
+                              X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF,
+                              X86_EFL_OF | X86_EFL_AF | X86_EFL_CF);
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'DAA'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_daa)
+{
+    uint8_t const  al       = pVCpu->cpum.GstCtx.al;
+    bool const     fCarry   = pVCpu->cpum.GstCtx.eflags.Bits.u1CF;
+
+    if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+        || (al & 0xf) >= 10)
+    {
+        pVCpu->cpum.GstCtx.al = al + 6;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+    }
+    else
+        pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+
+    if (al >= 0x9a || fCarry)
+    {
+        pVCpu->cpum.GstCtx.al += 0x60;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+    }
+    else
+        pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+
+    iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'DAS'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_das)
+{
+    uint8_t const  uInputAL = pVCpu->cpum.GstCtx.al;
+    bool const     fCarry   = pVCpu->cpum.GstCtx.eflags.Bits.u1CF;
+
+    if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+        || (uInputAL & 0xf) >= 10)
+    {
+        pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+        if (uInputAL < 6)
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+        pVCpu->cpum.GstCtx.al = uInputAL - 6;
+    }
+    else
+    {
+        pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+    }
+
+    if (uInputAL >= 0x9a || fCarry)
+    {
+        pVCpu->cpum.GstCtx.al -= 0x60;
+        pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+    }
+
+    iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'AAA'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_aaa)
+{
+    if (IEM_IS_GUEST_CPU_AMD(pVCpu))
+    {
+        if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+            || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
+        {
+            iemAImpl_add_u16(&pVCpu->cpum.GstCtx.ax, 0x106, &pVCpu->cpum.GstCtx.eflags.uBoth);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+        }
+        else
+        {
+            iemHlpUpdateArithEFlagsU16(pVCpu, pVCpu->cpum.GstCtx.ax, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+        }
+        pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
+    }
+    else
+    {
+        if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+            || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
+        {
+            pVCpu->cpum.GstCtx.ax += UINT16_C(0x106);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+        }
+        else
+        {
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+        }
+        pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
+        iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+    }
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'AAS'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_aas)
+{
+    if (IEM_IS_GUEST_CPU_AMD(pVCpu))
+    {
+        if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+            || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
+        {
+            iemAImpl_sub_u16(&pVCpu->cpum.GstCtx.ax, 0x106, &pVCpu->cpum.GstCtx.eflags.uBoth);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+        }
+        else
+        {
+            iemHlpUpdateArithEFlagsU16(pVCpu, pVCpu->cpum.GstCtx.ax, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+        }
+        pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
+    }
+    else
+    {
+        if (   pVCpu->cpum.GstCtx.eflags.Bits.u1AF
+            || (pVCpu->cpum.GstCtx.ax & 0xf) >= 10)
+        {
+            pVCpu->cpum.GstCtx.ax -= UINT16_C(0x106);
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 1;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 1;
+        }
+        else
+        {
+            pVCpu->cpum.GstCtx.eflags.Bits.u1AF = 0;
+            pVCpu->cpum.GstCtx.eflags.Bits.u1CF = 0;
+        }
+        pVCpu->cpum.GstCtx.ax &= UINT16_C(0xff0f);
+        iemHlpUpdateArithEFlagsU8(pVCpu, pVCpu->cpum.GstCtx.al, X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF, X86_EFL_OF);
+    }
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements the 16-bit version of 'BOUND'.
+ *
+ * @note    We have separate 16-bit and 32-bit variants of this function due to
+ *          the decoder using unsigned parameters, whereas we want signed one to
+ *          do the job.  This is significant for a recompiler.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_bound_16, int16_t, idxArray, int16_t, idxLowerBound, int16_t, idxUpperBound)
+{
+    /*
+     * Check if the index is inside the bounds, otherwise raise #BR.
+     */
+    if (   idxArray >= idxLowerBound
+        && idxArray <= idxUpperBound)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return iemRaiseBoundRangeExceeded(pVCpu);
+}
+
+
+/**
+ * Implements the 32-bit version of 'BOUND'.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_bound_32, int32_t, idxArray, int32_t, idxLowerBound, int32_t, idxUpperBound)
+{
+    /*
+     * Check if the index is inside the bounds, otherwise raise #BR.
+     */
+    if (   idxArray >= idxLowerBound
+        && idxArray <= idxUpperBound)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return iemRaiseBoundRangeExceeded(pVCpu);
+}
+
+
+
+/*
+ * Instantiate the various string operation combinations.
+ */
+#define OP_SIZE     8
+#define ADDR_SIZE   16
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     8
+#define ADDR_SIZE   32
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     8
+#define ADDR_SIZE   64
+#include "IEMAllCImplStrInstr.cpp.h"
+
+#define OP_SIZE     16
+#define ADDR_SIZE   16
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     16
+#define ADDR_SIZE   32
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     16
+#define ADDR_SIZE   64
+#include "IEMAllCImplStrInstr.cpp.h"
+
+#define OP_SIZE     32
+#define ADDR_SIZE   16
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     32
+#define ADDR_SIZE   32
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     32
+#define ADDR_SIZE   64
+#include "IEMAllCImplStrInstr.cpp.h"
+
+#define OP_SIZE     64
+#define ADDR_SIZE   32
+#include "IEMAllCImplStrInstr.cpp.h"
+#define OP_SIZE     64
+#define ADDR_SIZE   64
+#include "IEMAllCImplStrInstr.cpp.h"
+
+
+/**
+ * Implements 'XGETBV'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_xgetbv)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+    if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)
+    {
+        uint32_t uEcx = pVCpu->cpum.GstCtx.ecx;
+        switch (uEcx)
+        {
+            case 0:
+                break;
+
+            case 1: /** @todo Implement XCR1 support. */
+            default:
+                Log(("xgetbv ecx=%RX32 -> #GP(0)\n", uEcx));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+
+        }
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_XCRx);
+        pVCpu->cpum.GstCtx.rax = RT_LO_U32(pVCpu->cpum.GstCtx.aXcr[uEcx]);
+        pVCpu->cpum.GstCtx.rdx = RT_HI_U32(pVCpu->cpum.GstCtx.aXcr[uEcx]);
+
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+    Log(("xgetbv CR4.OSXSAVE=0 -> UD\n"));
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Implements 'XSETBV'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_xsetbv)
+{
+    if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE)
+    {
+        if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_XSETBV))
+        {
+            Log2(("xsetbv: Guest intercept -> #VMEXIT\n"));
+            IEM_SVM_UPDATE_NRIP(pVCpu);
+            IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XSETBV, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        if (pVCpu->iem.s.uCpl == 0)
+        {
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_XCRx);
+
+            if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+                IEM_VMX_VMEXIT_INSTR_RET(pVCpu, VMX_EXIT_XSETBV, cbInstr);
+
+            uint32_t uEcx = pVCpu->cpum.GstCtx.ecx;
+            uint64_t uNewValue = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx);
+            switch (uEcx)
+            {
+                case 0:
+                {
+                    int rc = CPUMSetGuestXcr0(pVCpu, uNewValue);
+                    if (rc == VINF_SUCCESS)
+                        break;
+                    Assert(rc == VERR_CPUM_RAISE_GP_0);
+                    Log(("xsetbv ecx=%RX32 (newvalue=%RX64) -> #GP(0)\n", uEcx, uNewValue));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+                }
+
+                case 1: /** @todo Implement XCR1 support. */
+                default:
+                    Log(("xsetbv ecx=%RX32 (newvalue=%RX64) -> #GP(0)\n", uEcx, uNewValue));
+                    return iemRaiseGeneralProtectionFault0(pVCpu);
+
+            }
+
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+
+        Log(("xsetbv cpl=%u -> GP(0)\n", pVCpu->iem.s.uCpl));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+    Log(("xsetbv CR4.OSXSAVE=0 -> UD\n"));
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+#ifndef RT_ARCH_ARM64
+# ifdef IN_RING3
+
+/** Argument package for iemCImpl_cmpxchg16b_fallback_rendezvous_callback. */
+struct IEMCIMPLCX16ARGS
+{
+    PRTUINT128U     pu128Dst;
+    PRTUINT128U     pu128RaxRdx;
+    PRTUINT128U     pu128RbxRcx;
+    uint32_t       *pEFlags;
+#  ifdef VBOX_STRICT
+    uint32_t        cCalls;
+#  endif
+};
+
+/**
+ * @callback_method_impl{FNVMMEMTRENDEZVOUS,
+ *                       Worker for iemCImpl_cmpxchg16b_fallback_rendezvous}
+ */
+static DECLCALLBACK(VBOXSTRICTRC) iemCImpl_cmpxchg16b_fallback_rendezvous_callback(PVM pVM, PVMCPUCC pVCpu, void *pvUser)
+{
+    RT_NOREF(pVM, pVCpu);
+    struct IEMCIMPLCX16ARGS *pArgs = (struct IEMCIMPLCX16ARGS *)pvUser;
+#  ifdef VBOX_STRICT
+    Assert(pArgs->cCalls == 0);
+    pArgs->cCalls++;
+#  endif
+
+    iemAImpl_cmpxchg16b_fallback(pArgs->pu128Dst, pArgs->pu128RaxRdx, pArgs->pu128RbxRcx, pArgs->pEFlags);
+    return VINF_SUCCESS;
+}
+
+# endif /* IN_RING3 */
+
+/**
+ * Implements 'CMPXCHG16B' fallback using rendezvous.
+ */
+IEM_CIMPL_DEF_4(iemCImpl_cmpxchg16b_fallback_rendezvous, PRTUINT128U, pu128Dst, PRTUINT128U, pu128RaxRdx,
+                PRTUINT128U, pu128RbxRcx, uint32_t *, pEFlags)
+{
+# ifdef IN_RING3
+    struct IEMCIMPLCX16ARGS Args;
+    Args.pu128Dst       = pu128Dst;
+    Args.pu128RaxRdx    = pu128RaxRdx;
+    Args.pu128RbxRcx    = pu128RbxRcx;
+    Args.pEFlags        = pEFlags;
+#  ifdef VBOX_STRICT
+    Args.cCalls         = 0;
+#  endif
+    VBOXSTRICTRC rcStrict = VMMR3EmtRendezvous(pVCpu->CTX_SUFF(pVM), VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE,
+                                               iemCImpl_cmpxchg16b_fallback_rendezvous_callback, &Args);
+    Assert(Args.cCalls == 1);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /* Duplicated tail code. */
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pu128Dst, IEM_ACCESS_DATA_RW);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            pVCpu->cpum.GstCtx.eflags.u = *pEFlags; /* IEM_MC_COMMIT_EFLAGS */
+            if (!(*pEFlags & X86_EFL_ZF))
+            {
+                pVCpu->cpum.GstCtx.rax = pu128RaxRdx->s.Lo;
+                pVCpu->cpum.GstCtx.rdx = pu128RaxRdx->s.Hi;
+            }
+            rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+    }
+    return rcStrict;
+# else
+    RT_NOREF(pVCpu, cbInstr, pu128Dst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+    return VERR_IEM_ASPECT_NOT_IMPLEMENTED; /* This should get us to ring-3 for now.  Should perhaps be replaced later. */
+# endif
+}
+
+#endif /* RT_ARCH_ARM64 */
+
+/**
+ * Implements 'CLFLUSH' and 'CLFLUSHOPT'.
+ *
+ * This is implemented in C because it triggers a load like behaviour without
+ * actually reading anything.  Since that's not so common, it's implemented
+ * here.
+ *
+ * @param   iEffSeg         The effective segment.
+ * @param   GCPtrEff        The address of the image.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_clflush_clflushopt, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
+{
+    /*
+     * Pretend to do a load w/o reading (see also iemCImpl_monitor and iemMemMap).
+     */
+    VBOXSTRICTRC rcStrict = iemMemApplySegment(pVCpu, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, iEffSeg, 1, &GCPtrEff);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        RTGCPHYS GCPhysMem;
+        /** @todo access size.   */
+        rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, GCPtrEff, 1, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA, &GCPhysMem);
+        if (rcStrict == VINF_SUCCESS)
+        {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+                && IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
+            {
+                /*
+                 * CLFLUSH/CLFLUSHOPT does not access the memory, but flushes the cache-line
+                 * that contains the address. However, if the address falls in the APIC-access
+                 * page, the address flushed must instead be the corresponding address in the
+                 * virtual-APIC page.
+                 *
+                 * See Intel spec. 29.4.4 "Instruction-Specific Considerations".
+                 */
+                rcStrict = iemVmxVirtApicAccessUnused(pVCpu, &GCPhysMem, 1, IEM_ACCESS_TYPE_READ | IEM_ACCESS_WHAT_DATA);
+                if (   rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE
+                    && rcStrict != VINF_VMX_MODIFIES_BEHAVIOR)
+                    return rcStrict;
+            }
+#endif
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'FINIT' and 'FNINIT'.
+ *
+ * @param   fCheckXcpts     Whether to check for umasked pending exceptions or
+ *                          not.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_finit, bool, fCheckXcpts)
+{
+    /*
+     * Exceptions.
+     */
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+    if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS))
+        return iemRaiseDeviceNotAvailable(pVCpu);
+
+    iemFpuActualizeStateForChange(pVCpu);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_X87);
+
+    /* FINIT: Raise #MF on pending exception(s): */
+    if (fCheckXcpts && (pVCpu->cpum.GstCtx.XState.x87.FSW & X86_FSW_ES))
+        return iemRaiseMathFault(pVCpu);
+
+    /*
+     * Reset the state.
+     */
+    PX86XSAVEAREA pXState = &pVCpu->cpum.GstCtx.XState;
+
+    /* Rotate the stack to account for changed TOS. */
+    iemFpuRotateStackSetTop(&pXState->x87, 0);
+
+    pXState->x87.FCW        = 0x37f;
+    pXState->x87.FSW        = 0;
+    pXState->x87.FTW        = 0x00;     /* 0 - empty. */
+    /** @todo Intel says the instruction and data pointers are not cleared on
+     *        387, presume that 8087 and 287 doesn't do so either. */
+    /** @todo test this stuff.   */
+    if (IEM_GET_TARGET_CPU(pVCpu) > IEMTARGETCPU_386)
+    {
+        pXState->x87.FPUDP  = 0;
+        pXState->x87.DS     = 0; //??
+        pXState->x87.Rsrvd2 = 0;
+        pXState->x87.FPUIP  = 0;
+        pXState->x87.CS     = 0; //??
+        pXState->x87.Rsrvd1 = 0;
+    }
+    pXState->x87.FOP        = 0;
+
+    iemHlpUsedFpu(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'FXSAVE'.
+ *
+ * @param   iEffSeg         The effective segment.
+ * @param   GCPtrEff        The address of the image.
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fxsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX);
+
+    /*
+     * Raise exceptions.
+     */
+    if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_TS | X86_CR0_EM))
+        return iemRaiseDeviceNotAvailable(pVCpu);
+
+    /*
+     * Access the memory.
+     */
+    void *pvMem512;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE,
+                                      15 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_GP_OR_AC);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    PX86FXSTATE  pDst = (PX86FXSTATE)pvMem512;
+    PCX86FXSTATE pSrc = &pVCpu->cpum.GstCtx.XState.x87;
+
+    /*
+     * Store the registers.
+     */
+    /** @todo CPU/VM detection possible! If CR4.OSFXSR=0 MXCSR it's
+     * implementation specific whether MXCSR and XMM0-XMM7 are saved. */
+
+    /* common for all formats */
+    pDst->FCW           = pSrc->FCW;
+    pDst->FSW           = pSrc->FSW;
+    pDst->FTW           = pSrc->FTW & UINT16_C(0xff);
+    pDst->FOP           = pSrc->FOP;
+    pDst->MXCSR         = pSrc->MXCSR;
+    pDst->MXCSR_MASK    = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
+    for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
+    {
+        /** @todo Testcase: What actually happens to the 6 reserved bytes? I'm clearing
+         *        them for now... */
+        pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
+        pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
+        pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
+        pDst->aRegs[i].au32[3] = 0;
+    }
+
+    /* FPU IP, CS, DP and DS. */
+    pDst->FPUIP  = pSrc->FPUIP;
+    pDst->CS     = pSrc->CS;
+    pDst->FPUDP  = pSrc->FPUDP;
+    pDst->DS     = pSrc->DS;
+    if (enmEffOpSize == IEMMODE_64BIT)
+    {
+        /* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
+        pDst->Rsrvd1 = pSrc->Rsrvd1;
+        pDst->Rsrvd2 = pSrc->Rsrvd2;
+    }
+    else
+    {
+        pDst->Rsrvd1 = 0;
+        pDst->Rsrvd2 = 0;
+    }
+
+    /* XMM registers. Skipped in 64-bit CPL0 if EFER.FFXSR (AMD only) is set. */
+    if (   !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_FFXSR)
+        || pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
+        || pVCpu->iem.s.uCpl != 0)
+    {
+        uint32_t cXmmRegs = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? 16 : 8;
+        for (uint32_t i = 0; i < cXmmRegs; i++)
+            pDst->aXMM[i] = pSrc->aXMM[i];
+        /** @todo Testcase: What happens to the reserved XMM registers? Untouched,
+         *        right? */
+    }
+
+    /*
+     * Commit the memory.
+     */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'FXRSTOR'.
+ *
+ * @param   iEffSeg         The effective segment register for @a GCPtrEff.
+ * @param   GCPtrEff        The address of the image.
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fxrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX);
+
+    /*
+     * Raise exceptions.
+     */
+    if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_TS | X86_CR0_EM))
+        return iemRaiseDeviceNotAvailable(pVCpu);
+
+    /*
+     * Access the memory.
+     */
+    void *pvMem512;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_R,
+                                      15 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_GP_OR_AC);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    PCX86FXSTATE pSrc = (PCX86FXSTATE)pvMem512;
+    PX86FXSTATE  pDst = &pVCpu->cpum.GstCtx.XState.x87;
+
+    /*
+     * Check the state for stuff which will #GP(0).
+     */
+    uint32_t const fMXCSR      = pSrc->MXCSR;
+    uint32_t const fMXCSR_MASK = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
+    if (fMXCSR & ~fMXCSR_MASK)
+    {
+        Log(("fxrstor: MXCSR=%#x (MXCSR_MASK=%#x) -> #GP(0)\n", fMXCSR, fMXCSR_MASK));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Load the registers.
+     */
+    /** @todo CPU/VM detection possible! If CR4.OSFXSR=0 MXCSR it's
+     * implementation specific whether MXCSR and XMM0-XMM7 are
+     * restored according to Intel.
+     * AMD says MXCSR and XMM registers are never loaded if
+     * CR4.OSFXSR=0.
+     */
+
+    /* common for all formats */
+    pDst->FCW       = pSrc->FCW;
+    pDst->FSW       = pSrc->FSW;
+    pDst->FTW       = pSrc->FTW & UINT16_C(0xff);
+    pDst->FOP       = pSrc->FOP;
+    pDst->MXCSR     = fMXCSR;
+    /* (MXCSR_MASK is read-only) */
+    for (uint32_t i = 0; i < RT_ELEMENTS(pSrc->aRegs); i++)
+    {
+        pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
+        pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
+        pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
+        pDst->aRegs[i].au32[3] = 0;
+    }
+
+    /* FPU IP, CS, DP and DS. */
+    /** @todo AMD says this is only done if FSW.ES is set after loading. */
+    if (enmEffOpSize == IEMMODE_64BIT)
+    {
+        pDst->FPUIP  = pSrc->FPUIP;
+        pDst->CS     = pSrc->CS;
+        pDst->Rsrvd1 = pSrc->Rsrvd1;
+        pDst->FPUDP  = pSrc->FPUDP;
+        pDst->DS     = pSrc->DS;
+        pDst->Rsrvd2 = pSrc->Rsrvd2;
+    }
+    else
+    {
+        pDst->FPUIP  = pSrc->FPUIP;
+        pDst->CS     = pSrc->CS;
+        pDst->Rsrvd1 = 0;
+        pDst->FPUDP  = pSrc->FPUDP;
+        pDst->DS     = pSrc->DS;
+        pDst->Rsrvd2 = 0;
+    }
+
+    /* XMM registers. Skipped in 64-bit CPL0 if EFER.FFXSR (AMD only) is set.
+     * Does not affect MXCSR, only registers.
+     */
+    if (   !(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_FFXSR)
+        || pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT
+        || pVCpu->iem.s.uCpl != 0)
+    {
+        uint32_t cXmmRegs = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? 16 : 8;
+        for (uint32_t i = 0; i < cXmmRegs; i++)
+            pDst->aXMM[i] = pSrc->aXMM[i];
+    }
+
+    pDst->FCW &= ~X86_FCW_ZERO_MASK | X86_FCW_IC_MASK; /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */
+    iemFpuRecalcExceptionStatus(pDst);
+
+    if (pDst->FSW & X86_FSW_ES)
+        Log11(("fxrstor: %04x:%08RX64: loading state with pending FPU exception (FSW=%#x)\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pSrc->FSW));
+
+    /*
+     * Unmap the memory.
+     */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemHlpUsedFpu(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'XSAVE'.
+ *
+ * @param   iEffSeg         The effective segment.
+ * @param   GCPtrEff        The address of the image.
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ */
+IEM_CIMPL_DEF_3(iemCImpl_xsave, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+
+    /*
+     * Raise exceptions.
+     */
+    if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
+        return iemRaiseUndefinedOpcode(pVCpu);
+    /* When in VMX non-root mode and XSAVE/XRSTOR is not enabled, it results in #UD. */
+    if (    IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_XSAVES_XRSTORS))
+    {
+        Log(("xrstor: Not enabled for nested-guest execution -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)
+        return iemRaiseDeviceNotAvailable(pVCpu);
+
+    /*
+     * Calc the requested mask.
+     */
+    uint64_t const fReqComponents = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx) & pVCpu->cpum.GstCtx.aXcr[0];
+    AssertLogRelReturn(!(fReqComponents & ~(XSAVE_C_X87 | XSAVE_C_SSE | XSAVE_C_YMM)), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
+    uint64_t const fXInUse        = pVCpu->cpum.GstCtx.aXcr[0];
+
+/** @todo figure out the exact protocol for the memory access.  Currently we
+ *        just need this crap to work halfways to make it possible to test
+ *        AVX instructions. */
+/** @todo figure out the XINUSE and XMODIFIED   */
+
+    /*
+     * Access the x87 memory state.
+     */
+    /* The x87+SSE state.  */
+    void *pvMem512;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE,
+                                      63 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_GP_OR_AC);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    PX86FXSTATE  pDst = (PX86FXSTATE)pvMem512;
+    PCX86FXSTATE pSrc = &pVCpu->cpum.GstCtx.XState.x87;
+
+    /* The header.  */
+    PX86XSAVEHDR pHdr;
+    rcStrict = iemMemMap(pVCpu, (void **)&pHdr, sizeof(&pHdr), iEffSeg, GCPtrEff + 512, IEM_ACCESS_DATA_RW, 0 /* checked above */);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Store the X87 state.
+     */
+    if (fReqComponents & XSAVE_C_X87)
+    {
+        /* common for all formats */
+        pDst->FCW    = pSrc->FCW;
+        pDst->FSW    = pSrc->FSW;
+        pDst->FTW    = pSrc->FTW & UINT16_C(0xff);
+        pDst->FOP    = pSrc->FOP;
+        pDst->FPUIP  = pSrc->FPUIP;
+        pDst->CS     = pSrc->CS;
+        pDst->FPUDP  = pSrc->FPUDP;
+        pDst->DS     = pSrc->DS;
+        if (enmEffOpSize == IEMMODE_64BIT)
+        {
+            /* Save upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
+            pDst->Rsrvd1 = pSrc->Rsrvd1;
+            pDst->Rsrvd2 = pSrc->Rsrvd2;
+        }
+        else
+        {
+            pDst->Rsrvd1 = 0;
+            pDst->Rsrvd2 = 0;
+        }
+        for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
+        {
+            /** @todo Testcase: What actually happens to the 6 reserved bytes? I'm clearing
+             *        them for now... */
+            pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
+            pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
+            pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
+            pDst->aRegs[i].au32[3] = 0;
+        }
+
+    }
+
+    if (fReqComponents & (XSAVE_C_SSE | XSAVE_C_YMM))
+    {
+        pDst->MXCSR         = pSrc->MXCSR;
+        pDst->MXCSR_MASK    = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
+    }
+
+    if (fReqComponents & XSAVE_C_SSE)
+    {
+        /* XMM registers. */
+        uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
+        for (uint32_t i = 0; i < cXmmRegs; i++)
+            pDst->aXMM[i] = pSrc->aXMM[i];
+        /** @todo Testcase: What happens to the reserved XMM registers? Untouched,
+         *        right? */
+    }
+
+    /* Commit the x87 state bits. (probably wrong) */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Store AVX state.
+     */
+    if (fReqComponents & XSAVE_C_YMM)
+    {
+        /** @todo testcase: xsave64 vs xsave32 wrt XSAVE_C_YMM. */
+        AssertLogRelReturn(pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT] != UINT16_MAX, VERR_IEM_IPE_9);
+        PCX86XSAVEYMMHI pCompSrc = CPUMCTX_XSAVE_C_PTR(IEM_GET_CTX(pVCpu), XSAVE_C_YMM_BIT, PCX86XSAVEYMMHI);
+        PX86XSAVEYMMHI  pCompDst;
+        rcStrict = iemMemMap(pVCpu, (void **)&pCompDst, sizeof(*pCompDst), iEffSeg, GCPtrEff + pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT],
+                             IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE, 0 /* checked above */);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+
+        uint32_t cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
+        for (uint32_t i = 0; i < cXmmRegs; i++)
+            pCompDst->aYmmHi[i] = pCompSrc->aYmmHi[i];
+
+        rcStrict = iemMemCommitAndUnmap(pVCpu, pCompDst, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Update the header.
+     */
+    pHdr->bmXState = (pHdr->bmXState & ~fReqComponents)
+                   | (fReqComponents & fXInUse);
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pHdr, IEM_ACCESS_DATA_RW);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'XRSTOR'.
+ *
+ * @param   iEffSeg         The effective segment.
+ * @param   GCPtrEff        The address of the image.
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ */
+IEM_CIMPL_DEF_3(iemCImpl_xrstor, uint8_t, iEffSeg, RTGCPTR, GCPtrEff, IEMMODE, enmEffOpSize)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx);
+
+    /*
+     * Raise exceptions.
+     */
+    if (!(pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
+        return iemRaiseUndefinedOpcode(pVCpu);
+    /* When in VMX non-root mode and XSAVE/XRSTOR is not enabled, it results in #UD. */
+    if (    IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && !IEM_VMX_IS_PROCCTLS2_SET(pVCpu, VMX_PROC_CTLS2_XSAVES_XRSTORS))
+    {
+        Log(("xrstor: Not enabled for nested-guest execution -> #UD\n"));
+        return iemRaiseUndefinedOpcode(pVCpu);
+    }
+    if (pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS)
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    if (GCPtrEff & 63)
+    {
+        /** @todo CPU/VM detection possible! \#AC might not be signal for
+         * all/any misalignment sizes, intel says its an implementation detail. */
+        if (   (pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
+            && pVCpu->cpum.GstCtx.eflags.Bits.u1AC
+            && pVCpu->iem.s.uCpl == 3)
+            return iemRaiseAlignmentCheckException(pVCpu);
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+/** @todo figure out the exact protocol for the memory access.  Currently we
+ *        just need this crap to work halfways to make it possible to test
+ *        AVX instructions. */
+/** @todo figure out the XINUSE and XMODIFIED   */
+
+    /*
+     * Access the x87 memory state.
+     */
+    /* The x87+SSE state.  */
+    void *pvMem512;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &pvMem512, 512, iEffSeg, GCPtrEff, IEM_ACCESS_DATA_R,
+                                      63 | IEM_MEMMAP_F_ALIGN_GP | IEM_MEMMAP_F_ALIGN_GP_OR_AC);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    PCX86FXSTATE pSrc = (PCX86FXSTATE)pvMem512;
+    PX86FXSTATE  pDst = &pVCpu->cpum.GstCtx.XState.x87;
+
+    /*
+     * Calc the requested mask
+     */
+    PX86XSAVEHDR  pHdrDst = &pVCpu->cpum.GstCtx.XState.Hdr;
+    PCX86XSAVEHDR pHdrSrc;
+    rcStrict = iemMemMap(pVCpu, (void **)&pHdrSrc, sizeof(&pHdrSrc), iEffSeg, GCPtrEff + 512,
+                         IEM_ACCESS_DATA_R, 0 /* checked above */);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint64_t const fReqComponents = RT_MAKE_U64(pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.edx) & pVCpu->cpum.GstCtx.aXcr[0];
+    AssertLogRelReturn(!(fReqComponents & ~(XSAVE_C_X87 | XSAVE_C_SSE | XSAVE_C_YMM)), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
+    //uint64_t const fXInUse        = pVCpu->cpum.GstCtx.aXcr[0];
+    uint64_t const fRstorMask     = pHdrSrc->bmXState;
+    uint64_t const fCompMask      = pHdrSrc->bmXComp;
+
+    AssertLogRelReturn(!(fCompMask & XSAVE_C_X), VERR_IEM_ASPECT_NOT_IMPLEMENTED);
+
+    uint32_t const cXmmRegs = enmEffOpSize == IEMMODE_64BIT ? 16 : 8;
+
+    /* We won't need this any longer. */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pHdrSrc, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Load the X87 state.
+     */
+    if (fReqComponents & XSAVE_C_X87)
+    {
+        if (fRstorMask & XSAVE_C_X87)
+        {
+            pDst->FCW    = pSrc->FCW;
+            pDst->FSW    = pSrc->FSW;
+            pDst->FTW    = pSrc->FTW & UINT16_C(0xff);
+            pDst->FOP    = pSrc->FOP;
+            pDst->FPUIP  = pSrc->FPUIP;
+            pDst->CS     = pSrc->CS;
+            pDst->FPUDP  = pSrc->FPUDP;
+            pDst->DS     = pSrc->DS;
+            if (enmEffOpSize == IEMMODE_64BIT)
+            {
+                /* Load upper 16-bits of FPUIP (IP:CS:Rsvd1) and FPUDP (DP:DS:Rsvd2). */
+                pDst->Rsrvd1 = pSrc->Rsrvd1;
+                pDst->Rsrvd2 = pSrc->Rsrvd2;
+            }
+            else
+            {
+                pDst->Rsrvd1 = 0;
+                pDst->Rsrvd2 = 0;
+            }
+            for (uint32_t i = 0; i < RT_ELEMENTS(pDst->aRegs); i++)
+            {
+                pDst->aRegs[i].au32[0] = pSrc->aRegs[i].au32[0];
+                pDst->aRegs[i].au32[1] = pSrc->aRegs[i].au32[1];
+                pDst->aRegs[i].au32[2] = pSrc->aRegs[i].au32[2] & UINT32_C(0xffff);
+                pDst->aRegs[i].au32[3] = 0;
+            }
+
+            pDst->FCW &= ~X86_FCW_ZERO_MASK | X86_FCW_IC_MASK; /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */
+            iemFpuRecalcExceptionStatus(pDst);
+
+            if (pDst->FSW & X86_FSW_ES)
+                Log11(("xrstor: %04x:%08RX64: loading state with pending FPU exception (FSW=%#x)\n",
+                       pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pSrc->FSW));
+        }
+        else
+        {
+            pDst->FCW   = 0x37f;
+            pDst->FSW   = 0;
+            pDst->FTW   = 0x00;         /* 0 - empty. */
+            pDst->FPUDP = 0;
+            pDst->DS    = 0; //??
+            pDst->Rsrvd2= 0;
+            pDst->FPUIP = 0;
+            pDst->CS    = 0; //??
+            pDst->Rsrvd1= 0;
+            pDst->FOP   = 0;
+            for (uint32_t i = 0; i < RT_ELEMENTS(pSrc->aRegs); i++)
+            {
+                pDst->aRegs[i].au32[0] = 0;
+                pDst->aRegs[i].au32[1] = 0;
+                pDst->aRegs[i].au32[2] = 0;
+                pDst->aRegs[i].au32[3] = 0;
+            }
+        }
+        pHdrDst->bmXState |= XSAVE_C_X87; /* playing safe for now */
+    }
+
+    /* MXCSR */
+    if (fReqComponents & (XSAVE_C_SSE | XSAVE_C_YMM))
+    {
+        if (fRstorMask & (XSAVE_C_SSE | XSAVE_C_YMM))
+            pDst->MXCSR = pSrc->MXCSR;
+        else
+            pDst->MXCSR = 0x1f80;
+    }
+
+    /* XMM registers. */
+    if (fReqComponents & XSAVE_C_SSE)
+    {
+        if (fRstorMask & XSAVE_C_SSE)
+        {
+            for (uint32_t i = 0; i < cXmmRegs; i++)
+                pDst->aXMM[i] = pSrc->aXMM[i];
+            /** @todo Testcase: What happens to the reserved XMM registers? Untouched,
+             *        right? */
+        }
+        else
+        {
+            for (uint32_t i = 0; i < cXmmRegs; i++)
+            {
+                pDst->aXMM[i].au64[0] = 0;
+                pDst->aXMM[i].au64[1] = 0;
+            }
+        }
+        pHdrDst->bmXState |= XSAVE_C_SSE; /* playing safe for now */
+    }
+
+    /* Unmap the x87 state bits (so we've don't run out of mapping). */
+    rcStrict = iemMemCommitAndUnmap(pVCpu, pvMem512, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Restore AVX state.
+     */
+    if (fReqComponents & XSAVE_C_YMM)
+    {
+        AssertLogRelReturn(pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT] != UINT16_MAX, VERR_IEM_IPE_9);
+        PX86XSAVEYMMHI  pCompDst = CPUMCTX_XSAVE_C_PTR(IEM_GET_CTX(pVCpu), XSAVE_C_YMM_BIT, PX86XSAVEYMMHI);
+
+        if (fRstorMask & XSAVE_C_YMM)
+        {
+            /** @todo testcase: xsave64 vs xsave32 wrt XSAVE_C_YMM. */
+            PCX86XSAVEYMMHI pCompSrc;
+            rcStrict = iemMemMap(pVCpu, (void **)&pCompSrc, sizeof(*pCompDst),
+                                 iEffSeg, GCPtrEff + pVCpu->cpum.GstCtx.aoffXState[XSAVE_C_YMM_BIT],
+                                 IEM_ACCESS_DATA_R, 0 /* checked above */);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            for (uint32_t i = 0; i < cXmmRegs; i++)
+            {
+                pCompDst->aYmmHi[i].au64[0] = pCompSrc->aYmmHi[i].au64[0];
+                pCompDst->aYmmHi[i].au64[1] = pCompSrc->aYmmHi[i].au64[1];
+            }
+
+            rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)pCompSrc, IEM_ACCESS_DATA_R);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+        else
+        {
+            for (uint32_t i = 0; i < cXmmRegs; i++)
+            {
+                pCompDst->aYmmHi[i].au64[0] = 0;
+                pCompDst->aYmmHi[i].au64[1] = 0;
+            }
+        }
+        pHdrDst->bmXState |= XSAVE_C_YMM; /* playing safe for now */
+    }
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+
+
+/**
+ * Implements 'STMXCSR'.
+ *
+ * @param   iEffSeg         The effective segment register for @a GCPtrEff.
+ * @param   GCPtrEff        The address of the image.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_stmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX);
+
+    /*
+     * Raise exceptions.
+     */
+    if (   !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
+        && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR))
+    {
+        if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
+        {
+            /*
+             * Do the job.
+             */
+            VBOXSTRICTRC rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEff, pVCpu->cpum.GstCtx.XState.x87.MXCSR);
+            if (rcStrict == VINF_SUCCESS)
+                return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            return rcStrict;
+        }
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    }
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Implements 'VSTMXCSR'.
+ *
+ * @param   iEffSeg         The effective segment register for @a GCPtrEff.
+ * @param   GCPtrEff        The address of the image.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vstmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_XCRx);
+
+    /*
+     * Raise exceptions.
+     */
+    if (   (   !IEM_IS_GUEST_CPU_AMD(pVCpu)
+            ? (pVCpu->cpum.GstCtx.aXcr[0] & (XSAVE_C_SSE | XSAVE_C_YMM)) == (XSAVE_C_SSE | XSAVE_C_YMM)
+            : !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)) /* AMD Jaguar CPU (f0x16,m0,s1) behaviour */
+        && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSXSAVE))
+    {
+        if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
+        {
+            /*
+             * Do the job.
+             */
+            VBOXSTRICTRC rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrEff, pVCpu->cpum.GstCtx.XState.x87.MXCSR);
+            if (rcStrict == VINF_SUCCESS)
+                return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            return rcStrict;
+        }
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    }
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Implements 'LDMXCSR'.
+ *
+ * @param   iEffSeg         The effective segment register for @a GCPtrEff.
+ * @param   GCPtrEff        The address of the image.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_ldmxcsr, uint8_t, iEffSeg, RTGCPTR, GCPtrEff)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX);
+
+    /*
+     * Raise exceptions.
+     */
+    /** @todo testcase - order of LDMXCSR faults.  Does \#PF, \#GP and \#SS
+     *        happen after or before \#UD and \#EM? */
+    if (   !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_EM)
+        && (pVCpu->cpum.GstCtx.cr4 & X86_CR4_OSFXSR))
+    {
+        if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_TS))
+        {
+            /*
+             * Do the job.
+             */
+            uint32_t fNewMxCsr;
+            VBOXSTRICTRC rcStrict = iemMemFetchDataU32(pVCpu, &fNewMxCsr, iEffSeg, GCPtrEff);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                uint32_t const fMxCsrMask = CPUMGetGuestMxCsrMask(pVCpu->CTX_SUFF(pVM));
+                if (!(fNewMxCsr & ~fMxCsrMask))
+                {
+                    pVCpu->cpum.GstCtx.XState.x87.MXCSR = fNewMxCsr;
+                    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+                }
+                Log(("ldmxcsr: New MXCSR=%#RX32 & ~MASK=%#RX32 = %#RX32 -> #GP(0)\n",
+                     fNewMxCsr, fMxCsrMask, fNewMxCsr & ~fMxCsrMask));
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+            return rcStrict;
+        }
+        return iemRaiseDeviceNotAvailable(pVCpu);
+    }
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Commmon routine for fnstenv and fnsave.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   uPtr            Where to store the state.
+ */
+static void iemCImplCommonFpuStoreEnv(PVMCPUCC pVCpu, IEMMODE enmEffOpSize, RTPTRUNION uPtr)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+    PCX86FXSTATE pSrcX87 = &pVCpu->cpum.GstCtx.XState.x87;
+    if (enmEffOpSize == IEMMODE_16BIT)
+    {
+        uPtr.pu16[0] = pSrcX87->FCW;
+        uPtr.pu16[1] = pSrcX87->FSW;
+        uPtr.pu16[2] = iemFpuCalcFullFtw(pSrcX87);
+        if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+        {
+            /** @todo Testcase: How does this work when the FPUIP/CS was saved in
+             *        protected mode or long mode and we save it in real mode?  And vice
+             *        versa?  And with 32-bit operand size?  I think CPU is storing the
+             *        effective address ((CS << 4) + IP) in the offset register and not
+             *        doing any address calculations here. */
+            uPtr.pu16[3] = (uint16_t)pSrcX87->FPUIP;
+            uPtr.pu16[4] = ((pSrcX87->FPUIP >> 4) & UINT16_C(0xf000)) | pSrcX87->FOP;
+            uPtr.pu16[5] = (uint16_t)pSrcX87->FPUDP;
+            uPtr.pu16[6] = (pSrcX87->FPUDP  >> 4) & UINT16_C(0xf000);
+        }
+        else
+        {
+            uPtr.pu16[3] = pSrcX87->FPUIP;
+            uPtr.pu16[4] = pSrcX87->CS;
+            uPtr.pu16[5] = pSrcX87->FPUDP;
+            uPtr.pu16[6] = pSrcX87->DS;
+        }
+    }
+    else
+    {
+        /** @todo Testcase: what is stored in the "gray" areas? (figure 8-9 and 8-10) */
+        uPtr.pu16[0*2]   = pSrcX87->FCW;
+        uPtr.pu16[0*2+1] = 0xffff;  /* (0xffff observed on intel skylake.) */
+        uPtr.pu16[1*2]   = pSrcX87->FSW;
+        uPtr.pu16[1*2+1] = 0xffff;
+        uPtr.pu16[2*2]   = iemFpuCalcFullFtw(pSrcX87);
+        uPtr.pu16[2*2+1] = 0xffff;
+        if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+        {
+            uPtr.pu16[3*2]   = (uint16_t)pSrcX87->FPUIP;
+            uPtr.pu32[4]     = ((pSrcX87->FPUIP & UINT32_C(0xffff0000)) >> 4) | pSrcX87->FOP;
+            uPtr.pu16[5*2]   = (uint16_t)pSrcX87->FPUDP;
+            uPtr.pu32[6]     = (pSrcX87->FPUDP  & UINT32_C(0xffff0000)) >> 4;
+        }
+        else
+        {
+            uPtr.pu32[3]     = pSrcX87->FPUIP;
+            uPtr.pu16[4*2]   = pSrcX87->CS;
+            uPtr.pu16[4*2+1] = pSrcX87->FOP;
+            uPtr.pu32[5]     = pSrcX87->FPUDP;
+            uPtr.pu16[6*2]   = pSrcX87->DS;
+            uPtr.pu16[6*2+1] = 0xffff;
+        }
+    }
+}
+
+
+/**
+ * Commmon routine for fldenv and frstor
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   enmEffOpSize    The effective operand size.
+ * @param   uPtr                Where to store the state.
+ */
+static void iemCImplCommonFpuRestoreEnv(PVMCPUCC pVCpu, IEMMODE enmEffOpSize, RTCPTRUNION uPtr)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+    PX86FXSTATE pDstX87 = &pVCpu->cpum.GstCtx.XState.x87;
+    if (enmEffOpSize == IEMMODE_16BIT)
+    {
+        pDstX87->FCW = uPtr.pu16[0];
+        pDstX87->FSW = uPtr.pu16[1];
+        pDstX87->FTW = uPtr.pu16[2];
+        if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+        {
+            pDstX87->FPUIP = uPtr.pu16[3] | ((uint32_t)(uPtr.pu16[4] & UINT16_C(0xf000)) << 4);
+            pDstX87->FPUDP = uPtr.pu16[5] | ((uint32_t)(uPtr.pu16[6] & UINT16_C(0xf000)) << 4);
+            pDstX87->FOP   = uPtr.pu16[4] & UINT16_C(0x07ff);
+            pDstX87->CS    = 0;
+            pDstX87->Rsrvd1= 0;
+            pDstX87->DS    = 0;
+            pDstX87->Rsrvd2= 0;
+        }
+        else
+        {
+            pDstX87->FPUIP = uPtr.pu16[3];
+            pDstX87->CS    = uPtr.pu16[4];
+            pDstX87->Rsrvd1= 0;
+            pDstX87->FPUDP = uPtr.pu16[5];
+            pDstX87->DS    = uPtr.pu16[6];
+            pDstX87->Rsrvd2= 0;
+            /** @todo Testcase: Is FOP cleared when doing 16-bit protected mode fldenv? */
+        }
+    }
+    else
+    {
+        pDstX87->FCW = uPtr.pu16[0*2];
+        pDstX87->FSW = uPtr.pu16[1*2];
+        pDstX87->FTW = uPtr.pu16[2*2];
+        if (IEM_IS_REAL_OR_V86_MODE(pVCpu))
+        {
+            pDstX87->FPUIP = uPtr.pu16[3*2] | ((uPtr.pu32[4] & UINT32_C(0x0ffff000)) << 4);
+            pDstX87->FOP   = uPtr.pu32[4] & UINT16_C(0x07ff);
+            pDstX87->FPUDP = uPtr.pu16[5*2] | ((uPtr.pu32[6] & UINT32_C(0x0ffff000)) << 4);
+            pDstX87->CS    = 0;
+            pDstX87->Rsrvd1= 0;
+            pDstX87->DS    = 0;
+            pDstX87->Rsrvd2= 0;
+        }
+        else
+        {
+            pDstX87->FPUIP = uPtr.pu32[3];
+            pDstX87->CS    = uPtr.pu16[4*2];
+            pDstX87->Rsrvd1= 0;
+            pDstX87->FOP   = uPtr.pu16[4*2+1];
+            pDstX87->FPUDP = uPtr.pu32[5];
+            pDstX87->DS    = uPtr.pu16[6*2];
+            pDstX87->Rsrvd2= 0;
+        }
+    }
+
+    /* Make adjustments. */
+    pDstX87->FTW = iemFpuCompressFtw(pDstX87->FTW);
+#ifdef LOG_ENABLED
+    uint16_t const fOldFsw = pDstX87->FSW;
+#endif
+    pDstX87->FCW &= ~X86_FCW_ZERO_MASK | X86_FCW_IC_MASK; /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */
+    iemFpuRecalcExceptionStatus(pDstX87);
+#ifdef LOG_ENABLED
+    if ((pDstX87->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES))
+        Log11(("iemCImplCommonFpuRestoreEnv: %04x:%08RX64: %s FPU exception (FCW=%#x FSW=%#x -> %#x)\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fOldFsw & X86_FSW_ES ? "Supressed" : "Raised",
+               pDstX87->FCW, fOldFsw, pDstX87->FSW));
+#endif
+
+    /** @todo Testcase: Check if ES and/or B are automatically cleared if no
+     *        exceptions are pending after loading the saved state? */
+}
+
+
+/**
+ * Implements 'FNSTENV'.
+ *
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ * @param   iEffSeg         The effective segment register for @a GCPtrEffDst.
+ * @param   GCPtrEffDst     The address of the image.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fnstenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    RTPTRUNION   uPtr;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 14 : 28,
+                                      iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE,
+                                      enmEffOpSize == IEMMODE_16BIT ? 1 : 3 /** @todo ? */);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemCImplCommonFpuStoreEnv(pVCpu, enmEffOpSize, uPtr);
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, uPtr.pv, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Mask all math exceptions. Any possibly pending exceptions will be cleared. */
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    pFpuCtx->FCW |= X86_FCW_XCPT_MASK;
+#ifdef LOG_ENABLED
+    uint16_t fOldFsw = pFpuCtx->FSW;
+#endif
+    iemFpuRecalcExceptionStatus(pFpuCtx);
+#ifdef LOG_ENABLED
+    if ((pFpuCtx->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES))
+        Log11(("fnstenv: %04x:%08RX64: %s FPU exception (FCW=%#x, FSW %#x -> %#x)\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+               fOldFsw & X86_FSW_ES ? "Supressed" : "Raised", pFpuCtx->FCW, fOldFsw, pFpuCtx->FSW));
+#endif
+
+    iemHlpUsedFpu(pVCpu);
+
+    /* Note: C0, C1, C2 and C3 are documented as undefined, we leave them untouched! */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'FNSAVE'.
+ *
+ * @param   enmEffOpSize    The operand size.
+ * @param   iEffSeg         The effective segment register for @a GCPtrEffDst.
+ * @param   GCPtrEffDst     The address of the image.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fnsave, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffDst)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+
+    RTPTRUNION   uPtr;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, &uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 94 : 108,
+                                      iEffSeg, GCPtrEffDst, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE, 3 /** @todo ? */);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemCImplCommonFpuStoreEnv(pVCpu, enmEffOpSize, uPtr);
+    PRTFLOAT80U paRegs = (PRTFLOAT80U)(uPtr.pu8 + (enmEffOpSize == IEMMODE_16BIT ? 14 : 28));
+    for (uint32_t i = 0; i < RT_ELEMENTS(pFpuCtx->aRegs); i++)
+    {
+        paRegs[i].au32[0] = pFpuCtx->aRegs[i].au32[0];
+        paRegs[i].au32[1] = pFpuCtx->aRegs[i].au32[1];
+        paRegs[i].au16[4] = pFpuCtx->aRegs[i].au16[4];
+    }
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, uPtr.pv, IEM_ACCESS_DATA_W | IEM_ACCESS_PARTIAL_WRITE);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /* Rotate the stack to account for changed TOS. */
+    iemFpuRotateStackSetTop(pFpuCtx, 0);
+
+    /*
+     * Re-initialize the FPU context.
+     */
+    pFpuCtx->FCW   = 0x37f;
+    pFpuCtx->FSW   = 0;
+    pFpuCtx->FTW   = 0x00;       /* 0 - empty */
+    pFpuCtx->FPUDP = 0;
+    pFpuCtx->DS    = 0;
+    pFpuCtx->Rsrvd2= 0;
+    pFpuCtx->FPUIP = 0;
+    pFpuCtx->CS    = 0;
+    pFpuCtx->Rsrvd1= 0;
+    pFpuCtx->FOP   = 0;
+
+    iemHlpUsedFpu(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+
+/**
+ * Implements 'FLDENV'.
+ *
+ * @param   enmEffOpSize    The operand size (only REX.W really matters).
+ * @param   iEffSeg         The effective segment register for @a GCPtrEffSrc.
+ * @param   GCPtrEffSrc     The address of the image.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fldenv, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
+{
+    RTCPTRUNION  uPtr;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 14 : 28,
+                                      iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R,
+                                      enmEffOpSize == IEMMODE_16BIT ? 1 : 3 /** @todo ?*/);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemCImplCommonFpuRestoreEnv(pVCpu, enmEffOpSize, uPtr);
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uPtr.pv, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemHlpUsedFpu(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'FRSTOR'.
+ *
+ * @param   enmEffOpSize    The operand size.
+ * @param   iEffSeg         The effective segment register for @a GCPtrEffSrc.
+ * @param   GCPtrEffSrc     The address of the image.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_frstor, IEMMODE, enmEffOpSize, uint8_t, iEffSeg, RTGCPTR, GCPtrEffSrc)
+{
+    RTCPTRUNION  uPtr;
+    VBOXSTRICTRC rcStrict = iemMemMap(pVCpu, (void **)&uPtr.pv, enmEffOpSize == IEMMODE_16BIT ? 94 : 108,
+                                      iEffSeg, GCPtrEffSrc, IEM_ACCESS_DATA_R, 3 /** @todo ?*/ );
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    iemCImplCommonFpuRestoreEnv(pVCpu, enmEffOpSize, uPtr);
+    PCRTFLOAT80U paRegs = (PCRTFLOAT80U)(uPtr.pu8 + (enmEffOpSize == IEMMODE_16BIT ? 14 : 28));
+    for (uint32_t i = 0; i < RT_ELEMENTS(pFpuCtx->aRegs); i++)
+    {
+        pFpuCtx->aRegs[i].au32[0] = paRegs[i].au32[0];
+        pFpuCtx->aRegs[i].au32[1] = paRegs[i].au32[1];
+        pFpuCtx->aRegs[i].au32[2] = paRegs[i].au16[4];
+        pFpuCtx->aRegs[i].au32[3] = 0;
+    }
+
+    rcStrict = iemMemCommitAndUnmap(pVCpu, (void *)uPtr.pv, IEM_ACCESS_DATA_R);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    iemHlpUsedFpu(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'FLDCW'.
+ *
+ * @param   u16Fcw          The new FCW.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_fldcw, uint16_t, u16Fcw)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+
+    /** @todo Testcase: Check what happens when trying to load X86_FCW_PC_RSVD. */
+    /** @todo Testcase: Try see what happens when trying to set undefined bits
+     *        (other than 6 and 7).  Currently ignoring them. */
+    /** @todo Testcase: Test that it raises and loweres the FPU exception bits
+     *        according to FSW. (This is what is currently implemented.) */
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    pFpuCtx->FCW = u16Fcw & (~X86_FCW_ZERO_MASK | X86_FCW_IC_MASK); /* Intel 10980xe allows setting the IC bit. Win 3.11 CALC.EXE sets it. */
+#ifdef LOG_ENABLED
+    uint16_t fOldFsw = pFpuCtx->FSW;
+#endif
+    iemFpuRecalcExceptionStatus(pFpuCtx);
+#ifdef LOG_ENABLED
+    if ((pFpuCtx->FSW & X86_FSW_ES) ^ (fOldFsw & X86_FSW_ES))
+        Log11(("fldcw: %04x:%08RX64: %s FPU exception (FCW=%#x, FSW %#x -> %#x)\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+               fOldFsw & X86_FSW_ES ? "Supressed" : "Raised", pFpuCtx->FCW, fOldFsw, pFpuCtx->FSW));
+#endif
+
+    /* Note: C0, C1, C2 and C3 are documented as undefined, we leave them untouched! */
+    iemHlpUsedFpu(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+
+/**
+ * Implements the underflow case of fxch.
+ *
+ * @param   iStReg              The other stack register.
+ */
+IEM_CIMPL_DEF_1(iemCImpl_fxch_underflow, uint8_t, iStReg)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    unsigned const iReg1 = X86_FSW_TOP_GET(pFpuCtx->FSW);
+    unsigned const iReg2 = (iReg1 + iStReg) & X86_FSW_TOP_SMASK;
+    Assert(!(RT_BIT(iReg1) & pFpuCtx->FTW) || !(RT_BIT(iReg2) & pFpuCtx->FTW));
+
+    /** @todo Testcase: fxch underflow. Making assumptions that underflowed
+     *        registers are read as QNaN and then exchanged. This could be
+     *        wrong... */
+    if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        if (RT_BIT(iReg1) & pFpuCtx->FTW)
+        {
+            if (RT_BIT(iReg2) & pFpuCtx->FTW)
+                iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
+            else
+                pFpuCtx->aRegs[0].r80 = pFpuCtx->aRegs[iStReg].r80;
+            iemFpuStoreQNan(&pFpuCtx->aRegs[iStReg].r80);
+        }
+        else
+        {
+            pFpuCtx->aRegs[iStReg].r80 = pFpuCtx->aRegs[0].r80;
+            iemFpuStoreQNan(&pFpuCtx->aRegs[0].r80);
+        }
+        pFpuCtx->FSW &= ~X86_FSW_C_MASK;
+        pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF;
+    }
+    else
+    {
+        /* raise underflow exception, don't change anything. */
+        pFpuCtx->FSW &= ~(X86_FSW_TOP_MASK | X86_FSW_XCPT_MASK);
+        pFpuCtx->FSW |= X86_FSW_C1 | X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+        Log11(("fxch: %04x:%08RX64: Underflow exception (FSW=%#x)\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
+    }
+
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemHlpUsedFpu(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'FCOMI', 'FCOMIP', 'FUCOMI', and 'FUCOMIP'.
+ *
+ * @param   iStReg          The other stack register.
+ * @param   pfnAImpl        The assembly comparison implementation.
+ * @param   fPop            Whether we should pop the stack when done or not.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_fcomi_fucomi, uint8_t, iStReg, PFNIEMAIMPLFPUR80EFL, pfnAImpl, bool, fPop)
+{
+    Assert(iStReg < 8);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_X87);
+
+    /*
+     * Raise exceptions.
+     */
+    if (pVCpu->cpum.GstCtx.cr0 & (X86_CR0_EM | X86_CR0_TS))
+        return iemRaiseDeviceNotAvailable(pVCpu);
+
+    PX86FXSTATE pFpuCtx = &pVCpu->cpum.GstCtx.XState.x87;
+    uint16_t u16Fsw = pFpuCtx->FSW;
+    if (u16Fsw & X86_FSW_ES)
+        return iemRaiseMathFault(pVCpu);
+
+    /*
+     * Check if any of the register accesses causes #SF + #IA.
+     */
+    unsigned const iReg1 = X86_FSW_TOP_GET(u16Fsw);
+    unsigned const iReg2 = (iReg1 + iStReg) & X86_FSW_TOP_SMASK;
+    if ((pFpuCtx->FTW & (RT_BIT(iReg1) | RT_BIT(iReg2))) == (RT_BIT(iReg1) | RT_BIT(iReg2)))
+    {
+        uint32_t u32Eflags = pfnAImpl(pFpuCtx, &u16Fsw, &pFpuCtx->aRegs[0].r80, &pFpuCtx->aRegs[iStReg].r80);
+
+        pFpuCtx->FSW &= ~X86_FSW_C1;
+        pFpuCtx->FSW |= u16Fsw & ~X86_FSW_TOP_MASK;
+        if (   !(u16Fsw & X86_FSW_IE)
+            || (pFpuCtx->FCW & X86_FCW_IM) )
+        {
+            pVCpu->cpum.GstCtx.eflags.u &= ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF);
+            pVCpu->cpum.GstCtx.eflags.u |= u32Eflags & (X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF);
+        }
+    }
+    else if (pFpuCtx->FCW & X86_FCW_IM)
+    {
+        /* Masked underflow. */
+        pFpuCtx->FSW &= ~X86_FSW_C1;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF;
+        pVCpu->cpum.GstCtx.eflags.u &= ~(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF);
+        pVCpu->cpum.GstCtx.eflags.u |= X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF;
+    }
+    else
+    {
+        /* Raise underflow - don't touch EFLAGS or TOP. */
+        pFpuCtx->FSW &= ~X86_FSW_C1;
+        pFpuCtx->FSW |= X86_FSW_IE | X86_FSW_SF | X86_FSW_ES | X86_FSW_B;
+        Log11(("fxch: %04x:%08RX64: Raising IE+SF exception (FSW=%#x)\n",
+               pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pFpuCtx->FSW));
+        fPop = false;
+    }
+
+    /*
+     * Pop if necessary.
+     */
+    if (fPop)
+    {
+        pFpuCtx->FTW &= ~RT_BIT(iReg1);
+        iemFpuStackIncTop(pVCpu);
+    }
+
+    iemFpuUpdateOpcodeAndIpWorker(pVCpu, pFpuCtx);
+    iemHlpUsedFpu(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h b/src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h
new file mode 100644
index 00000000..4bef3cfb
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllCImplStrInstr.cpp.h
@@ -0,0 +1,1773 @@
+/* $Id: IEMAllCImplStrInstr.cpp.h $ */
+/** @file
+ * IEM - String Instruction Implementation Code Template.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*******************************************************************************
+*   Defined Constants And Macros                                               *
+*******************************************************************************/
+#if OP_SIZE == 8
+# define OP_rAX     al
+#elif OP_SIZE == 16
+# define OP_rAX     ax
+#elif OP_SIZE == 32
+# define OP_rAX     eax
+#elif OP_SIZE == 64
+# define OP_rAX     rax
+#else
+# error "Bad OP_SIZE."
+#endif
+#define OP_TYPE                     RT_CONCAT3(uint,OP_SIZE,_t)
+
+#if ADDR_SIZE == 16
+# define ADDR_rDI       di
+# define ADDR_rSI       si
+# define ADDR_rCX       cx
+# define ADDR2_TYPE     uint32_t
+# define ADDR_VMXSTRIO  0
+#elif ADDR_SIZE == 32
+# define ADDR_rDI       edi
+# define ADDR_rSI       esi
+# define ADDR_rCX       ecx
+# define ADDR2_TYPE     uint32_t
+# define ADDR_VMXSTRIO  1
+#elif ADDR_SIZE == 64
+# define ADDR_rDI       rdi
+# define ADDR_rSI       rsi
+# define ADDR_rCX       rcx
+# define ADDR2_TYPE     uint64_t
+# define ADDR_VMXSTRIO  2
+# define IS_64_BIT_CODE(a_pVCpu)    (true)
+#else
+# error "Bad ADDR_SIZE."
+#endif
+#define ADDR_TYPE                   RT_CONCAT3(uint,ADDR_SIZE,_t)
+
+#if ADDR_SIZE == 64 || OP_SIZE == 64
+# define IS_64_BIT_CODE(a_pVCpu)    (true)
+#elif ADDR_SIZE == 32
+# define IS_64_BIT_CODE(a_pVCpu)    ((a_pVCpu)->iem.s.enmCpuMode == IEMMODE_64BIT)
+#else
+# define IS_64_BIT_CODE(a_pVCpu)    (false)
+#endif
+
+/** @def IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN
+ * Used in the outer (page-by-page) loop to check for reasons for returnning
+ * before completing the instruction.   In raw-mode we temporarily enable
+ * interrupts to let the host interrupt us.  We cannot let big string operations
+ * hog the CPU, especially not in raw-mode.
+ */
+#define IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(a_pVM, a_pVCpu, a_fEflags) \
+    do { \
+        if (RT_LIKELY(   !VMCPU_FF_IS_ANY_SET(a_pVCpu, (a_fEflags) & X86_EFL_IF ? VMCPU_FF_YIELD_REPSTR_MASK \
+                                                                                : VMCPU_FF_YIELD_REPSTR_NOINT_MASK) \
+                      && !VM_FF_IS_ANY_SET(a_pVM, VM_FF_YIELD_REPSTR_MASK) \
+                      )) \
+        { /* probable */ } \
+        else  \
+        { \
+            LogFlow(("%s: Leaving early (outer)! ffcpu=%#RX64 ffvm=%#x\n", \
+                     __FUNCTION__, (uint64_t)(a_pVCpu)->fLocalForcedActions, (a_pVM)->fGlobalForcedActions)); \
+            return VINF_SUCCESS; \
+        } \
+    } while (0)
+
+/** @def IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN
+ * This is used in some of the inner loops to make sure we respond immediately
+ * to VMCPU_FF_IOM as well as outside requests.  Use this for expensive
+ * instructions. Use IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN for
+ * ones that are typically cheap. */
+#define IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(a_pVM, a_pVCpu, a_fExitExpr) \
+    do { \
+        if (RT_LIKELY(   (   !VMCPU_FF_IS_ANY_SET(a_pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_REPSTR_MASK) \
+                          && !VM_FF_IS_ANY_SET(a_pVM,         VM_FF_HIGH_PRIORITY_POST_REPSTR_MASK)) \
+                      || (a_fExitExpr) )) \
+        { /* very likely */ } \
+        else \
+        { \
+            LogFlow(("%s: Leaving early (inner)! ffcpu=%#RX64 ffvm=%#x\n", \
+                     __FUNCTION__, (uint64_t)(a_pVCpu)->fLocalForcedActions, (a_pVM)->fGlobalForcedActions)); \
+            return VINF_SUCCESS; \
+        } \
+    } while (0)
+
+
+/** @def IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN
+ * This is used in the inner loops where
+ * IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN isn't used.  It only
+ * checks the CPU FFs so that we respond immediately to the pending IOM FF
+ * (status code is hidden in IEMCPU::rcPassUp by IEM memory commit code).
+ */
+#define IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(a_pVM, a_pVCpu, a_fExitExpr) \
+    do { \
+        if (RT_LIKELY(   !VMCPU_FF_IS_ANY_SET(a_pVCpu, VMCPU_FF_HIGH_PRIORITY_POST_REPSTR_MASK) \
+                      || (a_fExitExpr) )) \
+        { /* very likely */ } \
+        else \
+        { \
+            LogFlow(("%s: Leaving early (inner)! ffcpu=%#RX64 (ffvm=%#x)\n", \
+                     __FUNCTION__, (uint64_t)(a_pVCpu)->fLocalForcedActions, (a_pVM)->fGlobalForcedActions)); \
+            return VINF_SUCCESS; \
+        } \
+    } while (0)
+
+
+/**
+ * Implements 'REPE CMPS'.
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_repe_cmps_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg  = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg) | CPUMCTX_EXTRN_ES);
+
+    PCCPUMSELREGHID pSrc1Hid     = iemSRegGetHid(pVCpu, iEffSeg);
+    uint64_t        uSrc1Base    = 0; /* gcc may not be used uninitialized */
+    VBOXSTRICTRC    rcStrict     = iemMemSegCheckReadAccessEx(pVCpu, pSrc1Hid, iEffSeg, &uSrc1Base);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint64_t        uSrc2Base    = 0; /* gcc may not be used uninitialized */
+    rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uSrc2Base);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr       = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uSrc1AddrReg = pVCpu->cpum.GstCtx.ADDR_rSI;
+    ADDR_TYPE       uSrc2AddrReg = pVCpu->cpum.GstCtx.ADDR_rDI;
+    uint32_t        uEFlags      = pVCpu->cpum.GstCtx.eflags.u;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtSrc1Addr = uSrc1AddrReg + (ADDR2_TYPE)uSrc1Base;
+        ADDR2_TYPE  uVirtSrc2Addr = uSrc2AddrReg + (ADDR2_TYPE)uSrc2Base;
+        uint32_t    cLeftSrc1Page = (GUEST_PAGE_SIZE - (uVirtSrc1Addr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftSrc1Page > uCounterReg)
+            cLeftSrc1Page = uCounterReg;
+        uint32_t    cLeftSrc2Page = (GUEST_PAGE_SIZE - (uVirtSrc2Addr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        uint32_t    cLeftPage     = RT_MIN(cLeftSrc1Page, cLeftSrc2Page);
+
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Optimize reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uSrc1AddrReg < pSrc1Hid->u32Limit
+                    && uSrc1AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrc1Hid->u32Limit
+                    && uSrc2AddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uSrc2AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysSrc1Mem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc1Addr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysSrc1Mem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            RTGCPHYS GCPhysSrc2Mem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc2Addr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysSrc2Mem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            PGMPAGEMAPLOCK PgLockSrc2Mem;
+            OP_TYPE const *puSrc2Mem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, (void **)&puSrc2Mem, &PgLockSrc2Mem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                PGMPAGEMAPLOCK PgLockSrc1Mem;
+                OP_TYPE const *puSrc1Mem;
+                rcStrict = iemMemPageMap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, (void **)&puSrc1Mem, &PgLockSrc1Mem);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    if (!memcmp(puSrc2Mem, puSrc1Mem, cLeftPage * (OP_SIZE / 8)))
+                    {
+                        /* All matches, only compare the last itme to get the right eflags. */
+                        RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[cLeftPage-1], puSrc2Mem[cLeftPage-1], &uEFlags);
+                        uSrc1AddrReg += cLeftPage * cbIncr;
+                        uSrc2AddrReg += cLeftPage * cbIncr;
+                        uCounterReg  -= cLeftPage;
+                    }
+                    else
+                    {
+                        /* Some mismatch, compare each item (and keep volatile
+                           memory in mind). */
+                        uint32_t off = 0;
+                        do
+                        {
+                            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[off], puSrc2Mem[off], &uEFlags);
+                            off++;
+                        } while (   off < cLeftPage
+                                 && (uEFlags & X86_EFL_ZF));
+                        uSrc1AddrReg += cbIncr * off;
+                        uSrc2AddrReg += cbIncr * off;
+                        uCounterReg  -= off;
+                    }
+
+                    /* Update the registers before looping. */
+                    pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg;
+                    pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg;
+                    pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg;
+                    pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+
+                    iemMemPageUnmap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, puSrc1Mem, &PgLockSrc1Mem);
+                    iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem);
+                    if (   uCounterReg == 0
+                        || !(uEFlags & X86_EFL_ZF))
+                        break;
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+                    continue;
+                }
+                iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem);
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uValue1;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue1, iEffSeg, uSrc1AddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            OP_TYPE uValue2;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue2, X86_SREG_ES, uSrc2AddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)(&uValue1, uValue2, &uEFlags);
+
+            pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || !(uEFlags & X86_EFL_ZF));
+        } while (   (int32_t)cLeftPage > 0
+                 && (uEFlags & X86_EFL_ZF));
+
+        /*
+         * Next page? Must check for interrupts and stuff here.
+         */
+        if (   uCounterReg == 0
+            || !(uEFlags & X86_EFL_ZF))
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+    }
+
+    /*
+     * Done.
+     */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'REPNE CMPS'.
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_repne_cmps_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg) | CPUMCTX_EXTRN_ES);
+
+    PCCPUMSELREGHID pSrc1Hid     = iemSRegGetHid(pVCpu, iEffSeg);
+    uint64_t        uSrc1Base    = 0; /* gcc may not be used uninitialized */;
+    VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pSrc1Hid, iEffSeg, &uSrc1Base);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint64_t        uSrc2Base    = 0; /* gcc may not be used uninitialized */
+    rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uSrc2Base);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr       = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uSrc1AddrReg = pVCpu->cpum.GstCtx.ADDR_rSI;
+    ADDR_TYPE       uSrc2AddrReg = pVCpu->cpum.GstCtx.ADDR_rDI;
+    uint32_t        uEFlags      = pVCpu->cpum.GstCtx.eflags.u;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtSrc1Addr = uSrc1AddrReg + (ADDR2_TYPE)uSrc1Base;
+        ADDR2_TYPE  uVirtSrc2Addr = uSrc2AddrReg + (ADDR2_TYPE)uSrc2Base;
+        uint32_t    cLeftSrc1Page = (GUEST_PAGE_SIZE - (uVirtSrc1Addr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftSrc1Page > uCounterReg)
+            cLeftSrc1Page = uCounterReg;
+        uint32_t    cLeftSrc2Page = (GUEST_PAGE_SIZE - (uVirtSrc2Addr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        uint32_t    cLeftPage = RT_MIN(cLeftSrc1Page, cLeftSrc2Page);
+
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Optimize reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uSrc1AddrReg < pSrc1Hid->u32Limit
+                    && uSrc1AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrc1Hid->u32Limit
+                    && uSrc2AddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uSrc2AddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+                )
+           )
+        {
+            RTGCPHYS GCPhysSrc1Mem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc1Addr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysSrc1Mem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            RTGCPHYS GCPhysSrc2Mem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrc2Addr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysSrc2Mem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            OP_TYPE const *puSrc2Mem;
+            PGMPAGEMAPLOCK PgLockSrc2Mem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, (void **)&puSrc2Mem, &PgLockSrc2Mem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                OP_TYPE const *puSrc1Mem;
+                PGMPAGEMAPLOCK PgLockSrc1Mem;
+                rcStrict = iemMemPageMap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, (void **)&puSrc1Mem, &PgLockSrc1Mem);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    if (memcmp(puSrc2Mem, puSrc1Mem, cLeftPage * (OP_SIZE / 8)))
+                    {
+                        /* All matches, only compare the last item to get the right eflags. */
+                        RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[cLeftPage-1], puSrc2Mem[cLeftPage-1], &uEFlags);
+                        uSrc1AddrReg += cLeftPage * cbIncr;
+                        uSrc2AddrReg += cLeftPage * cbIncr;
+                        uCounterReg  -= cLeftPage;
+                    }
+                    else
+                    {
+                        /* Some mismatch, compare each item (and keep volatile
+                           memory in mind). */
+                        uint32_t off = 0;
+                        do
+                        {
+                            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&puSrc1Mem[off], puSrc2Mem[off], &uEFlags);
+                            off++;
+                        } while (   off < cLeftPage
+                                 && !(uEFlags & X86_EFL_ZF));
+                        uSrc1AddrReg += cbIncr * off;
+                        uSrc2AddrReg += cbIncr * off;
+                        uCounterReg  -= off;
+                    }
+
+                    /* Update the registers before looping. */
+                    pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg;
+                    pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg;
+                    pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg;
+                    pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+
+                    iemMemPageUnmap(pVCpu, GCPhysSrc1Mem, IEM_ACCESS_DATA_R, puSrc1Mem, &PgLockSrc1Mem);
+                    iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem);
+                    if (   uCounterReg == 0
+                        || (uEFlags & X86_EFL_ZF))
+                        break;
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+                    continue;
+                }
+                iemMemPageUnmap(pVCpu, GCPhysSrc2Mem, IEM_ACCESS_DATA_R, puSrc2Mem, &PgLockSrc2Mem);
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uValue1;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue1, iEffSeg, uSrc1AddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            OP_TYPE uValue2;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue2, X86_SREG_ES, uSrc2AddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)(&uValue1, uValue2, &uEFlags);
+
+            pVCpu->cpum.GstCtx.ADDR_rSI = uSrc1AddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rDI = uSrc2AddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || (uEFlags & X86_EFL_ZF));
+        } while (   (int32_t)cLeftPage > 0
+                 && !(uEFlags & X86_EFL_ZF));
+
+        /*
+         * Next page? Must check for interrupts and stuff here.
+         */
+        if (   uCounterReg == 0
+            || (uEFlags & X86_EFL_ZF))
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+    }
+
+    /*
+     * Done.
+     */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'REPE SCAS'.
+ */
+IEM_CIMPL_DEF_0(RT_CONCAT4(iemCImpl_repe_scas_,OP_rAX,_m,ADDR_SIZE))
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES);
+    uint64_t        uBaseAddr   = 0; /* gcc may not be used uninitialized */
+    VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    OP_TYPE const   uValueReg   = pVCpu->cpum.GstCtx.OP_rAX;
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rDI;
+    uint32_t        uEFlags     = pVCpu->cpum.GstCtx.eflags.u;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE const *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                /* Search till we find a mismatching item. */
+                OP_TYPE  uTmpValue;
+                bool     fQuit;
+                uint32_t i = 0;
+                do
+                {
+                    uTmpValue = puMem[i++];
+                    fQuit = uTmpValue != uValueReg;
+                } while (i < cLeftPage && !fQuit);
+
+                /* Update the regs. */
+                RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags);
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= i;
+                pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg    += i * cbIncr;
+                pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+                Assert(!(uEFlags & X86_EFL_ZF) == fQuit);
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem);
+                if (   fQuit
+                    || uCounterReg == 0)
+                    break;
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uTmpValue;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uTmpValue, X86_SREG_ES, uAddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags);
+
+            pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || !(uEFlags & X86_EFL_ZF));
+        } while (   (int32_t)cLeftPage > 0
+                 && (uEFlags & X86_EFL_ZF));
+
+        /*
+         * Next page? Must check for interrupts and stuff here.
+         */
+        if (   uCounterReg == 0
+            || !(uEFlags & X86_EFL_ZF))
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+    }
+
+    /*
+     * Done.
+     */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'REPNE SCAS'.
+ */
+IEM_CIMPL_DEF_0(RT_CONCAT4(iemCImpl_repne_scas_,OP_rAX,_m,ADDR_SIZE))
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES);
+    uint64_t        uBaseAddr   = 0; /* gcc may not be used uninitialized */
+    VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    OP_TYPE const   uValueReg   = pVCpu->cpum.GstCtx.OP_rAX;
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rDI;
+    uint32_t        uEFlags     = pVCpu->cpum.GstCtx.eflags.u;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE const *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                /* Search till we find a mismatching item. */
+                OP_TYPE  uTmpValue;
+                bool     fQuit;
+                uint32_t i = 0;
+                do
+                {
+                    uTmpValue = puMem[i++];
+                    fQuit = uTmpValue == uValueReg;
+                } while (i < cLeftPage && !fQuit);
+
+                /* Update the regs. */
+                RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags);
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= i;
+                pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg    += i * cbIncr;
+                pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+                Assert(!!(uEFlags & X86_EFL_ZF) == fQuit);
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem);
+                if (   fQuit
+                    || uCounterReg == 0)
+                    break;
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uTmpValue;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uTmpValue, X86_SREG_ES, uAddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            RT_CONCAT(iemAImpl_cmp_u,OP_SIZE)((OP_TYPE *)&uValueReg, uTmpValue, &uEFlags);
+            pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            pVCpu->cpum.GstCtx.eflags.u = uEFlags;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0 || (uEFlags & X86_EFL_ZF));
+        } while (   (int32_t)cLeftPage > 0
+                 && !(uEFlags & X86_EFL_ZF));
+
+        /*
+         * Next page? Must check for interrupts and stuff here.
+         */
+        if (   uCounterReg == 0
+            || (uEFlags & X86_EFL_ZF))
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, uEFlags);
+    }
+
+    /*
+     * Done.
+     */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+
+
+/**
+ * Implements 'REP MOVS'.
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_rep_movs_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg) | CPUMCTX_EXTRN_ES);
+
+    PCCPUMSELREGHID pSrcHid   = iemSRegGetHid(pVCpu, iEffSeg);
+    uint64_t        uSrcBase  = 0; /* gcc may not be used uninitialized */
+    VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pSrcHid, iEffSeg, &uSrcBase);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint64_t        uDstBase  = 0; /* gcc may not be used uninitialized */
+    rcStrict = iemMemSegCheckWriteAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uDstBase);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uSrcAddrReg = pVCpu->cpum.GstCtx.ADDR_rSI;
+    ADDR_TYPE       uDstAddrReg = pVCpu->cpum.GstCtx.ADDR_rDI;
+
+    /*
+     * Be careful with handle bypassing.
+     */
+    if (pVCpu->iem.s.fBypassHandlers)
+    {
+        Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__));
+        return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
+    }
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtSrcAddr = uSrcAddrReg + (ADDR2_TYPE)uSrcBase;
+        ADDR2_TYPE  uVirtDstAddr = uDstAddrReg + (ADDR2_TYPE)uDstBase;
+        uint32_t    cLeftSrcPage = (GUEST_PAGE_SIZE - (uVirtSrcAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftSrcPage > uCounterReg)
+            cLeftSrcPage = uCounterReg;
+        uint32_t    cLeftDstPage = (GUEST_PAGE_SIZE - (uVirtDstAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        uint32_t    cLeftPage = RT_MIN(cLeftSrcPage, cLeftDstPage);
+
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uSrcAddrReg < pSrcHid->u32Limit
+                    && uSrcAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrcHid->u32Limit
+                    && uDstAddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uDstAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysSrcMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtSrcAddr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysSrcMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            RTGCPHYS GCPhysDstMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtDstAddr, OP_SIZE / 8, IEM_ACCESS_DATA_W, &GCPhysDstMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            PGMPAGEMAPLOCK PgLockDstMem;
+            OP_TYPE *puDstMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysDstMem, IEM_ACCESS_DATA_W, (void **)&puDstMem, &PgLockDstMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                PGMPAGEMAPLOCK PgLockSrcMem;
+                OP_TYPE const *puSrcMem;
+                rcStrict = iemMemPageMap(pVCpu, GCPhysSrcMem, IEM_ACCESS_DATA_R, (void **)&puSrcMem, &PgLockSrcMem);
+                if (rcStrict == VINF_SUCCESS)
+                {
+                    Assert(   (GCPhysSrcMem         >> GUEST_PAGE_SHIFT) != (GCPhysDstMem         >> GUEST_PAGE_SHIFT)
+                           || ((uintptr_t)puSrcMem  >> GUEST_PAGE_SHIFT) == ((uintptr_t)puDstMem  >> GUEST_PAGE_SHIFT));
+
+                    /* Perform the operation exactly (don't use memcpy to avoid
+                       having to consider how its implementation would affect
+                       any overlapping source and destination area). */
+                    OP_TYPE const  *puSrcCur = puSrcMem;
+                    OP_TYPE        *puDstCur = puDstMem;
+                    uint32_t        cTodo    = cLeftPage;
+                    while (cTodo-- > 0)
+                        *puDstCur++ = *puSrcCur++;
+
+                    /* Update the registers. */
+                    pVCpu->cpum.GstCtx.ADDR_rSI = uSrcAddrReg += cLeftPage * cbIncr;
+                    pVCpu->cpum.GstCtx.ADDR_rDI = uDstAddrReg += cLeftPage * cbIncr;
+                    pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage;
+
+                    iemMemPageUnmap(pVCpu, GCPhysSrcMem, IEM_ACCESS_DATA_R, puSrcMem, &PgLockSrcMem);
+                    iemMemPageUnmap(pVCpu, GCPhysDstMem, IEM_ACCESS_DATA_W, puDstMem, &PgLockDstMem);
+
+                    if (uCounterReg == 0)
+                        break;
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+                iemMemPageUnmap(pVCpu, GCPhysDstMem, IEM_ACCESS_DATA_W, puDstMem, &PgLockDstMem);
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uValue;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue, iEffSeg, uSrcAddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            rcStrict = RT_CONCAT(iemMemStoreDataU,OP_SIZE)(pVCpu, X86_SREG_ES, uDstAddrReg, uValue);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            pVCpu->cpum.GstCtx.ADDR_rSI = uSrcAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rDI = uDstAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            cLeftPage--;
+            IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0);
+        } while ((int32_t)cLeftPage > 0);
+
+        /*
+         * Next page.  Must check for interrupts and stuff here.
+         */
+        if (uCounterReg == 0)
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+    }
+
+    /*
+     * Done.
+     */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'REP STOS'.
+ */
+IEM_CIMPL_DEF_0(RT_CONCAT4(iemCImpl_stos_,OP_rAX,_m,ADDR_SIZE))
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES);
+
+    uint64_t        uBaseAddr   = 0; /* gcc may not be used uninitialized */
+    VBOXSTRICTRC rcStrict = iemMemSegCheckWriteAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    OP_TYPE const   uValue      = pVCpu->cpum.GstCtx.OP_rAX;
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rDI;
+
+    /*
+     * Be careful with handle bypassing.
+     */
+    /** @todo Permit doing a page if correctly aligned. */
+    if (pVCpu->iem.s.fBypassHandlers)
+    {
+        Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__));
+        return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
+    }
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_W, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, do a block processing
+             * until the end of the current page.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                /* Update the regs first so we can loop on cLeftPage. */
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage;
+                pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg    += cLeftPage * cbIncr;
+
+                /* Do the memsetting. */
+#if OP_SIZE == 8
+                memset(puMem, uValue, cLeftPage);
+/*#elif OP_SIZE == 32
+                ASMMemFill32(puMem, cLeftPage * (OP_SIZE / 8), uValue);*/
+#else
+                while (cLeftPage-- > 0)
+                    *puMem++ = uValue;
+#endif
+
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, puMem, &PgLockMem);
+
+                if (uCounterReg == 0)
+                    break;
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+            /* If we got an invalid physical address in the page table, just skip
+               ahead to the next page or the counter reaches zero.  This crazy
+               optimization is for a buggy EFI firmware that's driving me nuts. */
+            else if (rcStrict == VERR_PGM_PHYS_TLB_UNASSIGNED)
+            {
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage;
+                pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg    += cLeftPage * cbIncr;
+                if (uCounterReg == 0)
+                    break;
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            rcStrict = RT_CONCAT(iemMemStoreDataU,OP_SIZE)(pVCpu, X86_SREG_ES, uAddrReg, uValue);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+            pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0);
+        } while ((int32_t)cLeftPage > 0);
+
+        /*
+         * Next page.  Must check for interrupts and stuff here.
+         */
+        if (uCounterReg == 0)
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+    }
+
+    /*
+     * Done.
+     */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'REP LODS'.
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_lods_,OP_rAX,_m,ADDR_SIZE), int8_t, iEffSeg)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_SREG_FROM_IDX(iEffSeg));
+    PCCPUMSELREGHID pSrcHid   = iemSRegGetHid(pVCpu, iEffSeg);
+    uint64_t        uBaseAddr = 0; /* gcc may not be used uninitialized */
+    VBOXSTRICTRC rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pSrcHid, iEffSeg, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rSI;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pSrcHid->u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pSrcHid->u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, we can get away with
+             * just reading the last value on the page.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE const *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                /* Only get the last byte, the rest doesn't matter in direct access mode. */
+#if OP_SIZE == 32
+                pVCpu->cpum.GstCtx.rax      = puMem[cLeftPage - 1];
+#else
+                pVCpu->cpum.GstCtx.OP_rAX   = puMem[cLeftPage - 1];
+#endif
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cLeftPage;
+                pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg    += cLeftPage * cbIncr;
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem);
+
+                if (uCounterReg == 0)
+                    break;
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         */
+        do
+        {
+            OP_TYPE uTmpValue;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uTmpValue, iEffSeg, uAddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+#if OP_SIZE == 32
+            pVCpu->cpum.GstCtx.rax      = uTmpValue;
+#else
+            pVCpu->cpum.GstCtx.OP_rAX   = uTmpValue;
+#endif
+            pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+            cLeftPage--;
+            IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0);
+        } while ((int32_t)cLeftPage > 0);
+
+        if (rcStrict != VINF_SUCCESS)
+            break;
+
+        /*
+         * Next page.  Must check for interrupts and stuff here.
+         */
+        if (uCounterReg == 0)
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+    }
+
+    /*
+     * Done.
+     */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+#if OP_SIZE != 64
+
+/**
+ * Implements 'INS' (no rep)
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_ins_op,OP_SIZE,_addr,ADDR_SIZE), bool, fIoChecked)
+{
+    PVMCC           pVM  = pVCpu->CTX_SUFF(pVM);
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * Be careful with handle bypassing.
+     */
+    if (pVCpu->iem.s.fBypassHandlers)
+    {
+        Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__));
+        return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
+    }
+
+    /*
+     * ASSUMES the #GP for I/O permission is taken first, then any #GP for
+     * segmentation and finally any #PF due to virtual address translation.
+     * ASSUMES nothing is read from the I/O port before traps are taken.
+     */
+    if (!fIoChecked)
+    {
+        rcStrict = iemHlpCheckPortIOPermission(pVCpu, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Check nested-guest I/O intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VMXEXITINSTRINFO ExitInstrInfo;
+        ExitInstrInfo.u = 0;
+        ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO;
+        ExitInstrInfo.StrIo.iSegReg    = X86_SREG_ES;
+        rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_INS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, false /* fRep */,
+                                          ExitInstrInfo, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, pVCpu->cpum.GstCtx.dx, SVMIOIOTYPE_IN, OP_SIZE / 8, ADDR_SIZE, X86_SREG_ES,
+                                           false /* fRep */, true /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_ins_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", pVCpu->cpum.GstCtx.dx,
+                 OP_SIZE / 8, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    OP_TYPE        *puMem;
+    rcStrict = iemMemMap(pVCpu, (void **)&puMem, OP_SIZE / 8, X86_SREG_ES, pVCpu->cpum.GstCtx.ADDR_rDI,
+                         IEM_ACCESS_DATA_W, OP_SIZE / 8 - 1);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    uint32_t        u32Value = 0;
+    rcStrict = IOMIOPortRead(pVM, pVCpu, pVCpu->cpum.GstCtx.dx, &u32Value, OP_SIZE / 8);
+    if (IOM_SUCCESS(rcStrict))
+    {
+        /**
+         * @todo I/O breakpoint support for INS
+         */
+        *puMem = (OP_TYPE)u32Value;
+# ifdef IN_RING3
+        VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmap(pVCpu, puMem, IEM_ACCESS_DATA_W);
+# else
+        VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmapPostponeTroubleToR3(pVCpu, puMem, IEM_ACCESS_DATA_W);
+# endif
+        if (RT_LIKELY(rcStrict2 == VINF_SUCCESS))
+        {
+            if (!pVCpu->cpum.GstCtx.eflags.Bits.u1DF)
+                pVCpu->cpum.GstCtx.ADDR_rDI += OP_SIZE / 8;
+            else
+                pVCpu->cpum.GstCtx.ADDR_rDI -= OP_SIZE / 8;
+
+            /** @todo finish: work out how this should work wrt status codes. Not sure we
+             * can use iemSetPassUpStatus here, but it depends on what
+             * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */
+            rcStrict2 = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            if (rcStrict2 != VINF_SUCCESS)
+            {
+                iemSetPassUpStatus(pVCpu, rcStrict);
+                rcStrict = rcStrict2;
+            }
+            pVCpu->iem.s.cPotentialExits++;
+        }
+        else
+            AssertLogRelMsgFailedReturn(("rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)), RT_FAILURE_NP(rcStrict2) ? rcStrict2 : VERR_IEM_IPE_1);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'REP INS'.
+ */
+IEM_CIMPL_DEF_1(RT_CONCAT4(iemCImpl_rep_ins_op,OP_SIZE,_addr,ADDR_SIZE), bool, fIoChecked)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ES | CPUMCTX_EXTRN_TR);
+
+    /*
+     * Setup.
+     */
+    uint16_t const  u16Port    = pVCpu->cpum.GstCtx.dx;
+    VBOXSTRICTRC    rcStrict;
+    if (!fIoChecked)
+    {
+/** @todo check if this is too early for ecx=0. */
+        rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, OP_SIZE / 8);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Check nested-guest I/O intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VMXEXITINSTRINFO ExitInstrInfo;
+        ExitInstrInfo.u = 0;
+        ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO;
+        ExitInstrInfo.StrIo.iSegReg    = X86_SREG_ES;
+        rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_INS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, true /* fRep */,
+                                          ExitInstrInfo, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_IN, OP_SIZE / 8, ADDR_SIZE, X86_SREG_ES, true /* fRep */,
+                                           true /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_rep_ins_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, OP_SIZE / 8,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+    uint64_t        uBaseAddr   = 0; /* gcc may not be used uninitialized */
+    rcStrict = iemMemSegCheckWriteAccessEx(pVCpu, iemSRegUpdateHid(pVCpu, &pVCpu->cpum.GstCtx.es), X86_SREG_ES, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rDI;
+
+    /*
+     * Be careful with handle bypassing.
+     */
+    if (pVCpu->iem.s.fBypassHandlers)
+    {
+        Log(("%s: declining because we're bypassing handlers\n", __FUNCTION__));
+        return VERR_IEM_ASPECT_NOT_IMPLEMENTED;
+    }
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pVCpu->cpum.GstCtx.es.u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pVCpu->cpum.GstCtx.es.u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_W, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, use the IOM
+             * string I/O interface to do the work.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                uint32_t cTransfers = cLeftPage;
+                rcStrict = IOMIOPortReadString(pVM, pVCpu, u16Port, puMem, &cTransfers, OP_SIZE / 8);
+
+                uint32_t cActualTransfers = cLeftPage - cTransfers;
+                Assert(cActualTransfers <= cLeftPage);
+                pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg    += cbIncr * cActualTransfers;
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cActualTransfers;
+                puMem += cActualTransfers;
+
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_W, puMem, &PgLockMem);
+
+                if (rcStrict != VINF_SUCCESS)
+                {
+                    if (IOM_SUCCESS(rcStrict))
+                    {
+                        /** @todo finish: work out how this should work wrt status codes. Not sure we
+                         * can use iemSetPassUpStatus here, but it depends on what
+                         * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */
+                        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                        if (uCounterReg == 0)
+                            rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+                        pVCpu->iem.s.cPotentialExits++;
+                    }
+                    return rcStrict;
+                }
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (uCounterReg == 0)
+                    break;
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         *
+         * Note! We ASSUME the CPU will raise #PF or #GP before access the
+         *       I/O port, otherwise it wouldn't really be restartable.
+         */
+        /** @todo investigate what the CPU actually does with \#PF/\#GP
+         *        during INS. */
+        do
+        {
+            OP_TYPE *puMem;
+            rcStrict = iemMemMap(pVCpu, (void **)&puMem, OP_SIZE / 8, X86_SREG_ES, uAddrReg,
+                                 IEM_ACCESS_DATA_W, OP_SIZE / 8 - 1);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            uint32_t u32Value = 0;
+            rcStrict = IOMIOPortRead(pVM, pVCpu, u16Port, &u32Value, OP_SIZE / 8);
+            if (!IOM_SUCCESS(rcStrict))
+            {
+                iemMemRollback(pVCpu);
+                return rcStrict;
+            }
+
+            *puMem = (OP_TYPE)u32Value;
+# ifdef IN_RING3
+            VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmap(pVCpu, puMem, IEM_ACCESS_DATA_W);
+# else
+            VBOXSTRICTRC rcStrict2 = iemMemCommitAndUnmapPostponeTroubleToR3(pVCpu, puMem, IEM_ACCESS_DATA_W);
+# endif
+            if (rcStrict2 == VINF_SUCCESS)
+            { /* likely */ }
+            else
+                AssertLogRelMsgFailedReturn(("rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
+                                            RT_FAILURE(rcStrict2) ? rcStrict2 : VERR_IEM_IPE_1);
+
+            pVCpu->cpum.GstCtx.ADDR_rDI = uAddrReg += cbIncr;
+            pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+
+            cLeftPage--;
+            if (rcStrict != VINF_SUCCESS)
+            {
+                /** @todo finish: work out how this should work wrt status codes. Not sure we
+                 * can use iemSetPassUpStatus here, but it depends on what
+                 * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */
+                rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                if (uCounterReg == 0)
+                    rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+                pVCpu->iem.s.cPotentialExits++;
+                return rcStrict;
+            }
+
+            IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0);
+        } while ((int32_t)cLeftPage > 0);
+
+
+        /*
+         * Next page.  Must check for interrupts and stuff here.
+         */
+        if (uCounterReg == 0)
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+    }
+
+    /*
+     * Done.
+     */
+    pVCpu->iem.s.cPotentialExits++;
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Implements 'OUTS' (no rep)
+ */
+IEM_CIMPL_DEF_2(RT_CONCAT4(iemCImpl_outs_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg, bool, fIoChecked)
+{
+    PVMCC           pVM  = pVCpu->CTX_SUFF(pVM);
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * ASSUMES the #GP for I/O permission is taken first, then any #GP for
+     * segmentation and finally any #PF due to virtual address translation.
+     * ASSUMES nothing is read from the I/O port before traps are taken.
+     */
+    if (!fIoChecked)
+    {
+        rcStrict = iemHlpCheckPortIOPermission(pVCpu, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Check nested-guest I/O intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VMXEXITINSTRINFO ExitInstrInfo;
+        ExitInstrInfo.u = 0;
+        ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO;
+        ExitInstrInfo.StrIo.iSegReg    = iEffSeg;
+        rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_OUTS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, false /* fRep */,
+                                          ExitInstrInfo, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, pVCpu->cpum.GstCtx.dx, SVMIOIOTYPE_OUT, OP_SIZE / 8, ADDR_SIZE, iEffSeg,
+                                           false /* fRep */, true /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_outs_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", pVCpu->cpum.GstCtx.dx,
+                 OP_SIZE / 8, VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    OP_TYPE uValue;
+    rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue, iEffSeg, pVCpu->cpum.GstCtx.ADDR_rSI);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = IOMIOPortWrite(pVM, pVCpu, pVCpu->cpum.GstCtx.dx, uValue, OP_SIZE / 8);
+        if (IOM_SUCCESS(rcStrict))
+        {
+            if (!pVCpu->cpum.GstCtx.eflags.Bits.u1DF)
+                pVCpu->cpum.GstCtx.ADDR_rSI += OP_SIZE / 8;
+            else
+                pVCpu->cpum.GstCtx.ADDR_rSI -= OP_SIZE / 8;
+            /** @todo finish: work out how this should work wrt status codes. Not sure we
+             * can use iemSetPassUpStatus here, but it depends on what
+             * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */
+            if (rcStrict != VINF_SUCCESS)
+                rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+            rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            pVCpu->iem.s.cPotentialExits++;
+        }
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Implements 'REP OUTS'.
+ */
+IEM_CIMPL_DEF_2(RT_CONCAT4(iemCImpl_rep_outs_op,OP_SIZE,_addr,ADDR_SIZE), uint8_t, iEffSeg, bool, fIoChecked)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Setup.
+     */
+    uint16_t const  u16Port    = pVCpu->cpum.GstCtx.dx;
+    VBOXSTRICTRC    rcStrict;
+    if (!fIoChecked)
+    {
+/** @todo check if this is too early for ecx=0. */
+        rcStrict = iemHlpCheckPortIOPermission(pVCpu, u16Port, OP_SIZE / 8);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Check nested-guest I/O intercepts.
+     */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VMXEXITINSTRINFO ExitInstrInfo;
+        ExitInstrInfo.u = 0;
+        ExitInstrInfo.StrIo.u3AddrSize = ADDR_VMXSTRIO;
+        ExitInstrInfo.StrIo.iSegReg    = iEffSeg;
+        rcStrict = iemVmxVmexitInstrStrIo(pVCpu, VMXINSTRID_IO_OUTS, pVCpu->cpum.GstCtx.dx, OP_SIZE / 8, true /* fRep */,
+                                          ExitInstrInfo, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT))
+    {
+        rcStrict = iemSvmHandleIOIntercept(pVCpu, u16Port, SVMIOIOTYPE_OUT, OP_SIZE / 8, ADDR_SIZE, iEffSeg, true /* fRep */,
+                                           true /* fStrIo */, cbInstr);
+        if (rcStrict == VINF_SVM_VMEXIT)
+            return VINF_SUCCESS;
+        if (rcStrict != VINF_SVM_INTERCEPT_NOT_ACTIVE)
+        {
+            Log(("iemCImpl_rep_outs_op: iemSvmHandleIOIntercept failed (u16Port=%#x, cbReg=%u) rc=%Rrc\n", u16Port, OP_SIZE / 8,
+                 VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+#endif
+
+    ADDR_TYPE       uCounterReg = pVCpu->cpum.GstCtx.ADDR_rCX;
+    if (uCounterReg == 0)
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+
+    PCCPUMSELREGHID pHid      = iemSRegGetHid(pVCpu, iEffSeg);
+    uint64_t        uBaseAddr = 0; /* gcc may not be used uninitialized */
+    rcStrict = iemMemSegCheckReadAccessEx(pVCpu, pHid, iEffSeg, &uBaseAddr);
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    int8_t const    cbIncr      = pVCpu->cpum.GstCtx.eflags.Bits.u1DF ? -(OP_SIZE / 8) : (OP_SIZE / 8);
+    ADDR_TYPE       uAddrReg    = pVCpu->cpum.GstCtx.ADDR_rSI;
+
+    /*
+     * The loop.
+     */
+    for (;;)
+    {
+        /*
+         * Do segmentation and virtual page stuff.
+         */
+        ADDR2_TYPE  uVirtAddr = uAddrReg + (ADDR2_TYPE)uBaseAddr;
+        uint32_t    cLeftPage = (GUEST_PAGE_SIZE - (uVirtAddr & GUEST_PAGE_OFFSET_MASK)) / (OP_SIZE / 8);
+        if (cLeftPage > uCounterReg)
+            cLeftPage = uCounterReg;
+        if (   cLeftPage > 0 /* can be null if unaligned, do one fallback round. */
+            && cbIncr > 0    /** @todo Implement reverse direction string ops. */
+            && (   IS_64_BIT_CODE(pVCpu)
+                || (   uAddrReg < pHid->u32Limit
+                    && uAddrReg + (cLeftPage * (OP_SIZE / 8)) <= pHid->u32Limit)
+               )
+           )
+        {
+            RTGCPHYS GCPhysMem;
+            rcStrict = iemMemPageTranslateAndCheckAccess(pVCpu, uVirtAddr, OP_SIZE / 8, IEM_ACCESS_DATA_R, &GCPhysMem);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            /*
+             * If we can map the page without trouble, we use the IOM
+             * string I/O interface to do the job.
+             */
+            PGMPAGEMAPLOCK PgLockMem;
+            OP_TYPE const *puMem;
+            rcStrict = iemMemPageMap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, (void **)&puMem, &PgLockMem);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                uint32_t cTransfers = cLeftPage;
+                rcStrict = IOMIOPortWriteString(pVM, pVCpu, u16Port, puMem, &cTransfers, OP_SIZE / 8);
+
+                uint32_t cActualTransfers = cLeftPage - cTransfers;
+                Assert(cActualTransfers <= cLeftPage);
+                pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg    += cbIncr * cActualTransfers;
+                pVCpu->cpum.GstCtx.ADDR_rCX = uCounterReg -= cActualTransfers;
+                puMem += cActualTransfers;
+
+                iemMemPageUnmap(pVCpu, GCPhysMem, IEM_ACCESS_DATA_R, puMem, &PgLockMem);
+
+                if (rcStrict != VINF_SUCCESS)
+                {
+                    if (IOM_SUCCESS(rcStrict))
+                    {
+                        /** @todo finish: work out how this should work wrt status codes. Not sure we
+                         * can use iemSetPassUpStatus here, but it depends on what
+                         * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */
+                        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                        if (uCounterReg == 0)
+                            rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+                        pVCpu->iem.s.cPotentialExits++;
+                    }
+                    return rcStrict;
+                }
+
+                if (uCounterReg == 0)
+                    break;
+
+                /* If unaligned, we drop thru and do the page crossing access
+                   below. Otherwise, do the next page. */
+                if (!(uVirtAddr & (OP_SIZE / 8 - 1)))
+                {
+                    IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+                    continue;
+                }
+                cLeftPage = 0;
+            }
+        }
+
+        /*
+         * Fallback - slow processing till the end of the current page.
+         * In the cross page boundrary case we will end up here with cLeftPage
+         * as 0, we execute one loop then.
+         *
+         * Note! We ASSUME the CPU will raise #PF or #GP before access the
+         *       I/O port, otherwise it wouldn't really be restartable.
+         */
+        /** @todo investigate what the CPU actually does with \#PF/\#GP
+         *        during INS. */
+        do
+        {
+            OP_TYPE uValue;
+            rcStrict = RT_CONCAT(iemMemFetchDataU,OP_SIZE)(pVCpu, &uValue, iEffSeg, uAddrReg);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+
+            rcStrict = IOMIOPortWrite(pVM, pVCpu, u16Port, uValue, OP_SIZE / 8);
+            if (IOM_SUCCESS(rcStrict))
+            {
+                pVCpu->cpum.GstCtx.ADDR_rSI = uAddrReg += cbIncr;
+                pVCpu->cpum.GstCtx.ADDR_rCX = --uCounterReg;
+                cLeftPage--;
+            }
+            if (rcStrict != VINF_SUCCESS)
+            {
+                if (IOM_SUCCESS(rcStrict))
+                {
+                    /** @todo finish: work out how this should work wrt status codes. Not sure we
+                     * can use iemSetPassUpStatus here, but it depends on what
+                     * iemRegAddToRipAndFinishingClearingRF may eventually return (if anything)... */
+                    rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                    if (uCounterReg == 0)
+                        iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+                    pVCpu->iem.s.cPotentialExits++;
+                }
+                return rcStrict;
+            }
+            IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN(pVM, pVCpu, uCounterReg == 0);
+        } while ((int32_t)cLeftPage > 0);
+
+
+        /*
+         * Next page.  Must check for interrupts and stuff here.
+         */
+        if (uCounterReg == 0)
+            break;
+        IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN(pVM, pVCpu, pVCpu->cpum.GstCtx.eflags.u);
+    }
+
+    /*
+     * Done.
+     */
+    pVCpu->iem.s.cPotentialExits++;
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+#endif /* OP_SIZE != 64-bit */
+
+
+#undef OP_rAX
+#undef OP_SIZE
+#undef ADDR_SIZE
+#undef ADDR_rDI
+#undef ADDR_rSI
+#undef ADDR_rCX
+#undef ADDR_rIP
+#undef ADDR2_TYPE
+#undef ADDR_TYPE
+#undef ADDR2_TYPE
+#undef ADDR_VMXSTRIO
+#undef IS_64_BIT_CODE
+#undef IEM_CHECK_FF_YIELD_REPSTR_MAYBE_RETURN
+#undef IEM_CHECK_FF_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN
+#undef IEM_CHECK_FF_CPU_HIGH_PRIORITY_POST_REPSTR_MAYBE_RETURN
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp b/src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp
new file mode 100644
index 00000000..85e47c08
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllCImplSvmInstr.cpp
@@ -0,0 +1,1622 @@
+/* $Id: IEMAllCImplSvmInstr.cpp $ */
+/** @file
+ * IEM - AMD-V (Secure Virtual Machine) instruction implementation.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP   LOG_GROUP_IEM_SVM
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pgm.h>
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+# include <VBox/vmm/hm_svm.h>
+#endif
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/tm.h>
+#include "IEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/log.h>
+#include <VBox/disopcode.h> /* for OP_VMMCALL */
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/x86.h>
+
+#include "IEMInline.h"
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM /* Almost the whole file. */
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/**
+ * Check the common SVM instruction preconditions.
+ */
+# define IEM_SVM_INSTR_COMMON_CHECKS(a_pVCpu, a_Instr) \
+    do { \
+        if (!CPUMIsGuestSvmEnabled(IEM_GET_CTX(a_pVCpu))) \
+        { \
+            Log((RT_STR(a_Instr) ": EFER.SVME not enabled -> #UD\n")); \
+            return iemRaiseUndefinedOpcode(a_pVCpu); \
+        } \
+        if (IEM_IS_REAL_OR_V86_MODE(a_pVCpu)) \
+        { \
+            Log((RT_STR(a_Instr) ": Real or v8086 mode -> #UD\n")); \
+            return iemRaiseUndefinedOpcode(a_pVCpu); \
+        } \
+        if ((a_pVCpu)->iem.s.uCpl != 0) \
+        { \
+            Log((RT_STR(a_Instr) ": CPL != 0 -> #GP(0)\n")); \
+            return iemRaiseGeneralProtectionFault0(a_pVCpu); \
+        } \
+    } while (0)
+
+
+/**
+ * Converts an IEM exception event type to an SVM event type.
+ *
+ * @returns The SVM event type.
+ * @retval  UINT8_MAX if the specified type of event isn't among the set
+ *          of recognized IEM event types.
+ *
+ * @param   uVector         The vector of the event.
+ * @param   fIemXcptFlags   The IEM exception / interrupt flags.
+ */
+IEM_STATIC uint8_t iemGetSvmEventType(uint32_t uVector, uint32_t fIemXcptFlags)
+{
+    if (fIemXcptFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+    {
+        if (uVector != X86_XCPT_NMI)
+            return SVM_EVENT_EXCEPTION;
+        return SVM_EVENT_NMI;
+    }
+
+    /* See AMD spec. Table 15-1. "Guest Exception or Interrupt Types". */
+    if (fIemXcptFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_ICEBP_INSTR | IEM_XCPT_FLAGS_OF_INSTR))
+        return SVM_EVENT_EXCEPTION;
+
+    if (fIemXcptFlags & IEM_XCPT_FLAGS_T_EXT_INT)
+        return SVM_EVENT_EXTERNAL_IRQ;
+
+    if (fIemXcptFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+        return SVM_EVENT_SOFTWARE_INT;
+
+    AssertMsgFailed(("iemGetSvmEventType: Invalid IEM xcpt/int. type %#x, uVector=%#x\n", fIemXcptFlags, uVector));
+    return UINT8_MAX;
+}
+
+
+/**
+ * Performs an SVM world-switch (VMRUN, \#VMEXIT) updating PGM and IEM internals.
+ *
+ * @returns Strict VBox status code from PGMChangeMode.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(VBOXSTRICTRC) iemSvmWorldSwitch(PVMCPUCC pVCpu)
+{
+    /*
+     * Inform PGM about paging mode changes.
+     * We include X86_CR0_PE because PGM doesn't handle paged-real mode yet,
+     * see comment in iemMemPageTranslateAndCheckAccess().
+     */
+    int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0 | X86_CR0_PE, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
+                           true /* fForce */);
+    AssertRCReturn(rc, rc);
+
+    /* Invalidate IEM TLBs now that we've forced a PGM mode change. */
+    IEMTlbInvalidateAll(pVCpu);
+
+    /* Inform CPUM (recompiler), can later be removed. */
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
+
+    /* Re-initialize IEM cache/state after the drastic mode switch. */
+    iemReInitExec(pVCpu);
+    return rc;
+}
+
+
+/**
+ * SVM \#VMEXIT handler.
+ *
+ * @returns Strict VBox status code.
+ * @retval VINF_SVM_VMEXIT when the \#VMEXIT is successful.
+ * @retval VERR_SVM_VMEXIT_FAILED when the \#VMEXIT failed restoring the guest's
+ *         "host state" and a shutdown is required.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uExitCode   The exit code.
+ * @param   uExitInfo1  The exit info. 1 field.
+ * @param   uExitInfo2  The exit info. 2 field.
+ */
+VBOXSTRICTRC iemSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2) RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict;
+    if (   CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu))
+        || uExitCode == SVM_EXIT_INVALID)
+    {
+        Log2(("iemSvmVmexit: CS:RIP=%04x:%08RX64 uExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n",
+              pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, uExitInfo1, uExitInfo2));
+
+        /*
+         * Disable the global-interrupt flag to prevent interrupts during the 'atomic' world switch.
+         */
+        CPUMSetGuestGif(&pVCpu->cpum.GstCtx, false);
+
+        /*
+         * Map the nested-guest VMCB from its location in guest memory.
+         * Write exactly what the CPU does on #VMEXIT thereby preserving most other bits in the
+         * guest's VMCB in memory, see @bugref{7243#c113} and related comment on iemSvmVmrun().
+         */
+        PSVMVMCB       pVmcbMem;
+        PGMPAGEMAPLOCK PgLockMem;
+        PSVMVMCBCTRL   pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl;
+        rcStrict = iemMemPageMap(pVCpu, pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb, IEM_ACCESS_DATA_RW, (void **)&pVmcbMem,
+                                 &PgLockMem);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /*
+             * Notify HM in case the nested-guest was executed using hardware-assisted SVM (which
+             * would have modified some VMCB state) that might need to be restored on #VMEXIT before
+             * writing the VMCB back to guest memory.
+             */
+            HMNotifySvmNstGstVmexit(pVCpu, IEM_GET_CTX(pVCpu));
+
+            Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.es));
+            Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.cs));
+            Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+            Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ds));
+
+            /*
+             * Save the nested-guest state into the VMCB state-save area.
+             */
+            PSVMVMCBSTATESAVE pVmcbMemState = &pVmcbMem->guest;
+            HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, ES, es);
+            HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, CS, cs);
+            HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, SS, ss);
+            HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), pVmcbMemState, DS, ds);
+            pVmcbMemState->GDTR.u32Limit   = pVCpu->cpum.GstCtx.gdtr.cbGdt;
+            pVmcbMemState->GDTR.u64Base    = pVCpu->cpum.GstCtx.gdtr.pGdt;
+            pVmcbMemState->IDTR.u32Limit   = pVCpu->cpum.GstCtx.idtr.cbIdt;
+            pVmcbMemState->IDTR.u64Base    = pVCpu->cpum.GstCtx.idtr.pIdt;
+            pVmcbMemState->u64EFER         = pVCpu->cpum.GstCtx.msrEFER;
+            pVmcbMemState->u64CR4          = pVCpu->cpum.GstCtx.cr4;
+            pVmcbMemState->u64CR3          = pVCpu->cpum.GstCtx.cr3;
+            pVmcbMemState->u64CR2          = pVCpu->cpum.GstCtx.cr2;
+            pVmcbMemState->u64CR0          = pVCpu->cpum.GstCtx.cr0;
+            /** @todo Nested paging. */
+            pVmcbMemState->u64RFlags       = pVCpu->cpum.GstCtx.rflags.u;
+            pVmcbMemState->u64RIP          = pVCpu->cpum.GstCtx.rip;
+            pVmcbMemState->u64RSP          = pVCpu->cpum.GstCtx.rsp;
+            pVmcbMemState->u64RAX          = pVCpu->cpum.GstCtx.rax;
+            pVmcbMemState->u64DR7          = pVCpu->cpum.GstCtx.dr[7];
+            pVmcbMemState->u64DR6          = pVCpu->cpum.GstCtx.dr[6];
+            pVmcbMemState->u8CPL           = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl;   /* See comment in CPUMGetGuestCPL(). */
+            Assert(CPUMGetGuestCPL(pVCpu) == pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl);
+            if (CPUMIsGuestSvmNestedPagingEnabled(pVCpu, IEM_GET_CTX(pVCpu)))
+                pVmcbMemState->u64PAT = pVCpu->cpum.GstCtx.msrPAT;
+
+            /*
+             * Save additional state and intercept information.
+             *
+             *   - V_IRQ: Tracked using VMCPU_FF_INTERRUPT_NESTED_GUEST force-flag and updated below.
+             *   - V_TPR: Updated by iemCImpl_load_CrX or by the physical CPU for hardware-assisted
+             *     SVM execution.
+             *   - Interrupt shadow: Tracked using VMCPU_FF_INHIBIT_INTERRUPTS and RIP.
+             */
+            PSVMVMCBCTRL pVmcbMemCtrl = &pVmcbMem->ctrl;
+            if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))           /* V_IRQ. */
+                pVmcbMemCtrl->IntCtrl.n.u1VIrqPending = 0;
+            else
+            {
+                Assert(pVmcbCtrl->IntCtrl.n.u1VIrqPending);
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
+            }
+
+            pVmcbMemCtrl->IntCtrl.n.u8VTPR = pVmcbCtrl->IntCtrl.n.u8VTPR;           /* V_TPR. */
+
+            if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx))            /* Interrupt shadow. */
+                pVmcbMemCtrl->IntShadow.n.u1IntShadow = 0;
+            else
+            {
+                pVmcbMemCtrl->IntShadow.n.u1IntShadow = 1;
+                LogFlow(("iemSvmVmexit: Interrupt shadow till %#RX64\n", pVCpu->cpum.GstCtx.rip));
+                CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
+            }
+
+            /*
+             * Save nRIP, instruction length and byte fields.
+             */
+            pVmcbMemCtrl->u64NextRIP     = pVmcbCtrl->u64NextRIP;
+            pVmcbMemCtrl->cbInstrFetched = pVmcbCtrl->cbInstrFetched;
+            memcpy(&pVmcbMemCtrl->abInstr[0], &pVmcbCtrl->abInstr[0], sizeof(pVmcbMemCtrl->abInstr));
+
+            /*
+             * Save exit information.
+             */
+            pVmcbMemCtrl->u64ExitCode  = uExitCode;
+            pVmcbMemCtrl->u64ExitInfo1 = uExitInfo1;
+            pVmcbMemCtrl->u64ExitInfo2 = uExitInfo2;
+
+            /*
+             * Update the exit interrupt-information field if this #VMEXIT happened as a result
+             * of delivering an event through IEM.
+             *
+             * Don't update the exit interrupt-information field if the event wasn't being injected
+             * through IEM, as it would have been updated by real hardware if the nested-guest was
+             * executed using hardware-assisted SVM.
+             */
+            {
+                uint8_t  uExitIntVector;
+                uint32_t uExitIntErr;
+                uint32_t fExitIntFlags;
+                bool const fRaisingEvent = IEMGetCurrentXcpt(pVCpu, &uExitIntVector, &fExitIntFlags, &uExitIntErr,
+                                                             NULL /* uExitIntCr2 */);
+                if (fRaisingEvent)
+                {
+                    pVmcbCtrl->ExitIntInfo.n.u1Valid  = 1;
+                    pVmcbCtrl->ExitIntInfo.n.u8Vector = uExitIntVector;
+                    pVmcbCtrl->ExitIntInfo.n.u3Type   = iemGetSvmEventType(uExitIntVector, fExitIntFlags);
+                    if (fExitIntFlags & IEM_XCPT_FLAGS_ERR)
+                    {
+                        pVmcbCtrl->ExitIntInfo.n.u1ErrorCodeValid = true;
+                        pVmcbCtrl->ExitIntInfo.n.u32ErrorCode     = uExitIntErr;
+                    }
+                }
+            }
+
+            /*
+             * Save the exit interrupt-information field.
+             *
+             * We write the whole field including overwriting reserved bits as it was observed on an
+             * AMD Ryzen 5 Pro 1500 that the CPU does not preserve reserved bits in EXITINTINFO.
+             */
+            pVmcbMemCtrl->ExitIntInfo = pVmcbCtrl->ExitIntInfo;
+
+            /*
+             * Clear event injection.
+             */
+            pVmcbMemCtrl->EventInject.n.u1Valid = 0;
+
+            iemMemPageUnmap(pVCpu, pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb, IEM_ACCESS_DATA_RW, pVmcbMem, &PgLockMem);
+        }
+
+        /*
+         * Prepare for guest's "host mode" by clearing internal processor state bits.
+         *
+         * We don't need to zero out the state-save area, just the controls should be
+         * sufficient because it has the critical bit of indicating whether we're inside
+         * the nested-guest or not.
+         */
+        memset(pVmcbCtrl, 0, sizeof(*pVmcbCtrl));
+        Assert(!CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
+
+        /*
+         * Restore the subset of the inhibit flags that were preserved.
+         */
+        pVCpu->cpum.GstCtx.eflags.uBoth |= pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit;
+
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /** @todo Nested paging. */
+            /** @todo ASID. */
+
+            /*
+             * If we are switching to PAE mode host, validate the PDPEs first.
+             * Any invalid PDPEs here causes a VCPU shutdown.
+             */
+            PCSVMHOSTSTATE pHostState = &pVCpu->cpum.GstCtx.hwvirt.svm.HostState;
+            bool const fHostInPaeMode = CPUMIsPaePagingEnabled(pHostState->uCr0, pHostState->uCr4, pHostState->uEferMsr);
+            if (fHostInPaeMode)
+                rcStrict = PGMGstMapPaePdpesAtCr3(pVCpu, pHostState->uCr3);
+            if (RT_SUCCESS(rcStrict))
+            {
+                /*
+                 * Reload the host state.
+                 */
+                CPUMSvmVmExitRestoreHostState(pVCpu, IEM_GET_CTX(pVCpu));
+
+                /*
+                 * Update PGM, IEM and others of a world-switch.
+                 */
+                rcStrict = iemSvmWorldSwitch(pVCpu);
+                if (rcStrict == VINF_SUCCESS)
+                    rcStrict = VINF_SVM_VMEXIT;
+                else if (RT_SUCCESS(rcStrict))
+                {
+                    LogFlow(("iemSvmVmexit: Setting passup status from iemSvmWorldSwitch %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+                    iemSetPassUpStatus(pVCpu, rcStrict);
+                    rcStrict = VINF_SVM_VMEXIT;
+                }
+                else
+                    LogFlow(("iemSvmVmexit: iemSvmWorldSwitch unexpected failure. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            }
+            else
+            {
+                Log(("iemSvmVmexit: PAE PDPEs invalid while restoring host state. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+                rcStrict = VINF_EM_TRIPLE_FAULT;
+            }
+        }
+        else
+        {
+            AssertMsgFailed(("iemSvmVmexit: Mapping VMCB at %#RGp failed. rc=%Rrc\n", pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb, VBOXSTRICTRC_VAL(rcStrict)));
+            rcStrict = VINF_EM_TRIPLE_FAULT;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("iemSvmVmexit: Not in SVM guest mode! uExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uExitCode, uExitInfo1, uExitInfo2));
+        rcStrict = VERR_SVM_IPE_3;
+    }
+
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+    /* CLGI/STGI may not have been intercepted and thus not executed in IEM. */
+    if (   HMIsEnabled(pVCpu->CTX_SUFF(pVM))
+        && HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM)))
+        return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, false);
+# endif
+    return rcStrict;
+}
+
+
+/**
+ * Interface for HM and EM to emulate \#VMEXIT.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uExitCode   The exit code.
+ * @param   uExitInfo1  The exit info. 1 field.
+ * @param   uExitInfo2  The exit info. 2 field.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecSvmVmexit(PVMCPUCC pVCpu, uint64_t uExitCode, uint64_t uExitInfo1, uint64_t uExitInfo2)
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMEXIT_MASK);
+    VBOXSTRICTRC rcStrict = iemSvmVmexit(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
+    if (pVCpu->iem.s.cActiveMappings)
+        iemMemRollback(pVCpu);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Performs the operations necessary that are part of the vmrun instruction
+ * execution in the guest.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS successfully executed VMRUN and entered nested-guest
+ *          code execution.
+ * @retval  VINF_SVM_VMEXIT when executing VMRUN causes a \#VMEXIT
+ *          (SVM_EXIT_INVALID most likely).
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The length of the VMRUN instruction.
+ * @param   GCPhysVmcb          Guest physical address of the VMCB to run.
+ */
+static VBOXSTRICTRC iemSvmVmrun(PVMCPUCC pVCpu, uint8_t cbInstr, RTGCPHYS GCPhysVmcb) RT_NOEXCEPT
+{
+    LogFlow(("iemSvmVmrun\n"));
+
+    /*
+     * Cache the physical address of the VMCB for #VMEXIT exceptions.
+     */
+    pVCpu->cpum.GstCtx.hwvirt.svm.GCPhysVmcb = GCPhysVmcb;
+
+    /*
+     * Save the host state.
+     */
+    CPUMSvmVmRunSaveHostState(IEM_GET_CTX(pVCpu), cbInstr);
+
+    /*
+     * Read the guest VMCB.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    int rc = PGMPhysSimpleReadGCPhys(pVM, &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb, GCPhysVmcb, sizeof(SVMVMCB));
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * AMD-V seems to preserve reserved fields and only writes back selected, recognized
+         * fields on #VMEXIT. However, not all reserved  bits are preserved (e.g, EXITINTINFO)
+         * but in our implementation we try to preserve as much as we possibly can.
+         *
+         * We could read the entire page here and only write back the relevant fields on
+         * #VMEXIT but since our internal VMCB is also being used by HM during hardware-assisted
+         * SVM execution, it creates a potential for a nested-hypervisor to set bits that are
+         * currently reserved but may be recognized as features bits in future CPUs causing
+         * unexpected & undesired results. Hence, we zero out unrecognized fields here as we
+         * typically enter hardware-assisted SVM soon anyway, see @bugref{7243#c113}.
+         */
+        PSVMVMCBCTRL      pVmcbCtrl   = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl;
+        PSVMVMCBSTATESAVE pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.guest;
+
+        RT_ZERO(pVmcbCtrl->u8Reserved0);
+        RT_ZERO(pVmcbCtrl->u8Reserved1);
+        RT_ZERO(pVmcbCtrl->u8Reserved2);
+        RT_ZERO(pVmcbNstGst->u8Reserved0);
+        RT_ZERO(pVmcbNstGst->u8Reserved1);
+        RT_ZERO(pVmcbNstGst->u8Reserved2);
+        RT_ZERO(pVmcbNstGst->u8Reserved3);
+        RT_ZERO(pVmcbNstGst->u8Reserved4);
+        RT_ZERO(pVmcbNstGst->u8Reserved5);
+        pVmcbCtrl->u32Reserved0                   = 0;
+        pVmcbCtrl->TLBCtrl.n.u24Reserved          = 0;
+        pVmcbCtrl->IntCtrl.n.u6Reserved           = 0;
+        pVmcbCtrl->IntCtrl.n.u3Reserved           = 0;
+        pVmcbCtrl->IntCtrl.n.u5Reserved           = 0;
+        pVmcbCtrl->IntCtrl.n.u24Reserved          = 0;
+        pVmcbCtrl->IntShadow.n.u30Reserved        = 0;
+        pVmcbCtrl->ExitIntInfo.n.u19Reserved      = 0;
+        pVmcbCtrl->NestedPagingCtrl.n.u29Reserved = 0;
+        pVmcbCtrl->EventInject.n.u19Reserved      = 0;
+        pVmcbCtrl->LbrVirt.n.u30Reserved          = 0;
+
+        /*
+         * Validate guest-state and controls.
+         */
+        /* VMRUN must always be intercepted. */
+        if (!CPUMIsGuestSvmCtrlInterceptSet(pVCpu, IEM_GET_CTX(pVCpu), SVM_CTRL_INTERCEPT_VMRUN))
+        {
+            Log(("iemSvmVmrun: VMRUN instruction not intercepted -> #VMEXIT\n"));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* Nested paging. */
+        if (    pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
+            && !pVM->cpum.ro.GuestFeatures.fSvmNestedPaging)
+        {
+            Log(("iemSvmVmrun: Nested paging not supported -> Disabling\n"));
+            pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging = 0;
+        }
+
+        /* AVIC. */
+        if (    pVmcbCtrl->IntCtrl.n.u1AvicEnable
+            && !pVM->cpum.ro.GuestFeatures.fSvmAvic)
+        {
+            Log(("iemSvmVmrun: AVIC not supported -> Disabling\n"));
+            pVmcbCtrl->IntCtrl.n.u1AvicEnable = 0;
+        }
+
+        /* Last branch record (LBR) virtualization. */
+        if (    pVmcbCtrl->LbrVirt.n.u1LbrVirt
+            && !pVM->cpum.ro.GuestFeatures.fSvmLbrVirt)
+        {
+            Log(("iemSvmVmrun: LBR virtualization not supported -> Disabling\n"));
+            pVmcbCtrl->LbrVirt.n.u1LbrVirt = 0;
+        }
+
+        /* Virtualized VMSAVE/VMLOAD. */
+        if (    pVmcbCtrl->LbrVirt.n.u1VirtVmsaveVmload
+            && !pVM->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
+        {
+            Log(("iemSvmVmrun: Virtualized VMSAVE/VMLOAD not supported -> Disabling\n"));
+            pVmcbCtrl->LbrVirt.n.u1VirtVmsaveVmload = 0;
+        }
+
+        /* Virtual GIF. */
+        if (    pVmcbCtrl->IntCtrl.n.u1VGifEnable
+            && !pVM->cpum.ro.GuestFeatures.fSvmVGif)
+        {
+            Log(("iemSvmVmrun: Virtual GIF not supported -> Disabling\n"));
+            pVmcbCtrl->IntCtrl.n.u1VGifEnable = 0;
+        }
+
+        /* Guest ASID. */
+        if (!pVmcbCtrl->TLBCtrl.n.u32ASID)
+        {
+            Log(("iemSvmVmrun: Guest ASID is invalid -> #VMEXIT\n"));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* Guest AVIC. */
+        if (    pVmcbCtrl->IntCtrl.n.u1AvicEnable
+            && !pVM->cpum.ro.GuestFeatures.fSvmAvic)
+        {
+            Log(("iemSvmVmrun: AVIC not supported -> Disabling\n"));
+            pVmcbCtrl->IntCtrl.n.u1AvicEnable = 0;
+        }
+
+        /* Guest Secure Encrypted Virtualization. */
+        if (  (   pVmcbCtrl->NestedPagingCtrl.n.u1Sev
+               || pVmcbCtrl->NestedPagingCtrl.n.u1SevEs)
+            && !pVM->cpum.ro.GuestFeatures.fSvmAvic)
+        {
+            Log(("iemSvmVmrun: SEV not supported -> Disabling\n"));
+            pVmcbCtrl->NestedPagingCtrl.n.u1Sev = 0;
+            pVmcbCtrl->NestedPagingCtrl.n.u1SevEs = 0;
+        }
+
+        /* Flush by ASID. */
+        if (   !pVM->cpum.ro.GuestFeatures.fSvmFlusbByAsid
+            &&  pVmcbCtrl->TLBCtrl.n.u8TLBFlush != SVM_TLB_FLUSH_NOTHING
+            &&  pVmcbCtrl->TLBCtrl.n.u8TLBFlush != SVM_TLB_FLUSH_ENTIRE)
+        {
+            Log(("iemSvmVmrun: Flush-by-ASID not supported -> #VMEXIT\n"));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* IO permission bitmap. */
+        RTGCPHYS const GCPhysIOBitmap = pVmcbCtrl->u64IOPMPhysAddr;
+        if (   (GCPhysIOBitmap & X86_PAGE_4K_OFFSET_MASK)
+            || !PGMPhysIsGCPhysNormal(pVM, GCPhysIOBitmap)
+            || !PGMPhysIsGCPhysNormal(pVM, GCPhysIOBitmap + X86_PAGE_4K_SIZE)
+            || !PGMPhysIsGCPhysNormal(pVM, GCPhysIOBitmap + (X86_PAGE_4K_SIZE << 1)))
+        {
+            Log(("iemSvmVmrun: IO bitmap physaddr invalid. GCPhysIOBitmap=%#RX64 -> #VMEXIT\n", GCPhysIOBitmap));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* MSR permission bitmap. */
+        RTGCPHYS const GCPhysMsrBitmap = pVmcbCtrl->u64MSRPMPhysAddr;
+        if (   (GCPhysMsrBitmap & X86_PAGE_4K_OFFSET_MASK)
+            || !PGMPhysIsGCPhysNormal(pVM, GCPhysMsrBitmap)
+            || !PGMPhysIsGCPhysNormal(pVM, GCPhysMsrBitmap + X86_PAGE_4K_SIZE))
+        {
+            Log(("iemSvmVmrun: MSR bitmap physaddr invalid. GCPhysMsrBitmap=%#RX64 -> #VMEXIT\n", GCPhysMsrBitmap));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* CR0. */
+        if (   !(pVmcbNstGst->u64CR0 & X86_CR0_CD)
+            &&  (pVmcbNstGst->u64CR0 & X86_CR0_NW))
+        {
+            Log(("iemSvmVmrun: CR0 no-write through with cache disabled. CR0=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64CR0));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+        if (pVmcbNstGst->u64CR0 >> 32)
+        {
+            Log(("iemSvmVmrun: CR0 reserved bits set. CR0=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64CR0));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+        /** @todo Implement all reserved bits/illegal combinations for CR3, CR4. */
+
+        /* DR6 and DR7. */
+        if (   pVmcbNstGst->u64DR6 >> 32
+            || pVmcbNstGst->u64DR7 >> 32)
+        {
+            Log(("iemSvmVmrun: DR6 and/or DR7 reserved bits set. DR6=%#RX64 DR7=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64DR6,
+                 pVmcbNstGst->u64DR6));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /*
+         * PAT (Page Attribute Table) MSR.
+         *
+         * The CPU only validates and loads it when nested-paging is enabled.
+         * See AMD spec. "15.25.4 Nested Paging and VMRUN/#VMEXIT".
+         */
+        if (   pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
+            && !CPUMIsPatMsrValid(pVmcbNstGst->u64PAT))
+        {
+            Log(("iemSvmVmrun: PAT invalid. u64PAT=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64PAT));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /*
+         * Copy the IO permission bitmap into the cache.
+         */
+        AssertCompile(sizeof(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap) == SVM_IOPM_PAGES * X86_PAGE_4K_SIZE);
+        rc = PGMPhysSimpleReadGCPhys(pVM, pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, GCPhysIOBitmap,
+                                     sizeof(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap));
+        if (RT_FAILURE(rc))
+        {
+            Log(("iemSvmVmrun: Failed reading the IO permission bitmap at %#RGp. rc=%Rrc\n", GCPhysIOBitmap, rc));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /*
+         * Copy the MSR permission bitmap into the cache.
+         */
+        AssertCompile(sizeof(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap) == SVM_MSRPM_PAGES * X86_PAGE_4K_SIZE);
+        rc = PGMPhysSimpleReadGCPhys(pVM, pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap, GCPhysMsrBitmap,
+                                     sizeof(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap));
+        if (RT_FAILURE(rc))
+        {
+            Log(("iemSvmVmrun: Failed reading the MSR permission bitmap at %#RGp. rc=%Rrc\n", GCPhysMsrBitmap, rc));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /*
+         * Copy segments from nested-guest VMCB state to the guest-CPU state.
+         *
+         * We do this here as we need to use the CS attributes and it's easier this way
+         * then using the VMCB format selectors. It doesn't really matter where we copy
+         * the state, we restore the guest-CPU context state on the \#VMEXIT anyway.
+         */
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, ES, es);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, CS, cs);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, SS, ss);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), pVmcbNstGst, DS, ds);
+
+        /** @todo Segment attribute overrides by VMRUN. */
+
+        /*
+         * CPL adjustments and overrides.
+         *
+         * SS.DPL is apparently the CPU's CPL, see comment in CPUMGetGuestCPL().
+         * We shall thus adjust both CS.DPL and SS.DPL here.
+         */
+        pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl = pVmcbNstGst->u8CPL;
+        if (CPUMIsGuestInV86ModeEx(IEM_GET_CTX(pVCpu)))
+            pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl = 3;
+        if (CPUMIsGuestInRealModeEx(IEM_GET_CTX(pVCpu)))
+            pVCpu->cpum.GstCtx.cs.Attr.n.u2Dpl = pVCpu->cpum.GstCtx.ss.Attr.n.u2Dpl = 0;
+        Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pVCpu->cpum.GstCtx.ss));
+
+        /*
+         * Continue validating guest-state and controls.
+         *
+         * We pass CR0 as 0 to CPUMIsGuestEferMsrWriteValid() below to skip the illegal
+         * EFER.LME bit transition check. We pass the nested-guest's EFER as both the
+         * old and new EFER value to not have any guest EFER bits influence the new
+         * nested-guest EFER.
+         */
+        uint64_t uValidEfer;
+        rc = CPUMIsGuestEferMsrWriteValid(pVM, 0 /* CR0 */, pVmcbNstGst->u64EFER, pVmcbNstGst->u64EFER, &uValidEfer);
+        if (RT_FAILURE(rc))
+        {
+            Log(("iemSvmVmrun: EFER invalid uOldEfer=%#RX64 -> #VMEXIT\n", pVmcbNstGst->u64EFER));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /* Validate paging and CPU mode bits. */
+        bool const fSvm                     = RT_BOOL(uValidEfer & MSR_K6_EFER_SVME);
+        bool const fLongModeSupported       = RT_BOOL(pVM->cpum.ro.GuestFeatures.fLongMode);
+        bool const fLongModeEnabled         = RT_BOOL(uValidEfer & MSR_K6_EFER_LME);
+        bool const fPaging                  = RT_BOOL(pVmcbNstGst->u64CR0 & X86_CR0_PG);
+        bool const fPae                     = RT_BOOL(pVmcbNstGst->u64CR4 & X86_CR4_PAE);
+        bool const fProtMode                = RT_BOOL(pVmcbNstGst->u64CR0 & X86_CR0_PE);
+        bool const fLongModeWithPaging      = fLongModeEnabled && fPaging;
+        bool const fLongModeConformCS       = pVCpu->cpum.GstCtx.cs.Attr.n.u1Long && pVCpu->cpum.GstCtx.cs.Attr.n.u1DefBig;
+        /* Adjust EFER.LMA (this is normally done by the CPU when system software writes CR0). */
+        if (fLongModeWithPaging)
+            uValidEfer |= MSR_K6_EFER_LMA;
+        bool const fLongModeActiveOrEnabled = RT_BOOL(uValidEfer & (MSR_K6_EFER_LME | MSR_K6_EFER_LMA));
+        if (   !fSvm
+            || (!fLongModeSupported && fLongModeActiveOrEnabled)
+            || (fLongModeWithPaging && !fPae)
+            || (fLongModeWithPaging && !fProtMode)
+            || (   fLongModeEnabled
+                && fPaging
+                && fPae
+                && fLongModeConformCS))
+        {
+            Log(("iemSvmVmrun: EFER invalid. uValidEfer=%#RX64 -> #VMEXIT\n", uValidEfer));
+            return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+        }
+
+        /*
+         * Preserve the required force-flags.
+         *
+         * We only preserve the force-flags that would affect the execution of the
+         * nested-guest (or the guest).
+         *
+         *   - VMCPU_FF_BLOCK_NMIS needs to be preserved as it blocks NMI until the
+         *     execution of a subsequent IRET instruction in the guest.
+         *
+         * The remaining FFs (e.g. timers) can stay in place so that we will be able to
+         * generate interrupts that should cause #VMEXITs for the nested-guest.
+         *
+         * VMRUN has implicit GIF (Global Interrupt Flag) handling, we don't need to
+         * preserve VMCPU_FF_INHIBIT_INTERRUPTS.
+         */
+        pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_INHIBIT_NMI;
+        pVCpu->cpum.GstCtx.eflags.uBoth        &=                                  ~CPUMCTX_INHIBIT_NMI;
+
+        /*
+         * Pause filter.
+         */
+        if (pVM->cpum.ro.GuestFeatures.fSvmPauseFilter)
+        {
+            pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter = pVmcbCtrl->u16PauseFilterCount;
+            if (pVM->cpum.ro.GuestFeatures.fSvmPauseFilterThreshold)
+                pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilterThreshold = pVmcbCtrl->u16PauseFilterCount;
+        }
+
+        /*
+         * Interrupt shadow.
+         */
+        if (pVmcbCtrl->IntShadow.n.u1IntShadow)
+        {
+            LogFlow(("iemSvmVmrun: setting interrupt shadow. inhibit PC=%#RX64\n", pVmcbNstGst->u64RIP));
+            /** @todo will this cause trouble if the nested-guest is 64-bit but the guest is 32-bit? */
+            CPUMSetInInterruptShadowEx(&pVCpu->cpum.GstCtx, pVmcbNstGst->u64RIP);
+        }
+
+        /*
+         * TLB flush control.
+         * Currently disabled since it's redundant as we unconditionally flush the TLB
+         * in iemSvmWorldSwitch() below.
+         */
+# if 0
+        /** @todo @bugref{7243}: ASID based PGM TLB flushes. */
+        if (   pVmcbCtrl->TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE
+            || pVmcbCtrl->TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
+            || pVmcbCtrl->TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
+            PGMFlushTLB(pVCpu, pVmcbNstGst->u64CR3, true /* fGlobal */);
+# endif
+
+        /*
+         * Validate and map PAE PDPEs if the guest will be using PAE paging.
+         * Invalid PAE PDPEs here causes a #VMEXIT.
+         */
+        if (   !pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
+            && CPUMIsPaePagingEnabled(pVmcbNstGst->u64CR0, pVmcbNstGst->u64CR4, uValidEfer))
+        {
+            rc = PGMGstMapPaePdpesAtCr3(pVCpu, pVmcbNstGst->u64CR3);
+            if (RT_SUCCESS(rc))
+            { /* likely */ }
+            else
+            {
+                Log(("iemSvmVmrun: PAE PDPEs invalid -> #VMEXIT\n"));
+                return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+            }
+        }
+
+        /*
+         * Copy the remaining guest state from the VMCB to the guest-CPU context.
+         */
+        pVCpu->cpum.GstCtx.gdtr.cbGdt = pVmcbNstGst->GDTR.u32Limit;
+        pVCpu->cpum.GstCtx.gdtr.pGdt  = pVmcbNstGst->GDTR.u64Base;
+        pVCpu->cpum.GstCtx.idtr.cbIdt = pVmcbNstGst->IDTR.u32Limit;
+        pVCpu->cpum.GstCtx.idtr.pIdt  = pVmcbNstGst->IDTR.u64Base;
+        CPUMSetGuestCR0(pVCpu, pVmcbNstGst->u64CR0);
+        CPUMSetGuestCR4(pVCpu, pVmcbNstGst->u64CR4);
+        pVCpu->cpum.GstCtx.cr3        = pVmcbNstGst->u64CR3;
+        pVCpu->cpum.GstCtx.cr2        = pVmcbNstGst->u64CR2;
+        pVCpu->cpum.GstCtx.dr[6]      = pVmcbNstGst->u64DR6;
+        pVCpu->cpum.GstCtx.dr[7]      = pVmcbNstGst->u64DR7;
+        pVCpu->cpum.GstCtx.rflags.u   = pVmcbNstGst->u64RFlags;
+        pVCpu->cpum.GstCtx.rax        = pVmcbNstGst->u64RAX;
+        pVCpu->cpum.GstCtx.rsp        = pVmcbNstGst->u64RSP;
+        pVCpu->cpum.GstCtx.rip        = pVmcbNstGst->u64RIP;
+        CPUMSetGuestEferMsrNoChecks(pVCpu, pVCpu->cpum.GstCtx.msrEFER, uValidEfer);
+        if (pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging)
+            pVCpu->cpum.GstCtx.msrPAT = pVmcbNstGst->u64PAT;
+
+        /* Mask DR6, DR7 bits mandatory set/clear bits. */
+        pVCpu->cpum.GstCtx.dr[6] &= ~(X86_DR6_RAZ_MASK | X86_DR6_MBZ_MASK);
+        pVCpu->cpum.GstCtx.dr[6] |= X86_DR6_RA1_MASK;
+        pVCpu->cpum.GstCtx.dr[7] &= ~(X86_DR7_RAZ_MASK | X86_DR7_MBZ_MASK);
+        pVCpu->cpum.GstCtx.dr[7] |= X86_DR7_RA1_MASK;
+
+        /*
+         * Check for pending virtual interrupts.
+         */
+        if (pVmcbCtrl->IntCtrl.n.u1VIrqPending)
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
+        else
+            Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST));
+
+        /*
+         * Update PGM, IEM and others of a world-switch.
+         */
+        VBOXSTRICTRC rcStrict = iemSvmWorldSwitch(pVCpu);
+        if (rcStrict == VINF_SUCCESS)
+        { /* likely */ }
+        else if (RT_SUCCESS(rcStrict))
+        {
+            LogFlow(("iemSvmVmrun: iemSvmWorldSwitch returned %Rrc, setting passup status\n", VBOXSTRICTRC_VAL(rcStrict)));
+            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+        }
+        else
+        {
+            LogFlow(("iemSvmVmrun: iemSvmWorldSwitch unexpected failure. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+
+        /*
+         * Set the global-interrupt flag to allow interrupts in the guest.
+         */
+        CPUMSetGuestGif(&pVCpu->cpum.GstCtx, true);
+
+        /*
+         * Event injection.
+         */
+        PCSVMEVENT pEventInject = &pVmcbCtrl->EventInject;
+        pVCpu->cpum.GstCtx.hwvirt.svm.fInterceptEvents = !pEventInject->n.u1Valid;
+        if (pEventInject->n.u1Valid)
+        {
+            uint8_t   const uVector    = pEventInject->n.u8Vector;
+            TRPMEVENT const enmType    = HMSvmEventToTrpmEventType(pEventInject, uVector);
+            uint16_t  const uErrorCode = pEventInject->n.u1ErrorCodeValid ? pEventInject->n.u32ErrorCode : 0;
+
+            /* Validate vectors for hardware exceptions, see AMD spec. 15.20 "Event Injection". */
+            if (RT_UNLIKELY(enmType == TRPM_32BIT_HACK))
+            {
+                Log(("iemSvmVmrun: Invalid event type =%#x -> #VMEXIT\n", (uint8_t)pEventInject->n.u3Type));
+                return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+            }
+            if (pEventInject->n.u3Type == SVM_EVENT_EXCEPTION)
+            {
+                if (   uVector == X86_XCPT_NMI
+                    || uVector > X86_XCPT_LAST)
+                {
+                    Log(("iemSvmVmrun: Invalid vector for hardware exception. uVector=%#x -> #VMEXIT\n", uVector));
+                    return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+                }
+                if (   uVector == X86_XCPT_BR
+                    && CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)))
+                {
+                    Log(("iemSvmVmrun: Cannot inject #BR when not in long mode -> #VMEXIT\n"));
+                    return iemSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+                }
+                /** @todo any others? */
+            }
+
+            /*
+             * Invalidate the exit interrupt-information field here. This field is fully updated
+             * on #VMEXIT as events other than the one below can also cause intercepts during
+             * their injection (e.g. exceptions).
+             */
+            pVmcbCtrl->ExitIntInfo.n.u1Valid = 0;
+
+            /*
+             * Clear the event injection valid bit here. While the AMD spec. mentions that the CPU
+             * clears this bit from the VMCB unconditionally on #VMEXIT, internally the CPU could be
+             * clearing it at any time, most likely before/after injecting the event. Since VirtualBox
+             * doesn't have any virtual-CPU internal representation of this bit, we clear/update the
+             * VMCB here. This also has the added benefit that we avoid the risk of injecting the event
+             * twice if we fallback to executing the nested-guest using hardware-assisted SVM after
+             * injecting the event through IEM here.
+             */
+            pVmcbCtrl->EventInject.n.u1Valid = 0;
+
+            /** @todo NRIP: Software interrupts can only be pushed properly if we support
+             *        NRIP for the nested-guest to calculate the instruction length
+             *        below. */
+            LogFlow(("iemSvmVmrun: Injecting event: %04x:%08RX64 vec=%#x type=%d uErr=%u cr2=%#RX64 cr3=%#RX64 efer=%#RX64\n",
+                     pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uVector, enmType, uErrorCode, pVCpu->cpum.GstCtx.cr2,
+                     pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.msrEFER));
+
+            /*
+             * We shall not inject the event here right away. There may be paging mode related updates
+             * as a result of the world-switch above that are yet to be honored. Instead flag the event
+             * as pending for injection.
+             */
+            TRPMAssertTrap(pVCpu, uVector, enmType);
+            if (pEventInject->n.u1ErrorCodeValid)
+                TRPMSetErrorCode(pVCpu, uErrorCode);
+            if (   enmType == TRPM_TRAP
+                && uVector == X86_XCPT_PF)
+                TRPMSetFaultAddress(pVCpu, pVCpu->cpum.GstCtx.cr2);
+        }
+        else
+            LogFlow(("iemSvmVmrun: Entering nested-guest: %04x:%08RX64 cr0=%#RX64 cr3=%#RX64 cr4=%#RX64 efer=%#RX64 efl=%#x\n",
+                     pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3,
+                     pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER, pVCpu->cpum.GstCtx.eflags.u));
+
+        LogFlow(("iemSvmVmrun: returns %d\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+        /* If CLGI/STGI isn't intercepted we force IEM-only nested-guest execution here. */
+        if (   HMIsEnabled(pVM)
+            && HMIsSvmVGifActive(pVM))
+            return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, true);
+# endif
+
+        return rcStrict;
+    }
+
+    /* Shouldn't really happen as the caller should've validated the physical address already. */
+    Log(("iemSvmVmrun: Failed to read nested-guest VMCB at %#RGp (rc=%Rrc) -> #VMEXIT\n", GCPhysVmcb, rc));
+    return rc;
+}
+
+
+/**
+ * Checks if the event intercepts and performs the \#VMEXIT if the corresponding
+ * intercept is active.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_HM_INTERCEPT_NOT_ACTIVE if the intercept is not active or
+ *          we're not executing a nested-guest.
+ * @retval  VINF_SVM_VMEXIT if the intercept is active and the \#VMEXIT occurred
+ *          successfully.
+ * @retval  VERR_SVM_VMEXIT_FAILED if the intercept is active and the \#VMEXIT
+ *          failed and a shutdown needs to be initiated for the guest.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling thread.
+ * @param   u8Vector    The interrupt or exception vector.
+ * @param   fFlags      The exception flags (see IEM_XCPT_FLAGS_XXX).
+ * @param   uErr        The error-code associated with the exception.
+ * @param   uCr2        The CR2 value in case of a \#PF exception.
+ */
+VBOXSTRICTRC iemHandleSvmEventIntercept(PVMCPUCC pVCpu, uint8_t u8Vector, uint32_t fFlags, uint32_t uErr, uint64_t uCr2) RT_NOEXCEPT
+{
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
+
+    /*
+     * Handle SVM exception and software interrupt intercepts, see AMD spec. 15.12 "Exception Intercepts".
+     *
+     *   - NMI intercepts have their own exit code and do not cause SVM_EXIT_XCPT_2 #VMEXITs.
+     *   - External interrupts and software interrupts (INTn instruction) do not check the exception intercepts
+     *     even when they use a vector in the range 0 to 31.
+     *   - ICEBP should not trigger #DB intercept, but its own intercept.
+     *   - For #PF exceptions, its intercept is checked before CR2 is written by the exception.
+     */
+    /* Check NMI intercept */
+    if (   u8Vector == X86_XCPT_NMI
+        && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+        && IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_NMI))
+    {
+        Log2(("iemHandleSvmNstGstEventIntercept: NMI intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_NMI, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /* Check ICEBP intercept. */
+    if (   (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)
+        && IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_ICEBP))
+    {
+        Log2(("iemHandleSvmNstGstEventIntercept: ICEBP intercept -> #VMEXIT\n"));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_ICEBP, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /* Check CPU exception intercepts. */
+    if (   (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+        && IEM_SVM_IS_XCPT_INTERCEPT_SET(pVCpu, u8Vector))
+    {
+        Assert(u8Vector <= X86_XCPT_LAST);
+        uint64_t const uExitInfo1 = fFlags & IEM_XCPT_FLAGS_ERR ? uErr : 0;
+        uint64_t const uExitInfo2 = fFlags & IEM_XCPT_FLAGS_CR2 ? uCr2 : 0;
+        if (   IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists
+            && u8Vector == X86_XCPT_PF
+            && !(uErr & X86_TRAP_PF_ID))
+        {
+            PSVMVMCBCTRL  pVmcbCtrl = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl;
+# ifdef IEM_WITH_CODE_TLB
+            uint8_t const *pbInstrBuf = pVCpu->iem.s.pbInstrBuf;
+            uint8_t const  cbInstrBuf = pVCpu->iem.s.cbInstrBuf;
+            pVmcbCtrl->cbInstrFetched = RT_MIN(cbInstrBuf, SVM_CTRL_GUEST_INSTR_BYTES_MAX);
+            if (   pbInstrBuf
+                && cbInstrBuf > 0)
+                memcpy(&pVmcbCtrl->abInstr[0], pbInstrBuf, pVmcbCtrl->cbInstrFetched);
+# else
+            uint8_t const cbOpcode    = pVCpu->iem.s.cbOpcode;
+            pVmcbCtrl->cbInstrFetched = RT_MIN(cbOpcode, SVM_CTRL_GUEST_INSTR_BYTES_MAX);
+            if (cbOpcode > 0)
+                memcpy(&pVmcbCtrl->abInstr[0], &pVCpu->iem.s.abOpcode[0], pVmcbCtrl->cbInstrFetched);
+# endif
+        }
+        if (u8Vector == X86_XCPT_BR)
+            IEM_SVM_UPDATE_NRIP(pVCpu);
+        Log2(("iemHandleSvmNstGstEventIntercept: Xcpt intercept u32InterceptXcpt=%#RX32 u8Vector=%#x "
+              "uExitInfo1=%#RX64 uExitInfo2=%#RX64 -> #VMEXIT\n", pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb.ctrl.u32InterceptXcpt,
+              u8Vector, uExitInfo1, uExitInfo2));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_XCPT_0 + u8Vector, uExitInfo1, uExitInfo2);
+    }
+
+    /* Check software interrupt (INTn) intercepts. */
+    if (   (fFlags & (  IEM_XCPT_FLAGS_T_SOFT_INT
+                      | IEM_XCPT_FLAGS_BP_INSTR
+                      | IEM_XCPT_FLAGS_ICEBP_INSTR
+                      | IEM_XCPT_FLAGS_OF_INSTR)) == IEM_XCPT_FLAGS_T_SOFT_INT
+        && IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INTN))
+    {
+        uint64_t const uExitInfo1 = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmDecodeAssists ? u8Vector : 0;
+        Log2(("iemHandleSvmNstGstEventIntercept: Software INT intercept (u8Vector=%#x) -> #VMEXIT\n", u8Vector));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SWINT, uExitInfo1, 0 /* uExitInfo2 */);
+    }
+
+    return VINF_SVM_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Checks the SVM IO permission bitmap and performs the \#VMEXIT if the
+ * corresponding intercept is active.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_HM_INTERCEPT_NOT_ACTIVE if the intercept is not active or
+ *          we're not executing a nested-guest.
+ * @retval  VINF_SVM_VMEXIT if the intercept is active and the \#VMEXIT occurred
+ *          successfully.
+ * @retval  VERR_SVM_VMEXIT_FAILED if the intercept is active and the \#VMEXIT
+ *          failed and a shutdown needs to be initiated for the guest.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling thread.
+ * @param   u16Port         The IO port being accessed.
+ * @param   enmIoType       The type of IO access.
+ * @param   cbReg           The IO operand size in bytes.
+ * @param   cAddrSizeBits   The address size bits (for 16, 32 or 64).
+ * @param   iEffSeg         The effective segment number.
+ * @param   fRep            Whether this is a repeating IO instruction (REP prefix).
+ * @param   fStrIo          Whether this is a string IO instruction.
+ * @param   cbInstr         The length of the IO instruction in bytes.
+ */
+VBOXSTRICTRC iemSvmHandleIOIntercept(PVMCPUCC pVCpu, uint16_t u16Port, SVMIOIOTYPE enmIoType, uint8_t cbReg,
+                                     uint8_t cAddrSizeBits, uint8_t iEffSeg, bool fRep, bool fStrIo, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_IOIO_PROT));
+    Assert(cAddrSizeBits == 16 || cAddrSizeBits == 32 || cAddrSizeBits == 64);
+    Assert(cbReg == 1 || cbReg == 2 || cbReg == 4 || cbReg == 8);
+
+    Log3(("iemSvmHandleIOIntercept: u16Port=%#x (%u)\n", u16Port, u16Port));
+
+    SVMIOIOEXITINFO IoExitInfo;
+    bool const fIntercept = CPUMIsSvmIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap, u16Port, enmIoType, cbReg,
+                                                    cAddrSizeBits, iEffSeg, fRep, fStrIo, &IoExitInfo);
+    if (fIntercept)
+    {
+        Log3(("iemSvmHandleIOIntercept: u16Port=%#x (%u) -> #VMEXIT\n", u16Port, u16Port));
+        IEM_SVM_UPDATE_NRIP(pVCpu);
+        return iemSvmVmexit(pVCpu, SVM_EXIT_IOIO, IoExitInfo.u, pVCpu->cpum.GstCtx.rip + cbInstr);
+    }
+
+    /** @todo remove later (for debugging as VirtualBox always traps all IO
+     *        intercepts). */
+    AssertMsgFailed(("iemSvmHandleIOIntercept: We expect an IO intercept here!\n"));
+    return VINF_SVM_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Checks the SVM MSR permission bitmap and performs the \#VMEXIT if the
+ * corresponding intercept is active.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_HM_INTERCEPT_NOT_ACTIVE if the MSR permission bitmap does not
+ *          specify interception of the accessed MSR @a idMsr.
+ * @retval  VINF_SVM_VMEXIT if the intercept is active and the \#VMEXIT occurred
+ *          successfully.
+ * @retval  VERR_SVM_VMEXIT_FAILED if the intercept is active and the \#VMEXIT
+ *          failed and a shutdown needs to be initiated for the guest.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The MSR being accessed in the nested-guest.
+ * @param   fWrite      Whether this is an MSR write access, @c false implies an
+ *                      MSR read.
+ * @param   cbInstr     The length of the MSR read/write instruction in bytes.
+ */
+VBOXSTRICTRC iemSvmHandleMsrIntercept(PVMCPUCC pVCpu, uint32_t idMsr, bool fWrite) RT_NOEXCEPT
+{
+    /*
+     * Check if any MSRs are being intercepted.
+     */
+    Assert(CPUMIsGuestSvmCtrlInterceptSet(pVCpu, IEM_GET_CTX(pVCpu), SVM_CTRL_INTERCEPT_MSR_PROT));
+    Assert(CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
+
+    uint64_t const uExitInfo1 = fWrite ? SVM_EXIT1_MSR_WRITE : SVM_EXIT1_MSR_READ;
+
+    /*
+     * Get the byte and bit offset of the permission bits corresponding to the MSR.
+     */
+    uint16_t offMsrpm;
+    uint8_t  uMsrpmBit;
+    int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(uMsrpmBit == 0 || uMsrpmBit == 2 || uMsrpmBit == 4 || uMsrpmBit == 6);
+        Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
+        if (fWrite)
+            ++uMsrpmBit;
+
+        /*
+         * Check if the bit is set, if so, trigger a #VMEXIT.
+         */
+        if (pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit))
+        {
+            IEM_SVM_UPDATE_NRIP(pVCpu);
+            return iemSvmVmexit(pVCpu, SVM_EXIT_MSR, uExitInfo1, 0 /* uExitInfo2 */);
+        }
+    }
+    else
+    {
+        /*
+         * This shouldn't happen, but if it does, cause a #VMEXIT and let the "host" (nested hypervisor) deal with it.
+         */
+        Log(("iemSvmHandleMsrIntercept: Invalid/out-of-range MSR %#RX32 fWrite=%RTbool -> #VMEXIT\n", idMsr, fWrite));
+        return iemSvmVmexit(pVCpu, SVM_EXIT_MSR, uExitInfo1, 0 /* uExitInfo2 */);
+    }
+    return VINF_SVM_INTERCEPT_NOT_ACTIVE;
+}
+
+
+
+/**
+ * Implements 'VMRUN'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmrun)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF2(pVCpu, cbInstr);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    LogFlow(("iemCImpl_vmrun\n"));
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, vmrun);
+
+    /** @todo Check effective address size using address size prefix. */
+    RTGCPHYS const GCPhysVmcb = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
+    if (   (GCPhysVmcb & X86_PAGE_4K_OFFSET_MASK)
+        || !PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcb))
+    {
+        Log(("vmrun: VMCB physaddr (%#RGp) not valid -> #GP(0)\n", GCPhysVmcb));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMRUN))
+    {
+        Log(("vmrun: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMRUN, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    VBOXSTRICTRC rcStrict = iemSvmVmrun(pVCpu, cbInstr, GCPhysVmcb);
+    if (rcStrict == VERR_SVM_VMEXIT_FAILED)
+    {
+        Assert(!CPUMIsGuestInSvmNestedHwVirtMode(IEM_GET_CTX(pVCpu)));
+        rcStrict = VINF_EM_TRIPLE_FAULT;
+    }
+    return rcStrict;
+# endif
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMRUN instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmrun(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_SVM_VMRUN_MASK);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmrun);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Implements 'VMLOAD'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmload)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF2(pVCpu, cbInstr);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    LogFlow(("iemCImpl_vmload\n"));
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, vmload);
+
+    /** @todo Check effective address size using address size prefix. */
+    RTGCPHYS const GCPhysVmcb = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
+    if (   (GCPhysVmcb & X86_PAGE_4K_OFFSET_MASK)
+        || !PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcb))
+    {
+        Log(("vmload: VMCB physaddr (%#RGp) not valid -> #GP(0)\n", GCPhysVmcb));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMLOAD))
+    {
+        Log(("vmload: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMLOAD, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    SVMVMCBSTATESAVE VmcbNstGst;
+    VBOXSTRICTRC rcStrict = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcbNstGst, GCPhysVmcb + RT_UOFFSETOF(SVMVMCB, guest),
+                                                    sizeof(SVMVMCBSTATESAVE));
+    if (rcStrict == VINF_SUCCESS)
+    {
+        LogFlow(("vmload: Loading VMCB at %#RGp enmEffAddrMode=%d\n", GCPhysVmcb, pVCpu->iem.s.enmEffAddrMode));
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, FS, fs);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, GS, gs);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, TR, tr);
+        HMSVM_SEG_REG_COPY_FROM_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, LDTR, ldtr);
+
+        pVCpu->cpum.GstCtx.msrKERNELGSBASE = VmcbNstGst.u64KernelGSBase;
+        pVCpu->cpum.GstCtx.msrSTAR         = VmcbNstGst.u64STAR;
+        pVCpu->cpum.GstCtx.msrLSTAR        = VmcbNstGst.u64LSTAR;
+        pVCpu->cpum.GstCtx.msrCSTAR        = VmcbNstGst.u64CSTAR;
+        pVCpu->cpum.GstCtx.msrSFMASK       = VmcbNstGst.u64SFMASK;
+
+        pVCpu->cpum.GstCtx.SysEnter.cs     = VmcbNstGst.u64SysEnterCS;
+        pVCpu->cpum.GstCtx.SysEnter.esp    = VmcbNstGst.u64SysEnterESP;
+        pVCpu->cpum.GstCtx.SysEnter.eip    = VmcbNstGst.u64SysEnterEIP;
+
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+# endif
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMLOAD instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmload(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmload);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Implements 'VMSAVE'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmsave)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF2(pVCpu, cbInstr);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    LogFlow(("iemCImpl_vmsave\n"));
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, vmsave);
+
+    /** @todo Check effective address size using address size prefix. */
+    RTGCPHYS const GCPhysVmcb = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
+    if (   (GCPhysVmcb & X86_PAGE_4K_OFFSET_MASK)
+        || !PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcb))
+    {
+        Log(("vmsave: VMCB physaddr (%#RGp) not valid -> #GP(0)\n", GCPhysVmcb));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMSAVE))
+    {
+        Log(("vmsave: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMSAVE, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    SVMVMCBSTATESAVE VmcbNstGst;
+    VBOXSTRICTRC rcStrict = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcbNstGst, GCPhysVmcb + RT_UOFFSETOF(SVMVMCB, guest),
+                                                    sizeof(SVMVMCBSTATESAVE));
+    if (rcStrict == VINF_SUCCESS)
+    {
+        LogFlow(("vmsave: Saving VMCB at %#RGp enmEffAddrMode=%d\n", GCPhysVmcb, pVCpu->iem.s.enmEffAddrMode));
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_FS | CPUMCTX_EXTRN_GS | CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_LDTR
+                                | CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_SYSENTER_MSRS);
+
+        HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, FS, fs);
+        HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, GS, gs);
+        HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, TR, tr);
+        HMSVM_SEG_REG_COPY_TO_VMCB(IEM_GET_CTX(pVCpu), &VmcbNstGst, LDTR, ldtr);
+
+        VmcbNstGst.u64KernelGSBase  = pVCpu->cpum.GstCtx.msrKERNELGSBASE;
+        VmcbNstGst.u64STAR          = pVCpu->cpum.GstCtx.msrSTAR;
+        VmcbNstGst.u64LSTAR         = pVCpu->cpum.GstCtx.msrLSTAR;
+        VmcbNstGst.u64CSTAR         = pVCpu->cpum.GstCtx.msrCSTAR;
+        VmcbNstGst.u64SFMASK        = pVCpu->cpum.GstCtx.msrSFMASK;
+
+        VmcbNstGst.u64SysEnterCS    = pVCpu->cpum.GstCtx.SysEnter.cs;
+        VmcbNstGst.u64SysEnterESP   = pVCpu->cpum.GstCtx.SysEnter.esp;
+        VmcbNstGst.u64SysEnterEIP   = pVCpu->cpum.GstCtx.SysEnter.eip;
+
+        rcStrict = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmcb + RT_UOFFSETOF(SVMVMCB, guest), &VmcbNstGst,
+                                            sizeof(SVMVMCBSTATESAVE));
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+# endif
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMSAVE instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmsave(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmsave);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Implements 'CLGI'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_clgi)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF2(pVCpu, cbInstr);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    LogFlow(("iemCImpl_clgi\n"));
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, clgi);
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_CLGI))
+    {
+        Log(("clgi: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_CLGI, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    CPUMSetGuestGif(&pVCpu->cpum.GstCtx, false);
+
+#  if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+    iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, true);
+#  else
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+#  endif
+# endif
+}
+
+
+/**
+ * Interface for HM and EM to emulate the CLGI instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedClgi(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_clgi);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Implements 'STGI'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_stgi)
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF2(pVCpu, cbInstr);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    LogFlow(("iemCImpl_stgi\n"));
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, stgi);
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_STGI))
+    {
+        Log2(("stgi: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_STGI, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    CPUMSetGuestGif(&pVCpu->cpum.GstCtx, true);
+
+#  if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+    iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    return EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, false);
+#  else
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+#  endif
+# endif
+}
+
+
+/**
+ * Interface for HM and EM to emulate the STGI instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedStgi(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_stgi);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Implements 'INVLPGA'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_invlpga)
+{
+    /** @todo Check effective address size using address size prefix. */
+    RTGCPTR  const GCPtrPage = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? pVCpu->cpum.GstCtx.rax : pVCpu->cpum.GstCtx.eax;
+    /** @todo PGM needs virtual ASID support. */
+# if 0
+    uint32_t const uAsid     = pVCpu->cpum.GstCtx.ecx;
+# endif
+
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, invlpga);
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_INVLPGA))
+    {
+        Log2(("invlpga: Guest intercept (%RGp) -> #VMEXIT\n", GCPtrPage));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_INVLPGA, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    PGMInvalidatePage(pVCpu, GCPtrPage);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVLPGA instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvlpga(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_invlpga);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Implements 'SKINIT'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_skinit)
+{
+    IEM_SVM_INSTR_COMMON_CHECKS(pVCpu, invlpga);
+
+    uint32_t uIgnore;
+    uint32_t fFeaturesECX;
+    CPUMGetGuestCpuId(pVCpu, 0x80000001, 0 /* iSubLeaf */, -1 /*f64BitMode*/, &uIgnore, &uIgnore, &fFeaturesECX, &uIgnore);
+    if (!(fFeaturesECX & X86_CPUID_AMD_FEATURE_ECX_SKINIT))
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_SKINIT))
+    {
+        Log2(("skinit: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_SKINIT, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    RT_NOREF(cbInstr);
+    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * Implements SVM's implementation of PAUSE.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_svm_pause)
+{
+    bool fCheckIntercept = true;
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmPauseFilter)
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+
+        /* TSC based pause-filter thresholding. */
+        if (   IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvmPauseFilterThreshold
+            && pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilterThreshold > 0)
+        {
+            uint64_t const uTick = TMCpuTickGet(pVCpu);
+            if (uTick - pVCpu->cpum.GstCtx.hwvirt.svm.uPrevPauseTick > pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilterThreshold)
+                pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter = CPUMGetGuestSvmPauseFilterCount(pVCpu, IEM_GET_CTX(pVCpu));
+            pVCpu->cpum.GstCtx.hwvirt.svm.uPrevPauseTick = uTick;
+        }
+
+        /* Simple pause-filter counter. */
+        if (pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter > 0)
+        {
+            --pVCpu->cpum.GstCtx.hwvirt.svm.cPauseFilter;
+            fCheckIntercept = false;
+        }
+    }
+
+    if (fCheckIntercept)
+        IEM_SVM_CHECK_INSTR_INTERCEPT(pVCpu, SVM_CTRL_INTERCEPT_PAUSE, SVM_EXIT_PAUSE, 0, 0);
+
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
+
+/**
+ * Common code for iemCImpl_vmmcall and iemCImpl_vmcall (latter in IEMAllCImplVmxInstr.cpp.h).
+ */
+IEM_CIMPL_DEF_1(iemCImpl_Hypercall, uint16_t, uDisOpcode)
+{
+    if (EMAreHypercallInstructionsEnabled(pVCpu))
+    {
+        NOREF(uDisOpcode);
+        VBOXSTRICTRC rcStrict = GIMHypercallEx(pVCpu, IEM_GET_CTX(pVCpu), uDisOpcode, cbInstr);
+        if (RT_SUCCESS(rcStrict))
+        {
+            /** @todo finish: Sort out assertion here when iemRegAddToRipAndFinishingClearingRF
+             * starts returning non-VINF_SUCCESS statuses. */
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            if (   rcStrict == VINF_SUCCESS
+                || rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
+                return VINF_SUCCESS;
+            AssertMsgReturn(rcStrict == VINF_GIM_R3_HYPERCALL, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IEM_IPE_4);
+            return rcStrict;
+        }
+        AssertMsgReturn(   rcStrict == VERR_GIM_HYPERCALL_ACCESS_DENIED
+                        || rcStrict == VERR_GIM_HYPERCALLS_NOT_AVAILABLE
+                        || rcStrict == VERR_GIM_NOT_ENABLED
+                        || rcStrict == VERR_GIM_HYPERCALL_MEMORY_READ_FAILED
+                        || rcStrict == VERR_GIM_HYPERCALL_MEMORY_WRITE_FAILED,
+                        ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), VERR_IEM_IPE_4);
+
+        /* Raise #UD on all failures. */
+    }
+    return iemRaiseUndefinedOpcode(pVCpu);
+}
+
+
+/**
+ * Implements 'VMMCALL'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmmcall)
+{
+    if (IEM_SVM_IS_CTRL_INTERCEPT_SET(pVCpu, SVM_CTRL_INTERCEPT_VMMCALL))
+    {
+        Log(("vmmcall: Guest intercept -> #VMEXIT\n"));
+        IEM_SVM_VMEXIT_RET(pVCpu, SVM_EXIT_VMMCALL, 0 /* uExitInfo1 */, 0 /* uExitInfo2 */);
+    }
+
+    /* This is a little bit more complicated than the VT-x version because HM/SVM may
+       patch MOV CR8 instructions to speed up APIC.TPR access for 32-bit windows guests. */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (VM_IS_HM_ENABLED(pVM))
+    {
+        int rc = HMHCMaybeMovTprSvmHypercall(pVM, pVCpu);
+        if (RT_SUCCESS(rc))
+        {
+            Log(("vmmcall: MovTpr\n"));
+            return VINF_SUCCESS;
+        }
+    }
+
+    /* Join forces with vmcall. */
+    return IEM_CIMPL_CALL_1(iemCImpl_Hypercall, OP_VMMCALL);
+}
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp b/src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp
new file mode 100644
index 00000000..8ffcab9e
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllCImplVmxInstr.cpp
@@ -0,0 +1,10000 @@
+/* $Id: IEMAllCImplVmxInstr.cpp $ */
+/** @file
+ * IEM - VT-x instruction implementation.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP   LOG_GROUP_IEM_VMX
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pgm.h>
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+# include <VBox/vmm/hmvmxinline.h>
+#endif
+#include <VBox/vmm/tm.h>
+#include "IEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/disopcode.h>
+#include <iprt/asm-math.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/x86.h>
+
+#include "IEMInline.h"
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Gets the ModR/M, SIB and displacement byte(s) from decoded opcodes given their
+ * relative offsets.
+ */
+# ifdef IEM_WITH_CODE_TLB /** @todo IEM TLB */
+#  define IEM_MODRM_GET_U8(a_pVCpu, a_bModRm, a_offModRm)         do { a_bModRm  = 0; RT_NOREF(a_offModRm); } while (0)
+#  define IEM_SIB_GET_U8(a_pVCpu, a_bSib, a_offSib)               do { a_bSib    = 0; RT_NOREF(a_offSib);  } while (0)
+#  define IEM_DISP_GET_U16(a_pVCpu, a_u16Disp, a_offDisp)         do { a_u16Disp = 0; RT_NOREF(a_offDisp); } while (0)
+#  define IEM_DISP_GET_S8_SX_U16(a_pVCpu, a_u16Disp, a_offDisp)   do { a_u16Disp = 0; RT_NOREF(a_offDisp); } while (0)
+#  define IEM_DISP_GET_U32(a_pVCpu, a_u32Disp, a_offDisp)         do { a_u32Disp = 0; RT_NOREF(a_offDisp); } while (0)
+#  define IEM_DISP_GET_S8_SX_U32(a_pVCpu, a_u32Disp, a_offDisp)   do { a_u32Disp = 0; RT_NOREF(a_offDisp); } while (0)
+#  define IEM_DISP_GET_S32_SX_U64(a_pVCpu, a_u64Disp, a_offDisp)  do { a_u64Disp = 0; RT_NOREF(a_offDisp); } while (0)
+#  define IEM_DISP_GET_S8_SX_U64(a_pVCpu, a_u64Disp, a_offDisp)   do { a_u64Disp = 0; RT_NOREF(a_offDisp); } while (0)
+#  if 0
+#   error "Implement me: Getting ModR/M, SIB, displacement needs to work even when instruction crosses a page boundary."
+#  endif
+# else  /* !IEM_WITH_CODE_TLB */
+#  define IEM_MODRM_GET_U8(a_pVCpu, a_bModRm, a_offModRm) \
+    do \
+    { \
+        Assert((a_offModRm) < (a_pVCpu)->iem.s.cbOpcode); \
+        (a_bModRm) = (a_pVCpu)->iem.s.abOpcode[(a_offModRm)]; \
+    } while (0)
+
+#  define IEM_SIB_GET_U8(a_pVCpu, a_bSib, a_offSib)      IEM_MODRM_GET_U8(a_pVCpu, a_bSib, a_offSib)
+
+#  define IEM_DISP_GET_U16(a_pVCpu, a_u16Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) + 1 < (a_pVCpu)->iem.s.cbOpcode); \
+        uint8_t const bTmpLo = (a_pVCpu)->iem.s.abOpcode[(a_offDisp)]; \
+        uint8_t const bTmpHi = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 1]; \
+        (a_u16Disp) = RT_MAKE_U16(bTmpLo, bTmpHi); \
+    } while (0)
+
+#  define IEM_DISP_GET_S8_SX_U16(a_pVCpu, a_u16Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) < (a_pVCpu)->iem.s.cbOpcode); \
+        (a_u16Disp) = (int8_t)((a_pVCpu)->iem.s.abOpcode[(a_offDisp)]); \
+    } while (0)
+
+#  define IEM_DISP_GET_U32(a_pVCpu, a_u32Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) + 3 < (a_pVCpu)->iem.s.cbOpcode); \
+        uint8_t const bTmp0 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp)]; \
+        uint8_t const bTmp1 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 1]; \
+        uint8_t const bTmp2 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 2]; \
+        uint8_t const bTmp3 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 3]; \
+        (a_u32Disp) = RT_MAKE_U32_FROM_U8(bTmp0, bTmp1, bTmp2, bTmp3); \
+    } while (0)
+
+#  define IEM_DISP_GET_S8_SX_U32(a_pVCpu, a_u32Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) + 1 < (a_pVCpu)->iem.s.cbOpcode); \
+        (a_u32Disp) = (int8_t)((a_pVCpu)->iem.s.abOpcode[(a_offDisp)]); \
+    } while (0)
+
+#  define IEM_DISP_GET_S8_SX_U64(a_pVCpu, a_u64Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) + 1 < (a_pVCpu)->iem.s.cbOpcode); \
+        (a_u64Disp) = (int8_t)((a_pVCpu)->iem.s.abOpcode[(a_offDisp)]); \
+    } while (0)
+
+#  define IEM_DISP_GET_S32_SX_U64(a_pVCpu, a_u64Disp, a_offDisp) \
+    do \
+    { \
+        Assert((a_offDisp) + 3 < (a_pVCpu)->iem.s.cbOpcode); \
+        uint8_t const bTmp0 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp)]; \
+        uint8_t const bTmp1 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 1]; \
+        uint8_t const bTmp2 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 2]; \
+        uint8_t const bTmp3 = (a_pVCpu)->iem.s.abOpcode[(a_offDisp) + 3]; \
+        (a_u64Disp) = (int32_t)RT_MAKE_U32_FROM_U8(bTmp0, bTmp1, bTmp2, bTmp3); \
+    } while (0)
+# endif /* !IEM_WITH_CODE_TLB */
+
+/** Check for VMX instructions requiring to be in VMX operation.
+ * @note Any changes here, check if IEMOP_HLP_IN_VMX_OPERATION needs updating. */
+# define IEM_VMX_IN_VMX_OPERATION(a_pVCpu, a_szInstr, a_InsDiagPrefix) \
+    do \
+    { \
+        if (IEM_VMX_IS_ROOT_MODE(a_pVCpu)) \
+        { /* likely */ } \
+        else \
+        { \
+            Log((a_szInstr ": Not in VMX operation (root mode) -> #UD\n")); \
+            (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmDiag = a_InsDiagPrefix##_VmxRoot; \
+            return iemRaiseUndefinedOpcode(a_pVCpu); \
+        } \
+    } while (0)
+
+/** Marks a VM-entry failure with a diagnostic reason, logs and returns. */
+# define IEM_VMX_VMENTRY_FAILED_RET(a_pVCpu, a_pszInstr, a_pszFailure, a_VmxDiag) \
+    do \
+    { \
+        LogRel(("%s: VM-entry failed! enmDiag=%u (%s) -> %s\n", (a_pszInstr), (a_VmxDiag), \
+               HMGetVmxDiagDesc(a_VmxDiag), (a_pszFailure))); \
+        (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmDiag = (a_VmxDiag); \
+        return VERR_VMX_VMENTRY_FAILED; \
+    } while (0)
+
+/** Marks a VM-exit failure with a diagnostic reason and logs. */
+# define IEM_VMX_VMEXIT_FAILED(a_pVCpu, a_uExitReason, a_pszFailure, a_VmxDiag) \
+    do \
+    { \
+        LogRel(("VM-exit failed! uExitReason=%u enmDiag=%u (%s) -> %s\n", (a_uExitReason), (a_VmxDiag), \
+               HMGetVmxDiagDesc(a_VmxDiag), (a_pszFailure))); \
+        (a_pVCpu)->cpum.GstCtx.hwvirt.vmx.enmDiag  = (a_VmxDiag); \
+    } while (0)
+
+/** Marks a VM-exit failure with a diagnostic reason, logs and returns. */
+# define IEM_VMX_VMEXIT_FAILED_RET(a_pVCpu, a_uExitReason, a_pszFailure, a_VmxDiag) \
+    do \
+    { \
+        IEM_VMX_VMEXIT_FAILED(a_pVCpu, a_uExitReason, a_pszFailure, a_VmxDiag); \
+        return VERR_VMX_VMEXIT_FAILED; \
+    } while (0)
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** @todo NSTVMX: The following VM-exit intercepts are pending:
+ *  VMX_EXIT_IO_SMI
+ *  VMX_EXIT_SMI
+ *  VMX_EXIT_GETSEC
+ *  VMX_EXIT_RSM
+ *  VMX_EXIT_MONITOR (APIC access VM-exit caused by MONITOR pending)
+ *  VMX_EXIT_ERR_MACHINE_CHECK (we never need to raise this?)
+ *  VMX_EXIT_RDRAND
+ *  VMX_EXIT_VMFUNC
+ *  VMX_EXIT_ENCLS
+ *  VMX_EXIT_RDSEED
+ *  VMX_EXIT_PML_FULL
+ *  VMX_EXIT_XSAVES
+ *  VMX_EXIT_XRSTORS
+ */
+/**
+ * Map of VMCS field encodings to their virtual-VMCS structure offsets.
+ *
+ * The first array dimension is VMCS field encoding of Width OR'ed with Type and the
+ * second dimension is the Index, see VMXVMCSFIELD.
+ */
+uint16_t const g_aoffVmcsMap[16][VMX_V_VMCS_MAX_INDEX + 1] =
+{
+    /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_CONTROL: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u16Vpid),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u16PostIntNotifyVector),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u16EptpIndex),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u16HlatPrefixSize),
+        /*  4-11 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 12-19 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 20-27 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 28-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */
+    {
+        /*   0-7 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /*  8-15 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 16-23 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 24-31 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 32-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_GUEST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, GuestEs),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, GuestCs),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, GuestSs),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, GuestDs),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, GuestFs),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, GuestGs),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, GuestLdtr),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, GuestTr),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u16GuestIntStatus),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u16PmlIndex),
+        /* 10-17 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 18-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 26-33 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /*    34 */ UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_16BIT | VMX_VMCSFIELD_TYPE_HOST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, HostEs),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, HostCs),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, HostSs),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, HostDs),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, HostFs),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, HostGs),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, HostTr),
+        /*  7-14 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 15-22 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 23-30 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 31-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_CONTROL: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64AddrIoBitmapA),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64AddrIoBitmapB),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64AddrMsrBitmap),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64AddrExitMsrStore),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64AddrExitMsrLoad),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64AddrEntryMsrLoad),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u64ExecVmcsPtr),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u64AddrPml),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u64TscOffset),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u64AddrVirtApic),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u64AddrApicAccess),
+        /*    11 */ RT_UOFFSETOF(VMXVVMCS, u64AddrPostedIntDesc),
+        /*    12 */ RT_UOFFSETOF(VMXVVMCS, u64VmFuncCtls),
+        /*    13 */ RT_UOFFSETOF(VMXVVMCS, u64EptPtr),
+        /*    14 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap0),
+        /*    15 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap1),
+        /*    16 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap2),
+        /*    17 */ RT_UOFFSETOF(VMXVVMCS, u64EoiExitBitmap3),
+        /*    18 */ RT_UOFFSETOF(VMXVVMCS, u64AddrEptpList),
+        /*    19 */ RT_UOFFSETOF(VMXVVMCS, u64AddrVmreadBitmap),
+        /*    20 */ RT_UOFFSETOF(VMXVVMCS, u64AddrVmwriteBitmap),
+        /*    21 */ RT_UOFFSETOF(VMXVVMCS, u64AddrXcptVeInfo),
+        /*    22 */ RT_UOFFSETOF(VMXVVMCS, u64XssExitBitmap),
+        /*    23 */ RT_UOFFSETOF(VMXVVMCS, u64EnclsExitBitmap),
+        /*    24 */ RT_UOFFSETOF(VMXVVMCS, u64SppTablePtr),
+        /*    25 */ RT_UOFFSETOF(VMXVVMCS, u64TscMultiplier),
+        /*    26 */ RT_UOFFSETOF(VMXVVMCS, u64ProcCtls3),
+        /*    27 */ RT_UOFFSETOF(VMXVVMCS, u64EnclvExitBitmap),
+        /*    28 */ UINT16_MAX,
+        /*    29 */ UINT16_MAX,
+        /*    30 */ UINT16_MAX,
+        /*    31 */ RT_UOFFSETOF(VMXVVMCS, u64PconfigExitBitmap),
+        /*    32 */ RT_UOFFSETOF(VMXVVMCS, u64HlatPtr),
+        /*    33 */ UINT16_MAX,
+        /*    34 */ RT_UOFFSETOF(VMXVVMCS, u64ExitCtls2)
+    },
+    /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64RoGuestPhysAddr),
+        /*   1-8 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /*  9-16 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 17-24 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 25-32 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 33-34*/  UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_GUEST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64VmcsLinkPtr),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64GuestDebugCtlMsr),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPatMsr),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64GuestEferMsr),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPerfGlobalCtlMsr),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte0),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte1),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte2),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPdpte3),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u64GuestBndcfgsMsr),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRtitCtlMsr),
+        /*    11 */ UINT16_MAX,
+        /*    12 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPkrsMsr),
+        /* 13-20 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 21-28 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 29-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_64BIT | VMX_VMCSFIELD_TYPE_HOST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64HostPatMsr),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64HostEferMsr),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64HostPerfGlobalCtlMsr),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64HostPkrsMsr),
+        /*  4-11 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 12-19 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 20-27 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 28-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_CONTROL: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u32PinCtls),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u32ProcCtls),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u32XcptBitmap),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u32XcptPFMask),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u32XcptPFMatch),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u32Cr3TargetCount),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u32ExitCtls),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u32ExitMsrStoreCount),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u32ExitMsrLoadCount),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u32EntryCtls),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u32EntryMsrLoadCount),
+        /*    11 */ RT_UOFFSETOF(VMXVVMCS, u32EntryIntInfo),
+        /*    12 */ RT_UOFFSETOF(VMXVVMCS, u32EntryXcptErrCode),
+        /*    13 */ RT_UOFFSETOF(VMXVVMCS, u32EntryInstrLen),
+        /*    14 */ RT_UOFFSETOF(VMXVVMCS, u32TprThreshold),
+        /*    15 */ RT_UOFFSETOF(VMXVVMCS, u32ProcCtls2),
+        /*    16 */ RT_UOFFSETOF(VMXVVMCS, u32PleGap),
+        /*    17 */ RT_UOFFSETOF(VMXVVMCS, u32PleWindow),
+        /* 18-25 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 26-33 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /*    34 */ UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u32RoVmInstrError),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitReason),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitIntInfo),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitIntErrCode),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u32RoIdtVectoringInfo),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u32RoIdtVectoringErrCode),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitInstrLen),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u32RoExitInstrInfo),
+        /*  8-15 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 16-23 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 24-31 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 32-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_GUEST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u32GuestEsLimit),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u32GuestCsLimit),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSsLimit),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u32GuestDsLimit),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u32GuestFsLimit),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u32GuestGsLimit),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u32GuestLdtrLimit),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u32GuestTrLimit),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u32GuestGdtrLimit),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u32GuestIdtrLimit),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u32GuestEsAttr),
+        /*    11 */ RT_UOFFSETOF(VMXVVMCS, u32GuestCsAttr),
+        /*    12 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSsAttr),
+        /*    13 */ RT_UOFFSETOF(VMXVVMCS, u32GuestDsAttr),
+        /*    14 */ RT_UOFFSETOF(VMXVVMCS, u32GuestFsAttr),
+        /*    15 */ RT_UOFFSETOF(VMXVVMCS, u32GuestGsAttr),
+        /*    16 */ RT_UOFFSETOF(VMXVVMCS, u32GuestLdtrAttr),
+        /*    17 */ RT_UOFFSETOF(VMXVVMCS, u32GuestTrAttr),
+        /*    18 */ RT_UOFFSETOF(VMXVVMCS, u32GuestIntrState),
+        /*    19 */ RT_UOFFSETOF(VMXVVMCS, u32GuestActivityState),
+        /*    20 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSmBase),
+        /*    21 */ RT_UOFFSETOF(VMXVVMCS, u32GuestSysenterCS),
+        /*    22 */ UINT16_MAX,
+        /*    23 */ RT_UOFFSETOF(VMXVVMCS, u32PreemptTimer),
+        /* 24-31 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 32-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_32BIT | VMX_VMCSFIELD_TYPE_HOST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u32HostSysenterCs),
+        /*   1-8 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /*  9-16 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 17-24 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 25-32 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 33-34 */ UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_CONTROL: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64Cr0Mask),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64Cr4Mask),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64Cr0ReadShadow),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64Cr4ReadShadow),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target0),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target1),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target2),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u64Cr3Target3),
+        /*  8-15 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 16-23 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 24-27 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 32-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_VMEXIT_INFO: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64RoExitQual),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRcx),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRsi),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRdi),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64RoIoRip),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64RoGuestLinearAddr),
+        /*  6-13 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 14-21 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 22-29 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 30-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_GUEST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCr0),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCr3),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCr4),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64GuestEsBase),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64GuestCsBase),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSsBase),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u64GuestDsBase),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u64GuestFsBase),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u64GuestGsBase),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u64GuestLdtrBase),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u64GuestTrBase),
+        /*    11 */ RT_UOFFSETOF(VMXVVMCS, u64GuestGdtrBase),
+        /*    12 */ RT_UOFFSETOF(VMXVVMCS, u64GuestIdtrBase),
+        /*    13 */ RT_UOFFSETOF(VMXVVMCS, u64GuestDr7),
+        /*    14 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRsp),
+        /*    15 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRip),
+        /*    16 */ RT_UOFFSETOF(VMXVVMCS, u64GuestRFlags),
+        /*    17 */ RT_UOFFSETOF(VMXVVMCS, u64GuestPendingDbgXcpts),
+        /*    18 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSysenterEsp),
+        /*    19 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSysenterEip),
+        /*    20 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSCetMsr),
+        /*    21 */ RT_UOFFSETOF(VMXVVMCS, u64GuestSsp),
+        /*    22 */ RT_UOFFSETOF(VMXVVMCS, u64GuestIntrSspTableAddrMsr),
+        /* 23-27 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 31-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    },
+    /* VMX_VMCSFIELD_WIDTH_NATURAL | VMX_VMCSFIELD_TYPE_HOST_STATE: */
+    {
+        /*     0 */ RT_UOFFSETOF(VMXVVMCS, u64HostCr0),
+        /*     1 */ RT_UOFFSETOF(VMXVVMCS, u64HostCr3),
+        /*     2 */ RT_UOFFSETOF(VMXVVMCS, u64HostCr4),
+        /*     3 */ RT_UOFFSETOF(VMXVVMCS, u64HostFsBase),
+        /*     4 */ RT_UOFFSETOF(VMXVVMCS, u64HostGsBase),
+        /*     5 */ RT_UOFFSETOF(VMXVVMCS, u64HostTrBase),
+        /*     6 */ RT_UOFFSETOF(VMXVVMCS, u64HostGdtrBase),
+        /*     7 */ RT_UOFFSETOF(VMXVVMCS, u64HostIdtrBase),
+        /*     8 */ RT_UOFFSETOF(VMXVVMCS, u64HostSysenterEsp),
+        /*     9 */ RT_UOFFSETOF(VMXVVMCS, u64HostSysenterEip),
+        /*    10 */ RT_UOFFSETOF(VMXVVMCS, u64HostRsp),
+        /*    11 */ RT_UOFFSETOF(VMXVVMCS, u64HostRip),
+        /*    12 */ RT_UOFFSETOF(VMXVVMCS, u64HostSCetMsr),
+        /*    13 */ RT_UOFFSETOF(VMXVVMCS, u64HostSsp),
+        /*    14 */ RT_UOFFSETOF(VMXVVMCS, u64HostIntrSspTableAddrMsr),
+        /* 15-22 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 23-30 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,
+        /* 31-34 */ UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX
+    }
+};
+
+
+/**
+ * Gets a host selector from the VMCS.
+ *
+ * @param   pVmcs       Pointer to the virtual VMCS.
+ * @param   iSelReg     The index of the segment register (X86_SREG_XXX).
+ */
+DECLINLINE(RTSEL) iemVmxVmcsGetHostSelReg(PCVMXVVMCS pVmcs, uint8_t iSegReg)
+{
+    Assert(iSegReg < X86_SREG_COUNT);
+    RTSEL HostSel;
+    uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_16BIT;
+    uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_HOST_STATE;
+    uint8_t  const  uWidthType = (uWidth << 2) | uType;
+    uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS16_HOST_ES_SEL, VMX_BF_VMCSFIELD_INDEX);
+    Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+    uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+    uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+    uint8_t  const *pbField    = pbVmcs + offField;
+    HostSel = *(uint16_t *)pbField;
+    return HostSel;
+}
+
+
+/**
+ * Sets a guest segment register in the VMCS.
+ *
+ * @param   pVmcs       Pointer to the virtual VMCS.
+ * @param   iSegReg     The index of the segment register (X86_SREG_XXX).
+ * @param   pSelReg     Pointer to the segment register.
+ */
+static void iemVmxVmcsSetGuestSegReg(PCVMXVVMCS pVmcs, uint8_t iSegReg, PCCPUMSELREG pSelReg) RT_NOEXCEPT
+{
+    Assert(pSelReg);
+    Assert(iSegReg < X86_SREG_COUNT);
+
+    /* Selector. */
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_16BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS16_GUEST_ES_SEL, VMX_BF_VMCSFIELD_INDEX);
+        Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t        *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t        *pbField    = pbVmcs + offField;
+        *(uint16_t *)pbField = pSelReg->Sel;
+    }
+
+    /* Limit. */
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_32BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_LIMIT, VMX_BF_VMCSFIELD_INDEX);
+        Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t        *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t        *pbField    = pbVmcs + offField;
+        *(uint32_t *)pbField = pSelReg->u32Limit;
+    }
+
+    /* Base. */
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_NATURAL;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS_GUEST_ES_BASE, VMX_BF_VMCSFIELD_INDEX);
+        Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t  const *pbField    = pbVmcs + offField;
+        *(uint64_t *)pbField = pSelReg->u64Base;
+    }
+
+    /* Attributes. */
+    {
+        uint32_t const fValidAttrMask = X86DESCATTR_TYPE | X86DESCATTR_DT  | X86DESCATTR_DPL | X86DESCATTR_P
+                                      | X86DESCATTR_AVL  | X86DESCATTR_L   | X86DESCATTR_D   | X86DESCATTR_G
+                                      | X86DESCATTR_UNUSABLE;
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_32BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, VMX_BF_VMCSFIELD_INDEX);
+        Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t        *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t        *pbField    = pbVmcs + offField;
+        *(uint32_t *)pbField       = pSelReg->Attr.u & fValidAttrMask;
+    }
+}
+
+
+/**
+ * Gets a guest segment register from the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVmcs       Pointer to the virtual VMCS.
+ * @param   iSegReg     The index of the segment register (X86_SREG_XXX).
+ * @param   pSelReg     Where to store the segment register (only updated when
+ *                      VINF_SUCCESS is returned).
+ *
+ * @remarks Warning! This does not validate the contents of the retrieved segment
+ *          register.
+ */
+static int iemVmxVmcsGetGuestSegReg(PCVMXVVMCS pVmcs, uint8_t iSegReg, PCPUMSELREG pSelReg) RT_NOEXCEPT
+{
+    Assert(pSelReg);
+    Assert(iSegReg < X86_SREG_COUNT);
+
+    /* Selector. */
+    uint16_t u16Sel;
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_16BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS16_GUEST_ES_SEL, VMX_BF_VMCSFIELD_INDEX);
+        AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t  const *pbField    = pbVmcs + offField;
+        u16Sel = *(uint16_t *)pbField;
+    }
+
+    /* Limit. */
+    uint32_t u32Limit;
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_32BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_LIMIT, VMX_BF_VMCSFIELD_INDEX);
+        AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t  const *pbField    = pbVmcs + offField;
+        u32Limit = *(uint32_t *)pbField;
+    }
+
+    /* Base. */
+    uint64_t u64Base;
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_NATURAL;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS_GUEST_ES_BASE, VMX_BF_VMCSFIELD_INDEX);
+        AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t  const *pbField    = pbVmcs + offField;
+        u64Base = *(uint64_t *)pbField;
+        /** @todo NSTVMX: Should we zero out high bits here for 32-bit virtual CPUs? */
+    }
+
+    /* Attributes. */
+    uint32_t u32Attr;
+    {
+        uint8_t  const  uWidth     = VMX_VMCSFIELD_WIDTH_32BIT;
+        uint8_t  const  uType      = VMX_VMCSFIELD_TYPE_GUEST_STATE;
+        uint8_t  const  uWidthType = (uWidth << 2) | uType;
+        uint8_t  const  uIndex     = iSegReg + RT_BF_GET(VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, VMX_BF_VMCSFIELD_INDEX);
+        AssertReturn(uIndex <= VMX_V_VMCS_MAX_INDEX, VERR_IEM_IPE_3);
+        uint16_t const  offField   = g_aoffVmcsMap[uWidthType][uIndex];
+        uint8_t  const *pbVmcs     = (uint8_t *)pVmcs;
+        uint8_t  const *pbField    = pbVmcs + offField;
+        u32Attr = *(uint32_t *)pbField;
+    }
+
+    pSelReg->Sel      = u16Sel;
+    pSelReg->ValidSel = u16Sel;
+    pSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+    pSelReg->u32Limit = u32Limit;
+    pSelReg->u64Base  = u64Base;
+    pSelReg->Attr.u   = u32Attr;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Converts an IEM exception event type to a VMX event type.
+ *
+ * @returns The VMX event type.
+ * @param   uVector     The interrupt / exception vector.
+ * @param   fFlags      The IEM event flag (see IEM_XCPT_FLAGS_XXX).
+ */
+DECLINLINE(uint8_t) iemVmxGetEventType(uint32_t uVector, uint32_t fFlags)
+{
+    /* Paranoia (callers may use these interchangeably). */
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_NMI          == VMX_IDT_VECTORING_INFO_TYPE_NMI);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_HW_XCPT      == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_EXT_INT      == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_XCPT      == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_INT       == VMX_IDT_VECTORING_INFO_TYPE_SW_INT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_NMI          == VMX_ENTRY_INT_INFO_TYPE_NMI);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_HW_XCPT      == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_EXT_INT      == VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_XCPT      == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_SW_INT       == VMX_ENTRY_INT_INFO_TYPE_SW_INT);
+    AssertCompile(VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT);
+
+    if (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+    {
+        if (uVector == X86_XCPT_NMI)
+            return VMX_EXIT_INT_INFO_TYPE_NMI;
+        return VMX_EXIT_INT_INFO_TYPE_HW_XCPT;
+    }
+
+    if (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+    {
+        if (fFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_OF_INSTR))
+            return VMX_EXIT_INT_INFO_TYPE_SW_XCPT;
+        if (fFlags & IEM_XCPT_FLAGS_ICEBP_INSTR)
+            return VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT;
+        return VMX_EXIT_INT_INFO_TYPE_SW_INT;
+    }
+
+    Assert(fFlags & IEM_XCPT_FLAGS_T_EXT_INT);
+    return VMX_EXIT_INT_INFO_TYPE_EXT_INT;
+}
+
+
+/**
+ * Determines whether the guest is using PAE paging given the VMCS.
+ *
+ * @returns @c true if PAE paging mode is used, @c false otherwise.
+ * @param   pVmcs       Pointer to the virtual VMCS.
+ *
+ * @warning Only use this prior to switching the guest-CPU state with the
+ *          nested-guest CPU state!
+ */
+DECL_FORCE_INLINE(bool) iemVmxVmcsIsGuestPaePagingEnabled(PCVMXVVMCS pVmcs)
+{
+    return (   !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST)
+            &&  (pVmcs->u64GuestCr4.u & X86_CR4_PAE)
+            &&  (pVmcs->u64GuestCr0.u & X86_CR0_PG));
+}
+
+
+/**
+ * Sets the Exit qualification VMCS field.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u64ExitQual     The Exit qualification.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitQual(PVMCPUCC pVCpu, uint64_t u64ExitQual)
+{
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64RoExitQual.u = u64ExitQual;
+}
+
+
+/**
+ * Sets the VM-exit interruption information field.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitIntInfo    The VM-exit interruption information.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitIntInfo(PVMCPUCC pVCpu, uint32_t uExitIntInfo)
+{
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoExitIntInfo = uExitIntInfo;
+}
+
+
+/**
+ * Sets the VM-exit interruption error code.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uErrCode    The error code.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitIntErrCode(PVMCPUCC pVCpu, uint32_t uErrCode)
+{
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoExitIntErrCode = uErrCode;
+}
+
+
+/**
+ * Sets the IDT-vectoring information field.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uIdtVectorInfo  The IDT-vectoring information.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetIdtVectoringInfo(PVMCPUCC pVCpu, uint32_t uIdtVectorInfo)
+{
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoIdtVectoringInfo = uIdtVectorInfo;
+}
+
+
+/**
+ * Sets the IDT-vectoring error code field.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uErrCode    The error code.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetIdtVectoringErrCode(PVMCPUCC pVCpu, uint32_t uErrCode)
+{
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoIdtVectoringErrCode = uErrCode;
+}
+
+
+/**
+ * Sets the VM-exit guest-linear address VMCS field.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uGuestLinearAddr    The VM-exit guest-linear address.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitGuestLinearAddr(PVMCPUCC pVCpu, uint64_t uGuestLinearAddr)
+{
+    /* Bits 63:32 of guest-linear address MBZ if the guest isn't in long mode prior to the VM-exit. */
+    Assert(CPUMIsGuestInLongModeEx(IEM_GET_CTX(pVCpu)) || !(uGuestLinearAddr & UINT64_C(0xffffffff00000000)));
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64RoGuestLinearAddr.u = uGuestLinearAddr;
+}
+
+
+/**
+ * Sets the VM-exit guest-physical address VMCS field.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uGuestPhysAddr  The VM-exit guest-physical address.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitGuestPhysAddr(PVMCPUCC pVCpu, uint64_t uGuestPhysAddr)
+{
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64RoGuestPhysAddr.u = uGuestPhysAddr;
+}
+
+
+/**
+ * Sets the VM-exit instruction length VMCS field.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The VM-exit instruction length in bytes.
+ *
+ * @remarks Callers may clear this field to 0. Hence, this function does not check
+ *          the validity of the instruction length.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitInstrLen(PVMCPUCC pVCpu, uint32_t cbInstr)
+{
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoExitInstrLen = cbInstr;
+}
+
+
+/**
+ * Sets the VM-exit instruction info. VMCS field.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitInstrInfo  The VM-exit instruction information.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetExitInstrInfo(PVMCPUCC pVCpu, uint32_t uExitInstrInfo)
+{
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoExitInstrInfo = uExitInstrInfo;
+}
+
+
+/**
+ * Sets the guest pending-debug exceptions field.
+ *
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   uGuestPendingDbgXcpts   The guest pending-debug exceptions.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmcsSetGuestPendingDbgXcpts(PVMCPUCC pVCpu, uint64_t uGuestPendingDbgXcpts)
+{
+    Assert(!(uGuestPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_VALID_MASK));
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64GuestPendingDbgXcpts.u = uGuestPendingDbgXcpts;
+}
+
+
+/**
+ * Implements VMSucceed for VMX instruction success.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmSucceed(PVMCPUCC pVCpu)
+{
+    return CPUMSetGuestVmxVmSucceed(&pVCpu->cpum.GstCtx);
+}
+
+
+/**
+ * Implements VMFailInvalid for VMX instruction failure.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmFailInvalid(PVMCPUCC pVCpu)
+{
+    return CPUMSetGuestVmxVmFailInvalid(&pVCpu->cpum.GstCtx);
+}
+
+
+/**
+ * Implements VMFail for VMX instruction failure.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   enmInsErr   The VM instruction error.
+ */
+DECL_FORCE_INLINE(void) iemVmxVmFail(PVMCPUCC pVCpu, VMXINSTRERR enmInsErr)
+{
+    return CPUMSetGuestVmxVmFail(&pVCpu->cpum.GstCtx, enmInsErr);
+}
+
+
+/**
+ * Checks if the given auto-load/store MSR area count is valid for the
+ * implementation.
+ *
+ * @returns @c true if it's within the valid limit, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uMsrCount   The MSR area count to check.
+ */
+DECL_FORCE_INLINE(bool) iemVmxIsAutoMsrCountValid(PCVMCPU pVCpu, uint32_t uMsrCount)
+{
+    uint64_t const u64VmxMiscMsr     = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Misc;
+    uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(u64VmxMiscMsr);
+    Assert(cMaxSupportedMsrs <= VMX_V_AUTOMSR_AREA_SIZE / sizeof(VMXAUTOMSR));
+    if (uMsrCount <= cMaxSupportedMsrs)
+        return true;
+    return false;
+}
+
+
+/**
+ * Flushes the current VMCS contents back to guest memory.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(int) iemVmxWriteCurrentVmcsToGstMem(PVMCPUCC pVCpu)
+{
+    Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
+    int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), IEM_VMX_GET_CURRENT_VMCS(pVCpu),
+                                      &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs, sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs));
+    return rc;
+}
+
+
+/**
+ * Populates the current VMCS contents from guest memory.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECL_FORCE_INLINE(int) iemVmxReadCurrentVmcsFromGstMem(PVMCPUCC pVCpu)
+{
+    Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs,
+                                     IEM_VMX_GET_CURRENT_VMCS(pVCpu), sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs));
+    return rc;
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment base checks during VM-entry of a
+ * nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+static VMXVDIAG iemVmxGetDiagVmentrySegBase(unsigned iSegReg) RT_NOEXCEPT
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegBaseCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegBaseDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegBaseEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegBaseFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegBaseGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegBaseSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_1);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment base checks during VM-entry of a
+ * nested-guest that is in Virtual-8086 mode.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+static VMXVDIAG iemVmxGetDiagVmentrySegBaseV86(unsigned iSegReg) RT_NOEXCEPT
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegBaseV86Cs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegBaseV86Ds;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegBaseV86Es;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegBaseV86Fs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegBaseV86Gs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegBaseV86Ss;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_2);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment limit checks during VM-entry of a
+ * nested-guest that is in Virtual-8086 mode.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+static VMXVDIAG iemVmxGetDiagVmentrySegLimitV86(unsigned iSegReg) RT_NOEXCEPT
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegLimitV86Cs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegLimitV86Ds;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegLimitV86Es;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegLimitV86Fs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegLimitV86Gs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegLimitV86Ss;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_3);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attribute checks during VM-entry of a
+ * nested-guest that is in Virtual-8086 mode.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+static VMXVDIAG iemVmxGetDiagVmentrySegAttrV86(unsigned iSegReg) RT_NOEXCEPT
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrV86Cs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrV86Ds;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrV86Es;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrV86Fs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrV86Gs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrV86Ss;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_4);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attributes reserved bits failure
+ * during VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+static VMXVDIAG iemVmxGetDiagVmentrySegAttrRsvd(unsigned iSegReg) RT_NOEXCEPT
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrRsvdEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrRsvdSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_5);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attributes descriptor-type
+ * (code/segment or system) failure during VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+static VMXVDIAG iemVmxGetDiagVmentrySegAttrDescType(unsigned iSegReg) RT_NOEXCEPT
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrDescTypeSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_6);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attributes descriptor-type
+ * (code/segment or system) failure during VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+static VMXVDIAG iemVmxGetDiagVmentrySegAttrPresent(unsigned iSegReg) RT_NOEXCEPT
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrPresentCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrPresentDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrPresentEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrPresentFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrPresentGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrPresentSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_7);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attribute granularity failure during
+ * VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+static VMXVDIAG iemVmxGetDiagVmentrySegAttrGran(unsigned iSegReg) RT_NOEXCEPT
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrGranCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrGranDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrGranEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrGranFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrGranGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrGranSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_8);
+    }
+}
+
+/**
+ * Gets the instruction diagnostic for segment attribute DPL/RPL failure during
+ * VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+static VMXVDIAG iemVmxGetDiagVmentrySegAttrDplRpl(unsigned iSegReg) RT_NOEXCEPT
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrDplRplEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrDplRplSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_9);
+    }
+}
+
+
+/**
+ * Gets the instruction diagnostic for segment attribute type accessed failure
+ * during VM-entry of a nested-guest.
+ *
+ * @param   iSegReg     The segment index (X86_SREG_XXX).
+ */
+static VMXVDIAG iemVmxGetDiagVmentrySegAttrTypeAcc(unsigned iSegReg) RT_NOEXCEPT
+{
+    switch (iSegReg)
+    {
+        case X86_SREG_CS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccCs;
+        case X86_SREG_DS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccDs;
+        case X86_SREG_ES: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccEs;
+        case X86_SREG_FS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccFs;
+        case X86_SREG_GS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccGs;
+        case X86_SREG_SS: return kVmxVDiag_Vmentry_GuestSegAttrTypeAccSs;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET2(kVmxVDiag_Ipe_10);
+    }
+}
+
+
+/**
+ * Saves the guest control registers, debug registers and some MSRs are part of
+ * VM-exit.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxVmexitSaveGuestControlRegsMsrs(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /*
+     * Saves the guest control registers, debug registers and some MSRs.
+     * See Intel spec. 27.3.1 "Saving Control Registers, Debug Registers and MSRs".
+     */
+    PVMXVVMCS pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    /* Save control registers. */
+    pVmcs->u64GuestCr0.u = pVCpu->cpum.GstCtx.cr0;
+    pVmcs->u64GuestCr3.u = pVCpu->cpum.GstCtx.cr3;
+    pVmcs->u64GuestCr4.u = pVCpu->cpum.GstCtx.cr4;
+
+    /* Save SYSENTER CS, ESP, EIP. */
+    pVmcs->u32GuestSysenterCS    = pVCpu->cpum.GstCtx.SysEnter.cs;
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        pVmcs->u64GuestSysenterEsp.u = pVCpu->cpum.GstCtx.SysEnter.esp;
+        pVmcs->u64GuestSysenterEip.u = pVCpu->cpum.GstCtx.SysEnter.eip;
+    }
+    else
+    {
+        pVmcs->u64GuestSysenterEsp.s.Lo = pVCpu->cpum.GstCtx.SysEnter.esp;
+        pVmcs->u64GuestSysenterEip.s.Lo = pVCpu->cpum.GstCtx.SysEnter.eip;
+    }
+
+    /* Save debug registers (DR7 and IA32_DEBUGCTL MSR). */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_DEBUG)
+    {
+        pVmcs->u64GuestDr7.u = pVCpu->cpum.GstCtx.dr[7];
+        /** @todo NSTVMX: Support IA32_DEBUGCTL MSR */
+    }
+
+    /* Save PAT MSR. */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_PAT_MSR)
+        pVmcs->u64GuestPatMsr.u = pVCpu->cpum.GstCtx.msrPAT;
+
+    /* Save EFER MSR. */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_EFER_MSR)
+        pVmcs->u64GuestEferMsr.u = pVCpu->cpum.GstCtx.msrEFER;
+
+    /* We don't support clearing IA32_BNDCFGS MSR yet. */
+    Assert(!(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_CLEAR_BNDCFGS_MSR));
+
+    /* Nothing to do for SMBASE register - We don't support SMM yet. */
+}
+
+
+/**
+ * Saves the guest force-flags in preparation of entering the nested-guest.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxVmentrySaveNmiBlockingFF(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /* We shouldn't be called multiple times during VM-entry. */
+    Assert(pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit == 0);
+
+    /* MTF should not be set outside VMX non-root mode. */
+    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
+
+    /*
+     * Preserve the required force-flags.
+     *
+     * We cache and clear force-flags that would affect the execution of the
+     * nested-guest. Cached flags are then restored while returning to the guest
+     * if necessary.
+     *
+     *   - VMCPU_FF_INHIBIT_INTERRUPTS need not be cached as it only affects
+     *     interrupts until the completion of the current VMLAUNCH/VMRESUME
+     *     instruction. Interrupt inhibition for any nested-guest instruction
+     *     is supplied by the guest-interruptibility state VMCS field and will
+     *     be set up as part of loading the guest state. Technically
+     *     blocking-by-STI is possible with VMLAUNCH/VMRESUME but we currently
+     *     disallow it since we can't distinguish it from blocking-by-MovSS
+     *     and no nested-hypervisor we care about uses STI immediately
+     *     followed by VMLAUNCH/VMRESUME.
+     *
+     *   - VMCPU_FF_BLOCK_NMIS needs to be cached as VM-exits caused before
+     *     successful VM-entry (due to invalid guest-state) need to continue
+     *     blocking NMIs if it was in effect before VM-entry.
+     *
+     *   - MTF need not be preserved as it's used only in VMX non-root mode and
+     *     is supplied through the VM-execution controls.
+     *
+     * The remaining FFs (e.g. timers, APIC updates) can stay in place so that
+     * we will be able to generate interrupts that may cause VM-exits for
+     * the nested-guest.
+     */
+    pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit = pVCpu->cpum.GstCtx.eflags.uBoth & CPUMCTX_INHIBIT_NMI;
+}
+
+
+/**
+ * Restores the guest force-flags in preparation of exiting the nested-guest.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxVmexitRestoreNmiBlockingFF(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /** @todo r=bird: why aren't we clearing the nested guest flags first here?
+     *        If there is some other code doing that already, it would be great
+     *        to point to it here... */
+    pVCpu->cpum.GstCtx.eflags.uBoth        |= pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit;
+    pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit = 0;
+}
+
+
+/**
+ * Performs the VMX transition to/from VMX non-root mode.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+*/
+static int iemVmxTransition(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /*
+     * Inform PGM about paging mode changes.
+     * We include X86_CR0_PE because PGM doesn't handle paged-real mode yet,
+     * see comment in iemMemPageTranslateAndCheckAccess().
+     */
+    int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0 | X86_CR0_PE, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
+                           true /* fForce */);
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+        return rc;
+
+    /* Invalidate IEM TLBs now that we've forced a PGM mode change. */
+    IEMTlbInvalidateAll(pVCpu);
+
+    /* Inform CPUM (recompiler), can later be removed. */
+    CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_ALL);
+
+    /* Re-initialize IEM cache/state after the drastic mode switch. */
+    iemReInitExec(pVCpu);
+    return rc;
+}
+
+
+/**
+ * Calculates the current VMX-preemption timer value.
+ *
+ * @returns The current VMX-preemption timer value.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static uint32_t iemVmxCalcPreemptTimer(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /*
+     * Assume the following:
+     * PreemptTimerShift = 5
+     * VmcsPreemptTimer  = 2 (i.e. need to decrement by 1 every 2 * RT_BIT(5) = 20000 TSC ticks)
+     * EntryTick         = 50000 (TSC at time of VM-entry)
+     *
+     * CurTick   Delta    PreemptTimerVal
+     * ----------------------------------
+     *  60000    10000    2
+     *  80000    30000    1
+     *  90000    40000    0  -> VM-exit.
+     *
+     * If Delta >= VmcsPreemptTimer * RT_BIT(PreemptTimerShift) cause a VMX-preemption timer VM-exit.
+     * The saved VMX-preemption timer value is calculated as follows:
+     * PreemptTimerVal = VmcsPreemptTimer - (Delta / (VmcsPreemptTimer * RT_BIT(PreemptTimerShift)))
+     * E.g.:
+     *  Delta  = 10000
+     *    Tmp    = 10000 / (2 * 10000) = 0.5
+     *    NewPt  = 2 - 0.5 = 2
+     *  Delta  = 30000
+     *    Tmp    = 30000 / (2 * 10000) = 1.5
+     *    NewPt  = 2 - 1.5 = 1
+     *  Delta  = 40000
+     *    Tmp    = 40000 / 20000 = 2
+     *    NewPt  = 2 - 2 = 0
+     */
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+    uint32_t const uVmcsPreemptVal = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32PreemptTimer;
+    if (uVmcsPreemptVal > 0)
+    {
+        uint64_t const uCurTick      = TMCpuTickGetNoCheck(pVCpu);
+        uint64_t const uEntryTick    = pVCpu->cpum.GstCtx.hwvirt.vmx.uEntryTick;
+        uint64_t const uDelta        = uCurTick - uEntryTick;
+        uint32_t const uPreemptTimer = uVmcsPreemptVal
+                                     - ASMDivU64ByU32RetU32(uDelta, uVmcsPreemptVal * RT_BIT(VMX_V_PREEMPT_TIMER_SHIFT));
+        return uPreemptTimer;
+    }
+    return 0;
+}
+
+
+/**
+ * Saves guest segment registers, GDTR, IDTR, LDTR, TR as part of VM-exit.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxVmexitSaveGuestSegRegs(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /*
+     * Save guest segment registers, GDTR, IDTR, LDTR, TR.
+     * See Intel spec 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
+     */
+    /* CS, SS, ES, DS, FS, GS. */
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++)
+    {
+        PCCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
+        if (!pSelReg->Attr.n.u1Unusable)
+            iemVmxVmcsSetGuestSegReg(pVmcs, iSegReg, pSelReg);
+        else
+        {
+            /*
+             * For unusable segments the attributes are undefined except for CS and SS.
+             * For the rest we don't bother preserving anything but the unusable bit.
+             */
+            switch (iSegReg)
+            {
+                case X86_SREG_CS:
+                    pVmcs->GuestCs          = pSelReg->Sel;
+                    pVmcs->u64GuestCsBase.u = pSelReg->u64Base;
+                    pVmcs->u32GuestCsLimit  = pSelReg->u32Limit;
+                    pVmcs->u32GuestCsAttr   = pSelReg->Attr.u & (  X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
+                                                                 | X86DESCATTR_UNUSABLE);
+                    break;
+
+                case X86_SREG_SS:
+                    pVmcs->GuestSs        = pSelReg->Sel;
+                    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+                        pVmcs->u64GuestSsBase.u &= UINT32_C(0xffffffff);
+                    pVmcs->u32GuestSsAttr = pSelReg->Attr.u & (X86DESCATTR_DPL | X86DESCATTR_UNUSABLE);
+                    break;
+
+                case X86_SREG_DS:
+                    pVmcs->GuestDs        = pSelReg->Sel;
+                    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+                        pVmcs->u64GuestDsBase.u &= UINT32_C(0xffffffff);
+                    pVmcs->u32GuestDsAttr = X86DESCATTR_UNUSABLE;
+                    break;
+
+                case X86_SREG_ES:
+                    pVmcs->GuestEs        = pSelReg->Sel;
+                    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+                        pVmcs->u64GuestEsBase.u &= UINT32_C(0xffffffff);
+                    pVmcs->u32GuestEsAttr = X86DESCATTR_UNUSABLE;
+                    break;
+
+                case X86_SREG_FS:
+                    pVmcs->GuestFs          = pSelReg->Sel;
+                    pVmcs->u64GuestFsBase.u = pSelReg->u64Base;
+                    pVmcs->u32GuestFsAttr   = X86DESCATTR_UNUSABLE;
+                    break;
+
+                case X86_SREG_GS:
+                    pVmcs->GuestGs          = pSelReg->Sel;
+                    pVmcs->u64GuestGsBase.u = pSelReg->u64Base;
+                    pVmcs->u32GuestGsAttr   = X86DESCATTR_UNUSABLE;
+                    break;
+            }
+        }
+    }
+
+    /* Segment attribute bits 31:17 and 11:8 MBZ. */
+    uint32_t const fValidAttrMask = X86DESCATTR_TYPE | X86DESCATTR_DT  | X86DESCATTR_DPL | X86DESCATTR_P
+                                  | X86DESCATTR_AVL  | X86DESCATTR_L   | X86DESCATTR_D   | X86DESCATTR_G
+                                  | X86DESCATTR_UNUSABLE;
+    /* LDTR. */
+    {
+        PCCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.ldtr;
+        pVmcs->GuestLdtr          = pSelReg->Sel;
+        pVmcs->u64GuestLdtrBase.u = pSelReg->u64Base;
+        Assert(X86_IS_CANONICAL(pSelReg->u64Base));
+        pVmcs->u32GuestLdtrLimit  = pSelReg->u32Limit;
+        pVmcs->u32GuestLdtrAttr   = pSelReg->Attr.u & fValidAttrMask;
+    }
+
+    /* TR. */
+    {
+        PCCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.tr;
+        pVmcs->GuestTr          = pSelReg->Sel;
+        pVmcs->u64GuestTrBase.u = pSelReg->u64Base;
+        pVmcs->u32GuestTrLimit  = pSelReg->u32Limit;
+        pVmcs->u32GuestTrAttr   = pSelReg->Attr.u & fValidAttrMask;
+    }
+
+    /* GDTR. */
+    pVmcs->u64GuestGdtrBase.u = pVCpu->cpum.GstCtx.gdtr.pGdt;
+    pVmcs->u32GuestGdtrLimit  = pVCpu->cpum.GstCtx.gdtr.cbGdt;
+
+    /* IDTR. */
+    pVmcs->u64GuestIdtrBase.u = pVCpu->cpum.GstCtx.idtr.pIdt;
+    pVmcs->u32GuestIdtrLimit  = pVCpu->cpum.GstCtx.idtr.cbIdt;
+}
+
+
+/**
+ * Saves guest non-register state as part of VM-exit.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ */
+static void iemVmxVmexitSaveGuestNonRegState(PVMCPUCC pVCpu, uint32_t uExitReason) RT_NOEXCEPT
+{
+    /*
+     * Save guest non-register state.
+     * See Intel spec. 27.3.4 "Saving Non-Register State".
+     */
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    /*
+     * Activity state.
+     * Most VM-exits will occur in the active state. However, if the first instruction
+     * following the VM-entry is a HLT instruction, and the MTF VM-execution control is set,
+     * the VM-exit will be from the HLT activity state.
+     *
+     * See Intel spec. 25.5.2 "Monitor Trap Flag".
+     */
+    /** @todo NSTVMX: Does triple-fault VM-exit reflect a shutdown activity state or
+     *        not? */
+    EMSTATE const enmActivityState = EMGetState(pVCpu);
+    switch (enmActivityState)
+    {
+        case EMSTATE_HALTED:    pVmcs->u32GuestActivityState = VMX_VMCS_GUEST_ACTIVITY_HLT;     break;
+        default:                pVmcs->u32GuestActivityState = VMX_VMCS_GUEST_ACTIVITY_ACTIVE;  break;
+    }
+
+    /*
+     * Interruptibility-state.
+     */
+    /* NMI. */
+    pVmcs->u32GuestIntrState = 0;
+    if (pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+    {
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking)
+            pVmcs->u32GuestIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
+    }
+    else
+    {
+        if (CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
+            pVmcs->u32GuestIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
+    }
+
+    /* Blocking-by-STI. */
+    if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx))
+    { /* probable */}
+    else
+    {
+        /** @todo NSTVMX: We can't distinguish between blocking-by-MovSS and blocking-by-STI
+         *        currently. */
+        if (pVCpu->cpum.GstCtx.rip == pVCpu->cpum.GstCtx.uRipInhibitInt)
+            pVmcs->u32GuestIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_STI; /** @todo r=bird: Why the STI one? MOVSS seems to block more and the one to use. */
+
+        /* Clear inhibition unconditionally since we've ensured it isn't set prior to executing VMLAUNCH/VMRESUME. */
+        CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
+    }
+    /* Nothing to do for SMI/enclave. We don't support enclaves or SMM yet. */
+
+    /*
+     * Pending debug exceptions.
+     *
+     * For VM-exits where it is not applicable, we can safely zero out the field.
+     * For VM-exits where it is applicable, it's expected to be updated by the caller already.
+     */
+    if (    uExitReason != VMX_EXIT_INIT_SIGNAL
+        &&  uExitReason != VMX_EXIT_SMI
+        &&  uExitReason != VMX_EXIT_ERR_MACHINE_CHECK
+        && !VMXIsVmexitTrapLike(uExitReason))
+    {
+        /** @todo NSTVMX: also must exclude VM-exits caused by debug exceptions when
+         *        block-by-MovSS is in effect. */
+        pVmcs->u64GuestPendingDbgXcpts.u = 0;
+    }
+
+    /*
+     * Save the VMX-preemption timer value back into the VMCS if the feature is enabled.
+     *
+     * For VMX-preemption timer VM-exits, we should have already written back 0 if the
+     * feature is supported back into the VMCS, and thus there is nothing further to do here.
+     */
+    if (   uExitReason != VMX_EXIT_PREEMPT_TIMER
+        && (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
+        pVmcs->u32PreemptTimer = iemVmxCalcPreemptTimer(pVCpu);
+
+    /*
+     * Save the guest PAE PDPTEs.
+     */
+    if (   !CPUMIsGuestInPAEModeEx(&pVCpu->cpum.GstCtx)
+        || !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT))
+    {
+        /*
+         * Without EPT or when the nested-guest is not using PAE paging, the values saved
+         * in the VMCS during VM-exit are undefined. We zero them here for consistency.
+         */
+        pVmcs->u64GuestPdpte0.u = 0;
+        pVmcs->u64GuestPdpte1.u = 0;
+        pVmcs->u64GuestPdpte2.u = 0;
+        pVmcs->u64GuestPdpte3.u = 0;
+    }
+    else
+    {
+        /*
+         * With EPT and when the nested-guest is using PAE paging, we update the PDPTEs from
+         * the nested-guest CPU context. Both IEM (Mov CRx) and hardware-assisted execution
+         * of the nested-guest is expected to have updated them.
+         */
+        pVmcs->u64GuestPdpte0.u = pVCpu->cpum.GstCtx.aPaePdpes[0].u;
+        pVmcs->u64GuestPdpte1.u = pVCpu->cpum.GstCtx.aPaePdpes[1].u;
+        pVmcs->u64GuestPdpte2.u = pVCpu->cpum.GstCtx.aPaePdpes[2].u;
+        pVmcs->u64GuestPdpte3.u = pVCpu->cpum.GstCtx.aPaePdpes[3].u;
+    }
+
+    /* Clear PGM's copy of the EPT pointer for added safety. */
+    if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT)
+        PGMSetGuestEptPtr(pVCpu, 0 /* uEptPtr */);
+}
+
+
+/**
+ * Saves the guest-state as part of VM-exit.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ */
+static void iemVmxVmexitSaveGuestState(PVMCPUCC pVCpu, uint32_t uExitReason) RT_NOEXCEPT
+{
+    iemVmxVmexitSaveGuestControlRegsMsrs(pVCpu);
+    iemVmxVmexitSaveGuestSegRegs(pVCpu);
+
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64GuestRip.u    = pVCpu->cpum.GstCtx.rip;
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64GuestRsp.u    = pVCpu->cpum.GstCtx.rsp;
+    pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64GuestRFlags.u = pVCpu->cpum.GstCtx.rflags.u;  /** @todo NSTVMX: Check RFLAGS.RF handling. */
+
+    iemVmxVmexitSaveGuestNonRegState(pVCpu, uExitReason);
+}
+
+
+/**
+ * Saves the guest MSRs into the VM-exit MSR-store area as part of VM-exit.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason (for diagnostic purposes).
+ */
+static int iemVmxVmexitSaveGuestAutoMsrs(PVMCPUCC pVCpu, uint32_t uExitReason) RT_NOEXCEPT
+{
+    /*
+     * Save guest MSRs.
+     * See Intel spec. 27.4 "Saving MSRs".
+     */
+    PVMXVVMCS const    pVmcs      = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char * const pszFailure = "VMX-abort";
+
+    /*
+     * The VM-exit MSR-store area address need not be a valid guest-physical address if the
+     * VM-exit MSR-store count is 0. If this is the case, bail early without reading it.
+     * See Intel spec. 24.7.2 "VM-Exit Controls for MSRs".
+     */
+    uint32_t const cMsrs = RT_MIN(pVmcs->u32ExitMsrStoreCount, RT_ELEMENTS(pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrStoreArea));
+    if (!cMsrs)
+        return VINF_SUCCESS;
+
+    /*
+     * Verify the MSR auto-store count. Physical CPUs can behave unpredictably if the count
+     * is exceeded including possibly raising #MC exceptions during VMX transition. Our
+     * implementation causes a VMX-abort followed by a triple-fault.
+     */
+    bool const fIsMsrCountValid = iemVmxIsAutoMsrCountValid(pVCpu, cMsrs);
+    if (fIsMsrCountValid)
+    { /* likely */ }
+    else
+        IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStoreCount);
+
+    /*
+     * Optimization if the nested hypervisor is using the same guest-physical page for both
+     * the VM-entry MSR-load area as well as the VM-exit MSR store area.
+     */
+    PVMXAUTOMSR    pMsrArea;
+    RTGCPHYS const GCPhysVmEntryMsrLoadArea = pVmcs->u64AddrEntryMsrLoad.u;
+    RTGCPHYS const GCPhysVmExitMsrStoreArea = pVmcs->u64AddrExitMsrStore.u;
+    if (GCPhysVmEntryMsrLoadArea == GCPhysVmExitMsrStoreArea)
+        pMsrArea = pVCpu->cpum.GstCtx.hwvirt.vmx.aEntryMsrLoadArea;
+    else
+    {
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrStoreArea[0],
+                                         GCPhysVmExitMsrStoreArea, cMsrs * sizeof(VMXAUTOMSR));
+        if (RT_SUCCESS(rc))
+            pMsrArea = pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrStoreArea;
+        else
+        {
+            AssertMsgFailed(("VM-exit: Failed to read MSR auto-store area at %#RGp, rc=%Rrc\n", GCPhysVmExitMsrStoreArea, rc));
+            IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStorePtrReadPhys);
+        }
+    }
+
+    /*
+     * Update VM-exit MSR store area.
+     */
+    PVMXAUTOMSR pMsr = pMsrArea;
+    for (uint32_t idxMsr = 0; idxMsr < cMsrs; idxMsr++, pMsr++)
+    {
+        if (   !pMsr->u32Reserved
+            &&  pMsr->u32Msr != MSR_IA32_SMBASE
+            &&  pMsr->u32Msr >> 8 != MSR_IA32_X2APIC_START >> 8)
+        {
+            VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pMsr->u32Msr, &pMsr->u64Value);
+            if (rcStrict == VINF_SUCCESS)
+                continue;
+
+            /*
+             * If we're in ring-0, we cannot handle returns to ring-3 at this point and continue VM-exit.
+             * If any nested hypervisor loads MSRs that require ring-3 handling, we cause a VMX-abort
+             * recording the MSR index in the auxiliary info. field and indicated further by our
+             * own, specific diagnostic code. Later, we can try implement handling of the MSR in ring-0
+             * if possible, or come up with a better, generic solution.
+             */
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uAbortAux = pMsr->u32Msr;
+            VMXVDIAG const enmDiag = rcStrict == VINF_CPUM_R3_MSR_READ
+                                   ? kVmxVDiag_Vmexit_MsrStoreRing3
+                                   : kVmxVDiag_Vmexit_MsrStore;
+            IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, enmDiag);
+        }
+        else
+        {
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uAbortAux = pMsr->u32Msr;
+            IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStoreRsvd);
+        }
+    }
+
+    /*
+     * Commit the VM-exit MSR store are to guest memory.
+     */
+    int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmExitMsrStoreArea, pMsrArea, cMsrs * sizeof(VMXAUTOMSR));
+    if (RT_SUCCESS(rc))
+        return VINF_SUCCESS;
+
+    NOREF(uExitReason);
+    NOREF(pszFailure);
+
+    AssertMsgFailed(("VM-exit: Failed to write MSR auto-store area at %#RGp, rc=%Rrc\n", GCPhysVmExitMsrStoreArea, rc));
+    IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrStorePtrWritePhys);
+}
+
+
+/**
+ * Performs a VMX abort (due to an fatal error during VM-exit).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   enmAbort    The VMX abort reason.
+ */
+static VBOXSTRICTRC iemVmxAbort(PVMCPUCC pVCpu, VMXABORT enmAbort) RT_NOEXCEPT
+{
+    /*
+     * Perform the VMX abort.
+     * See Intel spec. 27.7 "VMX Aborts".
+     */
+    LogFunc(("enmAbort=%u (%s) -> RESET\n", enmAbort, VMXGetAbortDesc(enmAbort)));
+
+    /* We don't support SMX yet. */
+    pVCpu->cpum.GstCtx.hwvirt.vmx.enmAbort = enmAbort;
+    if (IEM_VMX_HAS_CURRENT_VMCS(pVCpu))
+    {
+        RTGCPHYS const GCPhysVmcs  = IEM_VMX_GET_CURRENT_VMCS(pVCpu);
+        uint32_t const offVmxAbort = RT_UOFFSETOF(VMXVVMCS, enmVmxAbort);
+        PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmcs + offVmxAbort, &enmAbort, sizeof(enmAbort));
+    }
+
+    return VINF_EM_TRIPLE_FAULT;
+}
+
+
+/**
+ * Loads host control registers, debug registers and MSRs as part of VM-exit.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxVmexitLoadHostControlRegsMsrs(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /*
+     * Load host control registers, debug registers and MSRs.
+     * See Intel spec. 27.5.1 "Loading Host Control Registers, Debug Registers, MSRs".
+     */
+    PCVMXVVMCS const pVmcs           = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    bool const       fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+
+    /* CR0. */
+    {
+        /* Bits 63:32, 28:19, 17, 15:6, ET, CD, NW and CR0 fixed bits are not modified. */
+        uint64_t const fCr0IgnMask   = VMX_EXIT_HOST_CR0_IGNORE_MASK;
+        uint64_t const uHostCr0      = pVmcs->u64HostCr0.u;
+        uint64_t const uGuestCr0     = pVCpu->cpum.GstCtx.cr0;
+        uint64_t const uValidHostCr0 = (uHostCr0 & ~fCr0IgnMask) | (uGuestCr0 & fCr0IgnMask);
+
+        /* Verify we have not modified CR0 fixed bits in VMX operation. */
+#ifdef VBOX_STRICT
+        uint64_t const uCr0Mb1 = iemVmxGetCr0Fixed0(pVCpu, true /* fVmxNonRootMode */);
+        bool const fUx         = RT_BOOL(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
+        AssertMsg(   (uValidHostCr0 & uCr0Mb1) == uCr0Mb1
+                  && (uValidHostCr0 & ~VMX_V_CR0_FIXED1) == 0,
+                  ("host=%#RX64 guest=%#RX64 mb1=%#RX64 valid_host_cr0=%#RX64 fUx=%RTbool\n",
+                   uHostCr0, uGuestCr0, uCr0Mb1, uValidHostCr0, fUx));
+#endif
+        Assert(!(uValidHostCr0 >> 32));
+        CPUMSetGuestCR0(pVCpu, uValidHostCr0);
+    }
+
+    /* CR4. */
+    {
+        /* CR4 fixed bits are not modified. */
+        uint64_t const uCr4Mb1       = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+        uint64_t const uCr4Mb0       = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+        uint64_t const uHostCr4      = pVmcs->u64HostCr4.u;
+        uint64_t       uValidHostCr4 = (uHostCr4 & uCr4Mb0) | uCr4Mb1;
+        if (fHostInLongMode)
+            uValidHostCr4 |= X86_CR4_PAE;
+        else
+            uValidHostCr4 &= ~(uint64_t)X86_CR4_PCIDE;
+
+        /* Verify we have not modified CR4 fixed bits in VMX non-root operation. */
+        AssertMsg(   (uValidHostCr4 &  uCr4Mb1) == uCr4Mb1
+                  && (uValidHostCr4 & ~uCr4Mb0) == 0,
+                  ("host=%#RX64 guest=%#RX64, uCr4Mb1=%#RX64 uCr4Mb0=%#RX64 valid_host_cr4=%#RX64\n",
+                   uHostCr4, pVCpu->cpum.GstCtx.cr4, uCr4Mb1, uCr4Mb0, uValidHostCr4));
+        CPUMSetGuestCR4(pVCpu, uValidHostCr4);
+    }
+
+    /* CR3 (host value validated while checking host-state during VM-entry). */
+    pVCpu->cpum.GstCtx.cr3 = pVmcs->u64HostCr3.u;
+
+    /* DR7. */
+    pVCpu->cpum.GstCtx.dr[7] = X86_DR7_INIT_VAL;
+
+    /** @todo NSTVMX: Support IA32_DEBUGCTL MSR */
+
+    /* Save SYSENTER CS, ESP, EIP (host value validated while checking host-state during VM-entry). */
+    pVCpu->cpum.GstCtx.SysEnter.eip = pVmcs->u64HostSysenterEip.u;
+    pVCpu->cpum.GstCtx.SysEnter.esp = pVmcs->u64HostSysenterEsp.u;
+    pVCpu->cpum.GstCtx.SysEnter.cs  = pVmcs->u32HostSysenterCs;
+
+    /* FS, GS bases are loaded later while we load host segment registers. */
+
+    /* EFER MSR (host value validated while checking host-state during VM-entry). */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
+        pVCpu->cpum.GstCtx.msrEFER = pVmcs->u64HostEferMsr.u;
+    else if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        if (fHostInLongMode)
+            pVCpu->cpum.GstCtx.msrEFER |=  (MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
+        else
+            pVCpu->cpum.GstCtx.msrEFER &= ~(MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
+    }
+
+    /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */
+
+    /* PAT MSR (host value is validated while checking host-state during VM-entry). */
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_PAT_MSR)
+        pVCpu->cpum.GstCtx.msrPAT = pVmcs->u64HostPatMsr.u;
+
+    /* We don't support IA32_BNDCFGS MSR yet. */
+}
+
+
+/**
+ * Loads host segment registers, GDTR, IDTR, LDTR and TR as part of VM-exit.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxVmexitLoadHostSegRegs(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /*
+     * Load host segment registers, GDTR, IDTR, LDTR and TR.
+     * See Intel spec. 27.5.2 "Loading Host Segment and Descriptor-Table Registers".
+     *
+     * Warning! Be careful to not touch fields that are reserved by VT-x,
+     * e.g. segment limit high bits stored in segment attributes (in bits 11:8).
+     */
+    PCVMXVVMCS const pVmcs           = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    bool const       fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+
+    /* CS, SS, ES, DS, FS, GS. */
+    for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++)
+    {
+        RTSEL const HostSel   = iemVmxVmcsGetHostSelReg(pVmcs, iSegReg);
+        bool const  fUnusable = RT_BOOL(HostSel == 0);
+        PCPUMSELREG pSelReg   = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
+
+        /* Selector. */
+        pSelReg->Sel      = HostSel;
+        pSelReg->ValidSel = HostSel;
+        pSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+
+        /* Limit. */
+        pSelReg->u32Limit = 0xffffffff;
+
+        /* Base. */
+        pSelReg->u64Base = 0;
+
+        /* Attributes. */
+        if (iSegReg == X86_SREG_CS)
+        {
+            pSelReg->Attr.n.u4Type        = X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_ACCESSED;
+            pSelReg->Attr.n.u1DescType    = 1;
+            pSelReg->Attr.n.u2Dpl         = 0;
+            pSelReg->Attr.n.u1Present     = 1;
+            pSelReg->Attr.n.u1Long        = fHostInLongMode;
+            pSelReg->Attr.n.u1DefBig      = !fHostInLongMode;
+            pSelReg->Attr.n.u1Granularity = 1;
+            Assert(!pSelReg->Attr.n.u1Unusable);
+            Assert(!fUnusable);
+        }
+        else
+        {
+            pSelReg->Attr.n.u4Type        = X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED;
+            pSelReg->Attr.n.u1DescType    = 1;
+            pSelReg->Attr.n.u2Dpl         = 0;
+            pSelReg->Attr.n.u1Present     = 1;
+            pSelReg->Attr.n.u1DefBig      = 1;
+            pSelReg->Attr.n.u1Granularity = 1;
+            pSelReg->Attr.n.u1Unusable    = fUnusable;
+        }
+    }
+
+    /* FS base. */
+    if (   !pVCpu->cpum.GstCtx.fs.Attr.n.u1Unusable
+        ||  fHostInLongMode)
+    {
+        Assert(X86_IS_CANONICAL(pVmcs->u64HostFsBase.u));
+        pVCpu->cpum.GstCtx.fs.u64Base = pVmcs->u64HostFsBase.u;
+    }
+
+    /* GS base. */
+    if (   !pVCpu->cpum.GstCtx.gs.Attr.n.u1Unusable
+        ||  fHostInLongMode)
+    {
+        Assert(X86_IS_CANONICAL(pVmcs->u64HostGsBase.u));
+        pVCpu->cpum.GstCtx.gs.u64Base = pVmcs->u64HostGsBase.u;
+    }
+
+    /* TR. */
+    Assert(X86_IS_CANONICAL(pVmcs->u64HostTrBase.u));
+    Assert(!pVCpu->cpum.GstCtx.tr.Attr.n.u1Unusable);
+    pVCpu->cpum.GstCtx.tr.Sel                  = pVmcs->HostTr;
+    pVCpu->cpum.GstCtx.tr.ValidSel             = pVmcs->HostTr;
+    pVCpu->cpum.GstCtx.tr.fFlags               = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.tr.u32Limit             = X86_SEL_TYPE_SYS_386_TSS_LIMIT_MIN;
+    pVCpu->cpum.GstCtx.tr.u64Base              = pVmcs->u64HostTrBase.u;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u4Type        = X86_SEL_TYPE_SYS_386_TSS_BUSY;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u1DescType    = 0;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u2Dpl         = 0;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u1Present     = 1;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u1DefBig      = 0;
+    pVCpu->cpum.GstCtx.tr.Attr.n.u1Granularity = 0;
+
+    /* LDTR (Warning! do not touch the base and limits here). */
+    pVCpu->cpum.GstCtx.ldtr.Sel                = 0;
+    pVCpu->cpum.GstCtx.ldtr.ValidSel           = 0;
+    pVCpu->cpum.GstCtx.ldtr.fFlags             = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.ldtr.Attr.u             = X86DESCATTR_UNUSABLE;
+
+    /* GDTR. */
+    Assert(X86_IS_CANONICAL(pVmcs->u64HostGdtrBase.u));
+    pVCpu->cpum.GstCtx.gdtr.pGdt  = pVmcs->u64HostGdtrBase.u;
+    pVCpu->cpum.GstCtx.gdtr.cbGdt = 0xffff;
+
+    /* IDTR.*/
+    Assert(X86_IS_CANONICAL(pVmcs->u64HostIdtrBase.u));
+    pVCpu->cpum.GstCtx.idtr.pIdt  = pVmcs->u64HostIdtrBase.u;
+    pVCpu->cpum.GstCtx.idtr.cbIdt = 0xffff;
+}
+
+
+/**
+ * Loads the host MSRs from the VM-exit MSR-load area as part of VM-exit.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VMX instruction name (for logging purposes).
+ */
+static int iemVmxVmexitLoadHostAutoMsrs(PVMCPUCC pVCpu, uint32_t uExitReason) RT_NOEXCEPT
+{
+    /*
+     * Load host MSRs.
+     * See Intel spec. 27.6 "Loading MSRs".
+     */
+    PCVMXVVMCS const   pVmcs      = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char * const pszFailure = "VMX-abort";
+
+    /*
+     * The VM-exit MSR-load area address need not be a valid guest-physical address if the
+     * VM-exit MSR load count is 0. If this is the case, bail early without reading it.
+     * See Intel spec. 24.7.2 "VM-Exit Controls for MSRs".
+     */
+    uint32_t const cMsrs = RT_MIN(pVmcs->u32ExitMsrLoadCount, RT_ELEMENTS(pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrLoadArea));
+    if (!cMsrs)
+        return VINF_SUCCESS;
+
+    /*
+     * Verify the MSR auto-load count. Physical CPUs can behave unpredictably if the count
+     * is exceeded including possibly raising #MC exceptions during VMX transition. Our
+     * implementation causes a VMX-abort followed by a triple-fault.
+     */
+    bool const fIsMsrCountValid = iemVmxIsAutoMsrCountValid(pVCpu, cMsrs);
+    if (fIsMsrCountValid)
+    { /* likely */ }
+    else
+        IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrLoadCount);
+
+    RTGCPHYS const GCPhysVmExitMsrLoadArea = pVmcs->u64AddrExitMsrLoad.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrLoadArea[0],
+                                     GCPhysVmExitMsrLoadArea, cMsrs * sizeof(VMXAUTOMSR));
+    if (RT_SUCCESS(rc))
+    {
+        PCVMXAUTOMSR pMsr = pVCpu->cpum.GstCtx.hwvirt.vmx.aExitMsrLoadArea;
+        for (uint32_t idxMsr = 0; idxMsr < cMsrs; idxMsr++, pMsr++)
+        {
+            if (   !pMsr->u32Reserved
+                &&  pMsr->u32Msr != MSR_K8_FS_BASE
+                &&  pMsr->u32Msr != MSR_K8_GS_BASE
+                &&  pMsr->u32Msr != MSR_K6_EFER
+                &&  pMsr->u32Msr != MSR_IA32_SMM_MONITOR_CTL
+                &&  pMsr->u32Msr >> 8 != MSR_IA32_X2APIC_START >> 8)
+            {
+                VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, pMsr->u32Msr, pMsr->u64Value);
+                if (rcStrict == VINF_SUCCESS)
+                    continue;
+
+                /*
+                 * If we're in ring-0, we cannot handle returns to ring-3 at this point and continue VM-exit.
+                 * If any nested hypervisor loads MSRs that require ring-3 handling, we cause a VMX-abort
+                 * recording the MSR index in the auxiliary info. field and indicated further by our
+                 * own, specific diagnostic code. Later, we can try implement handling of the MSR in ring-0
+                 * if possible, or come up with a better, generic solution.
+                 */
+                pVCpu->cpum.GstCtx.hwvirt.vmx.uAbortAux = pMsr->u32Msr;
+                VMXVDIAG const enmDiag = rcStrict == VINF_CPUM_R3_MSR_WRITE
+                                       ? kVmxVDiag_Vmexit_MsrLoadRing3
+                                       : kVmxVDiag_Vmexit_MsrLoad;
+                IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, enmDiag);
+            }
+            else
+                IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrLoadRsvd);
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("VM-exit: Failed to read MSR auto-load area at %#RGp, rc=%Rrc\n", GCPhysVmExitMsrLoadArea, rc));
+        IEM_VMX_VMEXIT_FAILED_RET(pVCpu, uExitReason, pszFailure, kVmxVDiag_Vmexit_MsrLoadPtrReadPhys);
+    }
+
+    NOREF(uExitReason);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the host state as part of VM-exit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason (for logging purposes).
+ */
+static VBOXSTRICTRC iemVmxVmexitLoadHostState(PVMCPUCC pVCpu, uint32_t uExitReason) RT_NOEXCEPT
+{
+    /*
+     * Load host state.
+     * See Intel spec. 27.5 "Loading Host State".
+     */
+    PCVMXVVMCS const pVmcs           = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    bool const       fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+
+    /* We cannot return from a long-mode guest to a host that is not in long mode. */
+    if (    CPUMIsGuestInLongMode(pVCpu)
+        && !fHostInLongMode)
+    {
+        Log(("VM-exit from long-mode guest to host not in long-mode -> VMX-Abort\n"));
+        return iemVmxAbort(pVCpu, VMXABORT_HOST_NOT_IN_LONG_MODE);
+    }
+
+    /*
+     * Check host PAE PDPTEs prior to loading the host state.
+     * See Intel spec. 26.5.4 "Checking and Loading Host Page-Directory-Pointer-Table Entries".
+     */
+    if (   (pVmcs->u64HostCr4.u & X86_CR4_PAE)
+        && !fHostInLongMode
+        && (   !CPUMIsGuestInPAEModeEx(&pVCpu->cpum.GstCtx)
+            || pVmcs->u64HostCr3.u != pVCpu->cpum.GstCtx.cr3))
+    {
+        int const rc = PGMGstMapPaePdpesAtCr3(pVCpu, pVmcs->u64HostCr3.u);
+        if (RT_SUCCESS(rc))
+        { /* likely*/ }
+        else
+        {
+            IEM_VMX_VMEXIT_FAILED(pVCpu, uExitReason, "VMX-abort", kVmxVDiag_Vmexit_HostPdpte);
+            return iemVmxAbort(pVCpu, VMXBOART_HOST_PDPTE);
+        }
+    }
+
+    iemVmxVmexitLoadHostControlRegsMsrs(pVCpu);
+    iemVmxVmexitLoadHostSegRegs(pVCpu);
+
+    /*
+     * Load host RIP, RSP and RFLAGS.
+     * See Intel spec. 27.5.3 "Loading Host RIP, RSP and RFLAGS"
+     */
+    pVCpu->cpum.GstCtx.rip      = pVmcs->u64HostRip.u;
+    pVCpu->cpum.GstCtx.rsp      = pVmcs->u64HostRsp.u;
+    pVCpu->cpum.GstCtx.rflags.u = X86_EFL_1;
+
+    /* Clear address range monitoring. */
+    EMMonitorWaitClear(pVCpu);
+
+    /* Perform the VMX transition (PGM updates). */
+    VBOXSTRICTRC rcStrict = iemVmxTransition(pVCpu);
+    if (rcStrict == VINF_SUCCESS)
+    { /* likely */ }
+    else if (RT_SUCCESS(rcStrict))
+    {
+        Log3(("VM-exit: iemVmxTransition returns %Rrc (uExitReason=%u) -> Setting passup status\n", VBOXSTRICTRC_VAL(rcStrict),
+              uExitReason));
+        rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+    }
+    else
+    {
+        Log3(("VM-exit: iemVmxTransition failed! rc=%Rrc (uExitReason=%u)\n", VBOXSTRICTRC_VAL(rcStrict), uExitReason));
+        return VBOXSTRICTRC_VAL(rcStrict);
+    }
+
+    Assert(rcStrict == VINF_SUCCESS);
+
+    /* Load MSRs from the VM-exit auto-load MSR area. */
+    int rc = iemVmxVmexitLoadHostAutoMsrs(pVCpu, uExitReason);
+    if (RT_FAILURE(rc))
+    {
+        Log(("VM-exit failed while loading host MSRs -> VMX-Abort\n"));
+        return iemVmxAbort(pVCpu, VMXABORT_LOAD_HOST_MSR);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets VM-exit instruction information along with any displacement for an
+ * instruction VM-exit.
+ *
+ * @returns The VM-exit instruction information.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ * @param   uInstrId        The VM-exit instruction identity (VMXINSTRID_XXX).
+ * @param   pGCPtrDisp      Where to store the displacement field. Optional, can be
+ *                          NULL.
+ */
+static uint32_t iemVmxGetExitInstrInfo(PVMCPUCC pVCpu, uint32_t uExitReason, VMXINSTRID uInstrId, PRTGCPTR pGCPtrDisp) RT_NOEXCEPT
+{
+    RTGCPTR          GCPtrDisp;
+    VMXEXITINSTRINFO ExitInstrInfo;
+    ExitInstrInfo.u = 0;
+
+    /*
+     * Get and parse the ModR/M byte from our decoded opcodes.
+     */
+    uint8_t bRm;
+    uint8_t const offModRm = pVCpu->iem.s.offModRm;
+    IEM_MODRM_GET_U8(pVCpu, bRm, offModRm);
+    if ((bRm & X86_MODRM_MOD_MASK) == (3 << X86_MODRM_MOD_SHIFT))
+    {
+        /*
+         * ModR/M indicates register addressing.
+         *
+         * The primary/secondary register operands are reported in the iReg1 or iReg2
+         * fields depending on whether it is a read/write form.
+         */
+        uint8_t idxReg1;
+        uint8_t idxReg2;
+        if (!VMXINSTRID_IS_MODRM_PRIMARY_OP_W(uInstrId))
+        {
+            idxReg1 = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg;
+            idxReg2 = (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB;
+        }
+        else
+        {
+            idxReg1 = (bRm & X86_MODRM_RM_MASK) | pVCpu->iem.s.uRexB;
+            idxReg2 = ((bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK) | pVCpu->iem.s.uRexReg;
+        }
+        ExitInstrInfo.All.u2Scaling       = 0;
+        ExitInstrInfo.All.iReg1           = idxReg1;
+        ExitInstrInfo.All.u3AddrSize      = pVCpu->iem.s.enmEffAddrMode;
+        ExitInstrInfo.All.fIsRegOperand   = 1;
+        ExitInstrInfo.All.uOperandSize    = pVCpu->iem.s.enmEffOpSize;
+        ExitInstrInfo.All.iSegReg         = 0;
+        ExitInstrInfo.All.iIdxReg         = 0;
+        ExitInstrInfo.All.fIdxRegInvalid  = 1;
+        ExitInstrInfo.All.iBaseReg        = 0;
+        ExitInstrInfo.All.fBaseRegInvalid = 1;
+        ExitInstrInfo.All.iReg2           = idxReg2;
+
+        /* Displacement not applicable for register addressing. */
+        GCPtrDisp = 0;
+    }
+    else
+    {
+        /*
+         * ModR/M indicates memory addressing.
+         */
+        uint8_t  uScale        = 0;
+        bool     fBaseRegValid = false;
+        bool     fIdxRegValid  = false;
+        uint8_t  iBaseReg      = 0;
+        uint8_t  iIdxReg       = 0;
+        if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_16BIT)
+        {
+            /*
+             * Parse the ModR/M, displacement for 16-bit addressing mode.
+             * See Intel instruction spec. Table 2-1. "16-Bit Addressing Forms with the ModR/M Byte".
+             */
+            uint16_t u16Disp = 0;
+            uint8_t const offDisp = offModRm + sizeof(bRm);
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 6)
+            {
+                /* Displacement without any registers. */
+                IEM_DISP_GET_U16(pVCpu, u16Disp, offDisp);
+            }
+            else
+            {
+                /* Register (index and base). */
+                switch (bRm & X86_MODRM_RM_MASK)
+                {
+                    case 0: fBaseRegValid = true; iBaseReg = X86_GREG_xBX; fIdxRegValid = true; iIdxReg = X86_GREG_xSI; break;
+                    case 1: fBaseRegValid = true; iBaseReg = X86_GREG_xBX; fIdxRegValid = true; iIdxReg = X86_GREG_xDI; break;
+                    case 2: fBaseRegValid = true; iBaseReg = X86_GREG_xBP; fIdxRegValid = true; iIdxReg = X86_GREG_xSI; break;
+                    case 3: fBaseRegValid = true; iBaseReg = X86_GREG_xBP; fIdxRegValid = true; iIdxReg = X86_GREG_xDI; break;
+                    case 4: fIdxRegValid  = true; iIdxReg  = X86_GREG_xSI; break;
+                    case 5: fIdxRegValid  = true; iIdxReg  = X86_GREG_xDI; break;
+                    case 6: fBaseRegValid = true; iBaseReg = X86_GREG_xBP; break;
+                    case 7: fBaseRegValid = true; iBaseReg = X86_GREG_xBX; break;
+                }
+
+                /* Register + displacement. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0:                                                  break;
+                    case 1: IEM_DISP_GET_S8_SX_U16(pVCpu, u16Disp, offDisp); break;
+                    case 2: IEM_DISP_GET_U16(pVCpu, u16Disp, offDisp);       break;
+                    default:
+                    {
+                        /* Register addressing, handled at the beginning. */
+                        AssertMsgFailed(("ModR/M %#x implies register addressing, memory addressing expected!", bRm));
+                        break;
+                    }
+                }
+            }
+
+            Assert(!uScale);                    /* There's no scaling/SIB byte for 16-bit addressing. */
+            GCPtrDisp = (int16_t)u16Disp;       /* Sign-extend the displacement. */
+        }
+        else if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_32BIT)
+        {
+            /*
+             * Parse the ModR/M, SIB, displacement for 32-bit addressing mode.
+             * See Intel instruction spec. Table 2-2. "32-Bit Addressing Forms with the ModR/M Byte".
+             */
+            uint32_t u32Disp = 0;
+            if ((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5)
+            {
+                /* Displacement without any registers. */
+                uint8_t const offDisp = offModRm + sizeof(bRm);
+                IEM_DISP_GET_U32(pVCpu, u32Disp, offDisp);
+            }
+            else
+            {
+                /* Register (and perhaps scale, index and base). */
+                uint8_t offDisp = offModRm + sizeof(bRm);
+                iBaseReg = (bRm & X86_MODRM_RM_MASK);
+                if (iBaseReg == 4)
+                {
+                    /* An SIB byte follows the ModR/M byte, parse it. */
+                    uint8_t bSib;
+                    uint8_t const offSib = offModRm + sizeof(bRm);
+                    IEM_SIB_GET_U8(pVCpu, bSib, offSib);
+
+                    /* A displacement may follow SIB, update its offset. */
+                    offDisp += sizeof(bSib);
+
+                    /* Get the scale. */
+                    uScale = (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                    /* Get the index register. */
+                    iIdxReg = (bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK;
+                    fIdxRegValid = RT_BOOL(iIdxReg != 4);
+
+                    /* Get the base register. */
+                    iBaseReg = bSib & X86_SIB_BASE_MASK;
+                    fBaseRegValid = true;
+                    if (iBaseReg == 5)
+                    {
+                        if ((bRm & X86_MODRM_MOD_MASK) == 0)
+                        {
+                            /* Mod is 0 implies a 32-bit displacement with no base. */
+                            fBaseRegValid = false;
+                            IEM_DISP_GET_U32(pVCpu, u32Disp, offDisp);
+                        }
+                        else
+                        {
+                            /* Mod is not 0 implies an 8-bit/32-bit displacement (handled below) with an EBP base. */
+                            iBaseReg = X86_GREG_xBP;
+                        }
+                    }
+                }
+
+                /* Register + displacement. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0: /* Handled above */                              break;
+                    case 1: IEM_DISP_GET_S8_SX_U32(pVCpu, u32Disp, offDisp); break;
+                    case 2: IEM_DISP_GET_U32(pVCpu, u32Disp, offDisp);       break;
+                    default:
+                    {
+                        /* Register addressing, handled at the beginning. */
+                        AssertMsgFailed(("ModR/M %#x implies register addressing, memory addressing expected!", bRm));
+                        break;
+                    }
+                }
+            }
+
+            GCPtrDisp = (int32_t)u32Disp;       /* Sign-extend the displacement. */
+        }
+        else
+        {
+            Assert(pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT);
+
+            /*
+             * Parse the ModR/M, SIB, displacement for 64-bit addressing mode.
+             * See Intel instruction spec. 2.2 "IA-32e Mode".
+             */
+            uint64_t u64Disp = 0;
+            bool const fRipRelativeAddr = RT_BOOL((bRm & (X86_MODRM_MOD_MASK | X86_MODRM_RM_MASK)) == 5);
+            if (fRipRelativeAddr)
+            {
+                /*
+                 * RIP-relative addressing mode.
+                 *
+                 * The displacement is 32-bit signed implying an offset range of +/-2G.
+                 * See Intel instruction spec. 2.2.1.6 "RIP-Relative Addressing".
+                 */
+                uint8_t const offDisp = offModRm + sizeof(bRm);
+                IEM_DISP_GET_S32_SX_U64(pVCpu, u64Disp, offDisp);
+            }
+            else
+            {
+                uint8_t offDisp = offModRm + sizeof(bRm);
+
+                /*
+                 * Register (and perhaps scale, index and base).
+                 *
+                 * REX.B extends the most-significant bit of the base register. However, REX.B
+                 * is ignored while determining whether an SIB follows the opcode. Hence, we
+                 * shall OR any REX.B bit -after- inspecting for an SIB byte below.
+                 *
+                 * See Intel instruction spec. Table 2-5. "Special Cases of REX Encodings".
+                 */
+                iBaseReg = (bRm & X86_MODRM_RM_MASK);
+                if (iBaseReg == 4)
+                {
+                    /* An SIB byte follows the ModR/M byte, parse it. Displacement (if any) follows SIB. */
+                    uint8_t bSib;
+                    uint8_t const offSib = offModRm + sizeof(bRm);
+                    IEM_SIB_GET_U8(pVCpu, bSib, offSib);
+
+                    /* Displacement may follow SIB, update its offset. */
+                    offDisp += sizeof(bSib);
+
+                    /* Get the scale. */
+                    uScale = (bSib >> X86_SIB_SCALE_SHIFT) & X86_SIB_SCALE_SMASK;
+
+                    /* Get the index. */
+                    iIdxReg = ((bSib >> X86_SIB_INDEX_SHIFT) & X86_SIB_INDEX_SMASK) | pVCpu->iem.s.uRexIndex;
+                    fIdxRegValid = RT_BOOL(iIdxReg != 4);   /* R12 -can- be used as an index register. */
+
+                    /* Get the base. */
+                    iBaseReg = (bSib & X86_SIB_BASE_MASK);
+                    fBaseRegValid = true;
+                    if (iBaseReg == 5)
+                    {
+                        if ((bRm & X86_MODRM_MOD_MASK) == 0)
+                        {
+                            /* Mod is 0 implies a signed 32-bit displacement with no base. */
+                            IEM_DISP_GET_S32_SX_U64(pVCpu, u64Disp, offDisp);
+                        }
+                        else
+                        {
+                            /* Mod is non-zero implies an 8-bit/32-bit displacement (handled below) with RBP or R13 as base. */
+                            iBaseReg = pVCpu->iem.s.uRexB ? X86_GREG_x13 : X86_GREG_xBP;
+                        }
+                    }
+                }
+                iBaseReg |= pVCpu->iem.s.uRexB;
+
+                /* Register + displacement. */
+                switch ((bRm >> X86_MODRM_MOD_SHIFT) & X86_MODRM_MOD_SMASK)
+                {
+                    case 0: /* Handled above */                               break;
+                    case 1: IEM_DISP_GET_S8_SX_U64(pVCpu, u64Disp, offDisp);  break;
+                    case 2: IEM_DISP_GET_S32_SX_U64(pVCpu, u64Disp, offDisp); break;
+                    default:
+                    {
+                        /* Register addressing, handled at the beginning. */
+                        AssertMsgFailed(("ModR/M %#x implies register addressing, memory addressing expected!", bRm));
+                        break;
+                    }
+                }
+            }
+
+            GCPtrDisp = fRipRelativeAddr ? pVCpu->cpum.GstCtx.rip + u64Disp : u64Disp;
+        }
+
+        /*
+         * The primary or secondary register operand is reported in iReg2 depending
+         * on whether the primary operand is in read/write form.
+         */
+        uint8_t idxReg2;
+        if (!VMXINSTRID_IS_MODRM_PRIMARY_OP_W(uInstrId))
+        {
+            idxReg2 = bRm & X86_MODRM_RM_MASK;
+            if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
+                idxReg2 |= pVCpu->iem.s.uRexB;
+        }
+        else
+        {
+            idxReg2 = (bRm >> X86_MODRM_REG_SHIFT) & X86_MODRM_REG_SMASK;
+            if (pVCpu->iem.s.enmEffAddrMode == IEMMODE_64BIT)
+                 idxReg2 |= pVCpu->iem.s.uRexReg;
+        }
+        ExitInstrInfo.All.u2Scaling      = uScale;
+        ExitInstrInfo.All.iReg1          = 0;   /* Not applicable for memory addressing. */
+        ExitInstrInfo.All.u3AddrSize     = pVCpu->iem.s.enmEffAddrMode;
+        ExitInstrInfo.All.fIsRegOperand  = 0;
+        ExitInstrInfo.All.uOperandSize   = pVCpu->iem.s.enmEffOpSize;
+        ExitInstrInfo.All.iSegReg        = pVCpu->iem.s.iEffSeg;
+        ExitInstrInfo.All.iIdxReg        = iIdxReg;
+        ExitInstrInfo.All.fIdxRegInvalid = !fIdxRegValid;
+        ExitInstrInfo.All.iBaseReg       = iBaseReg;
+        ExitInstrInfo.All.iIdxReg        = !fBaseRegValid;
+        ExitInstrInfo.All.iReg2          = idxReg2;
+    }
+
+    /*
+     * Handle exceptions to the norm for certain instructions.
+     * (e.g. some instructions convey an instruction identity in place of iReg2).
+     */
+    switch (uExitReason)
+    {
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+        {
+            Assert(VMXINSTRID_IS_VALID(uInstrId));
+            Assert(VMXINSTRID_GET_ID(uInstrId) == (uInstrId & 0x3));
+            ExitInstrInfo.GdtIdt.u2InstrId = VMXINSTRID_GET_ID(uInstrId);
+            ExitInstrInfo.GdtIdt.u2Undef0  = 0;
+            break;
+        }
+
+        case VMX_EXIT_LDTR_TR_ACCESS:
+        {
+            Assert(VMXINSTRID_IS_VALID(uInstrId));
+            Assert(VMXINSTRID_GET_ID(uInstrId) == (uInstrId & 0x3));
+            ExitInstrInfo.LdtTr.u2InstrId = VMXINSTRID_GET_ID(uInstrId);
+            ExitInstrInfo.LdtTr.u2Undef0 = 0;
+            break;
+        }
+
+        case VMX_EXIT_RDRAND:
+        case VMX_EXIT_RDSEED:
+        {
+            Assert(ExitInstrInfo.RdrandRdseed.u2OperandSize != 3);
+            break;
+        }
+    }
+
+    /* Update displacement and return the constructed VM-exit instruction information field. */
+    if (pGCPtrDisp)
+        *pGCPtrDisp = GCPtrDisp;
+
+    return ExitInstrInfo.u;
+}
+
+
+/**
+ * VMX VM-exit handler.
+ *
+ * @returns Strict VBox status code.
+ * @retval VINF_VMX_VMEXIT when the VM-exit is successful.
+ * @retval VINF_EM_TRIPLE_FAULT when VM-exit is unsuccessful and leads to a
+ *         triple-fault.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ * @param   u64ExitQual     The Exit qualification.
+ *
+ * @remarks We need not necessarily have completed VM-entry before a VM-exit is
+ *          called. Failures during VM-entry can cause VM-exits as well, so we
+ *          -cannot- assert we're in VMX non-root mode here.
+ */
+VBOXSTRICTRC iemVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual) RT_NOEXCEPT
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF3(pVCpu, uExitReason, u64ExitQual);
+    AssertMsgFailed(("VM-exit should only be invoked from ring-3 when nested-guest executes only in ring-3!\n"));
+    return VERR_IEM_IPE_7;
+# else
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    /* Just count this as an exit and be done with that. */
+    pVCpu->iem.s.cPotentialExits++;
+
+    /*
+     * Import all the guest-CPU state.
+     *
+     * HM on returning to guest execution would have to reset up a whole lot of state
+     * anyway, (e.g., VM-entry/VM-exit controls) and we do not ever import a part of
+     * the state and flag reloading the entire state on re-entry. So import the entire
+     * state here, see HMNotifyVmxNstGstVmexit() for more comments.
+     */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_ALL);
+
+    /*
+     * Ensure VM-entry interruption information valid bit is cleared.
+     *
+     * We do it here on every VM-exit so that even premature VM-exits (e.g. those caused
+     * by invalid-guest state or machine-check exceptions) also clear this bit.
+     *
+     * See Intel spec. 27.2 "Recording VM-exit Information And Updating VM-entry control fields".
+     */
+    if (VMX_ENTRY_INT_INFO_IS_VALID(pVmcs->u32EntryIntInfo))
+        pVmcs->u32EntryIntInfo &= ~VMX_ENTRY_INT_INFO_VALID;
+
+    /*
+     * Update the VM-exit reason and Exit qualification.
+     * Other VMCS read-only data fields are expected to be updated by the caller already.
+     */
+    pVmcs->u32RoExitReason = uExitReason;
+    pVmcs->u64RoExitQual.u = u64ExitQual;
+
+    Log2(("vmexit: reason=%u qual=%#RX64 cs:rip=%04x:%08RX64 cr0=%#RX64 cr3=%#RX64 cr4=%#RX64 eflags=%#RX32\n", uExitReason,
+          pVmcs->u64RoExitQual.u, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.cr0,
+          pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.eflags.u));
+
+    /*
+     * Update the IDT-vectoring information fields if the VM-exit is triggered during delivery of an event.
+     * See Intel spec. 27.2.4 "Information for VM Exits During Event Delivery".
+     */
+    {
+        uint8_t    uVector;
+        uint32_t   fFlags;
+        uint32_t   uErrCode;
+        bool const fInEventDelivery = IEMGetCurrentXcpt(pVCpu, &uVector, &fFlags, &uErrCode, NULL /* puCr2 */);
+        if (fInEventDelivery)
+        {
+            /*
+             * A VM-exit is not considered to occur during event delivery when the VM-exit is
+             * caused by a triple-fault or the original event results in a double-fault that
+             * causes the VM exit directly (exception bitmap). Therefore, we must not set the
+             * original event information into the IDT-vectoring information fields.
+             *
+             * See Intel spec. 27.2.4 "Information for VM Exits During Event Delivery".
+             */
+            if (   uExitReason != VMX_EXIT_TRIPLE_FAULT
+                && (   uExitReason != VMX_EXIT_XCPT_OR_NMI
+                    || !VMX_EXIT_INT_INFO_IS_XCPT_DF(pVmcs->u32RoExitIntInfo)))
+            {
+                uint8_t  const uIdtVectoringType = iemVmxGetEventType(uVector, fFlags);
+                uint8_t  const fErrCodeValid     = RT_BOOL(fFlags & IEM_XCPT_FLAGS_ERR);
+                uint32_t const uIdtVectoringInfo = RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_VECTOR,         uVector)
+                                                 | RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_TYPE,           uIdtVectoringType)
+                                                 | RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_ERR_CODE_VALID, fErrCodeValid)
+                                                 | RT_BF_MAKE(VMX_BF_IDT_VECTORING_INFO_VALID,          1);
+                iemVmxVmcsSetIdtVectoringInfo(pVCpu, uIdtVectoringInfo);
+                iemVmxVmcsSetIdtVectoringErrCode(pVCpu, uErrCode);
+                Log2(("vmexit: idt_info=%#RX32 idt_err_code=%#RX32 cr2=%#RX64\n", uIdtVectoringInfo, uErrCode,
+                      pVCpu->cpum.GstCtx.cr2));
+            }
+        }
+    }
+
+    /* The following VMCS fields should always be zero since we don't support injecting SMIs into a guest. */
+    Assert(pVmcs->u64RoIoRcx.u == 0);
+    Assert(pVmcs->u64RoIoRsi.u == 0);
+    Assert(pVmcs->u64RoIoRdi.u == 0);
+    Assert(pVmcs->u64RoIoRip.u == 0);
+
+    /*
+     * Save the guest state back into the VMCS.
+     * We only need to save the state when the VM-entry was successful.
+     */
+    bool const fVmentryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
+    if (!fVmentryFailed)
+    {
+        /* We should not cause an NMI-window/interrupt-window VM-exit when injecting events as part of VM-entry. */
+        if (!CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx))
+        {
+            Assert(uExitReason != VMX_EXIT_NMI_WINDOW);
+            Assert(uExitReason != VMX_EXIT_INT_WINDOW);
+        }
+
+        /* For exception or NMI VM-exits the VM-exit interruption info. field must be valid. */
+        Assert(uExitReason != VMX_EXIT_XCPT_OR_NMI || VMX_EXIT_INT_INFO_IS_VALID(pVmcs->u32RoExitIntInfo));
+
+        /*
+         * If we support storing EFER.LMA into IA32e-mode guest field on VM-exit, we need to do that now.
+         * See Intel spec. 27.2 "Recording VM-exit Information And Updating VM-entry Control".
+         *
+         * It is not clear from the Intel spec. if this is done only when VM-entry succeeds.
+         * If a VM-exit happens before loading guest EFER, we risk restoring the host EFER.LMA
+         * as guest-CPU state would not been modified. Hence for now, we do this only when
+         * the VM-entry succeeded.
+         */
+        /** @todo r=ramshankar: Figure out if this bit gets set to host EFER.LMA on real
+         *        hardware when VM-exit fails during VM-entry (e.g. VERR_VMX_INVALID_GUEST_STATE). */
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxExitSaveEferLma)
+        {
+            if (pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LMA)
+                pVmcs->u32EntryCtls |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
+            else
+                pVmcs->u32EntryCtls &= ~VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
+        }
+
+        /*
+         * The rest of the high bits of the VM-exit reason are only relevant when the VM-exit
+         * occurs in enclave mode/SMM which we don't support yet.
+         *
+         * If we ever add support for it, we can pass just the lower bits to the functions
+         * below, till then an assert should suffice.
+         */
+        Assert(!RT_HI_U16(uExitReason));
+
+        /* Save the guest state into the VMCS and restore guest MSRs from the auto-store guest MSR area. */
+        iemVmxVmexitSaveGuestState(pVCpu, uExitReason);
+        int rc = iemVmxVmexitSaveGuestAutoMsrs(pVCpu, uExitReason);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            return iemVmxAbort(pVCpu, VMXABORT_SAVE_GUEST_MSRS);
+
+        /* Clear any saved NMI-blocking state so we don't assert on next VM-entry (if it was in effect on the previous one). */
+        pVCpu->cpum.GstCtx.hwvirt.fSavedInhibit &= ~CPUMCTX_INHIBIT_NMI;
+    }
+    else
+    {
+        /* Restore the NMI-blocking state if VM-entry failed due to invalid guest state or while loading MSRs. */
+        uint32_t const uExitReasonBasic = VMX_EXIT_REASON_BASIC(uExitReason);
+        if (   uExitReasonBasic == VMX_EXIT_ERR_INVALID_GUEST_STATE
+            || uExitReasonBasic == VMX_EXIT_ERR_MSR_LOAD)
+            iemVmxVmexitRestoreNmiBlockingFF(pVCpu);
+    }
+
+    /*
+     * Stop any running VMX-preemption timer if necessary.
+     */
+    if (pVmcs->u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER)
+        CPUMStopGuestVmxPremptTimer(pVCpu);
+
+    /*
+     * Clear any pending VMX nested-guest force-flags.
+     * These force-flags have no effect on (outer) guest execution and will
+     * be re-evaluated and setup on the next nested-guest VM-entry.
+     */
+    VMCPU_FF_CLEAR_MASK(pVCpu, VMCPU_FF_VMX_ALL_MASK);
+
+    /*
+     * We're no longer in nested-guest execution mode.
+     *
+     * It is important to do this prior to loading the host state because
+     * PGM looks at fInVmxNonRootMode to determine if it needs to perform
+     * second-level address translation while switching to host CR3.
+     */
+    pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode = false;
+
+    /* Restore the host (outer guest) state. */
+    VBOXSTRICTRC rcStrict = iemVmxVmexitLoadHostState(pVCpu, uExitReason);
+    if (RT_SUCCESS(rcStrict))
+    {
+        Assert(rcStrict == VINF_SUCCESS);
+        rcStrict = VINF_VMX_VMEXIT;
+    }
+    else
+        Log(("vmexit: Loading host-state failed. uExitReason=%u rc=%Rrc\n", uExitReason, VBOXSTRICTRC_VAL(rcStrict)));
+
+    if (VM_IS_HM_ENABLED(pVCpu->CTX_SUFF(pVM)))
+    {
+        /* Notify HM that the current VMCS fields have been modified. */
+        HMNotifyVmxNstGstCurrentVmcsChanged(pVCpu);
+
+        /* Notify HM that we've completed the VM-exit. */
+        HMNotifyVmxNstGstVmexit(pVCpu);
+    }
+
+#  if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+    /* Revert any IEM-only nested-guest execution policy, otherwise return rcStrict. */
+    Log(("vmexit: Disabling IEM-only EM execution policy!\n"));
+    int rcSched = EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, false);
+    if (rcSched != VINF_SUCCESS)
+        iemSetPassUpStatus(pVCpu, rcSched);
+#  endif
+    return rcStrict;
+# endif
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to instruction execution.
+ *
+ * This is intended for instructions where the caller provides all the relevant
+ * VM-exit information.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ */
+static VBOXSTRICTRC iemVmxVmexitInstrWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    /*
+     * For instructions where any of the following fields are not applicable:
+     *   - Exit qualification must be cleared.
+     *   - VM-exit instruction info. is undefined.
+     *   - Guest-linear address is undefined.
+     *   - Guest-physical address is undefined.
+     *
+     * The VM-exit instruction length is mandatory for all VM-exits that are caused by
+     * instruction execution. For VM-exits that are not due to instruction execution this
+     * field is undefined.
+     *
+     * In our implementation in IEM, all undefined fields are generally cleared. However,
+     * if the caller supplies information (from say the physical CPU directly) it is
+     * then possible that the undefined fields are not cleared.
+     *
+     * See Intel spec. 27.2.1 "Basic VM-Exit Information".
+     * See Intel spec. 27.2.4 "Information for VM Exits Due to Instruction Execution".
+     */
+    Assert(pExitInfo);
+    AssertMsg(pExitInfo->uReason <= VMX_EXIT_MAX, ("uReason=%u\n", pExitInfo->uReason));
+    AssertMsg(pExitInfo->cbInstr >= 1 && pExitInfo->cbInstr <= 15,
+              ("uReason=%u cbInstr=%u\n", pExitInfo->uReason, pExitInfo->cbInstr));
+
+    /* Update all the relevant fields from the VM-exit instruction information struct. */
+    iemVmxVmcsSetExitInstrInfo(pVCpu, pExitInfo->InstrInfo.u);
+    iemVmxVmcsSetExitGuestLinearAddr(pVCpu, pExitInfo->u64GuestLinearAddr);
+    iemVmxVmcsSetExitGuestPhysAddr(pVCpu, pExitInfo->u64GuestPhysAddr);
+    iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr);
+
+    /* Perform the VM-exit. */
+    return iemVmxVmexit(pVCpu, pExitInfo->uReason, pExitInfo->u64Qual);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to instruction execution.
+ *
+ * This is intended for instructions that only provide the VM-exit instruction
+ * length.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ * @param   cbInstr         The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr) RT_NOEXCEPT
+{
+#ifdef VBOX_STRICT
+    /*
+     * To prevent us from shooting ourselves in the foot.
+     * The follow instructions should convey more than just the instruction length.
+     */
+    switch (uExitReason)
+    {
+        case VMX_EXIT_INVEPT:
+        case VMX_EXIT_INVPCID:
+        case VMX_EXIT_INVVPID:
+        case VMX_EXIT_LDTR_TR_ACCESS:
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+        case VMX_EXIT_VMCLEAR:
+        case VMX_EXIT_VMPTRLD:
+        case VMX_EXIT_VMPTRST:
+        case VMX_EXIT_VMREAD:
+        case VMX_EXIT_VMWRITE:
+        case VMX_EXIT_VMXON:
+        case VMX_EXIT_XRSTORS:
+        case VMX_EXIT_XSAVES:
+        case VMX_EXIT_RDRAND:
+        case VMX_EXIT_RDSEED:
+        case VMX_EXIT_IO_INSTR:
+            AssertMsgFailedReturn(("Use iemVmxVmexitInstrNeedsInfo for uExitReason=%u\n", uExitReason), VERR_IEM_IPE_5);
+            break;
+    }
+#endif
+
+    VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_INSTR_LEN(uExitReason, cbInstr);
+    return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to a triple-fault.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTripleFault(PVMCPUCC pVCpu)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_TRIPLE_FAULT, 0 /* u64ExitQual */);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to startup-IPI (SIPI).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   uVector         The SIPI vector.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitStartupIpi(PVMCPUCC pVCpu, uint8_t uVector)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, VMX_EXIT_SIPI, uVector);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a VM-exit.
+ *
+ * If a specialized version of a VM-exit handler exists, that must be used instead.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   uExitReason     The VM-exit reason.
+ * @param   u64ExitQual     The Exit qualification.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexit(PVMCPUCC pVCpu, uint32_t uExitReason, uint64_t u64ExitQual)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexit(pVCpu, uExitReason, u64ExitQual);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a VM-exit due to an instruction.
+ *
+ * This is meant to be used for those instructions that VMX provides additional
+ * decoding information beyond just the instruction length!
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo   Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitInstrWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a VM-exit due to an instruction.
+ *
+ * This is meant to be used for those instructions that VMX provides only the
+ * instruction length.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo   Pointer to the VM-exit information.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitInstr(PVMCPUCC pVCpu, uint32_t uExitReason, uint8_t cbInstr)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexitInstr(pVCpu, uExitReason, cbInstr);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to instruction execution.
+ *
+ * This is intended for instructions that have a ModR/M byte and update the VM-exit
+ * instruction information and Exit qualification fields.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ * @param   uInstrid        The instruction identity (VMXINSTRID_XXX).
+ * @param   cbInstr         The instruction length in bytes.
+ *
+ * @remarks Do not use this for INS/OUTS instruction.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrNeedsInfo(PVMCPUCC pVCpu, uint32_t uExitReason, VMXINSTRID uInstrId, uint8_t cbInstr) RT_NOEXCEPT
+{
+#ifdef VBOX_STRICT
+    /*
+    * To prevent us from shooting ourselves in the foot.
+    * The follow instructions convey specific info that require using their respective handlers.
+     */
+    switch (uExitReason)
+    {
+        case VMX_EXIT_INVEPT:
+        case VMX_EXIT_INVPCID:
+        case VMX_EXIT_INVVPID:
+        case VMX_EXIT_LDTR_TR_ACCESS:
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+        case VMX_EXIT_VMCLEAR:
+        case VMX_EXIT_VMPTRLD:
+        case VMX_EXIT_VMPTRST:
+        case VMX_EXIT_VMREAD:
+        case VMX_EXIT_VMWRITE:
+        case VMX_EXIT_VMXON:
+        case VMX_EXIT_XRSTORS:
+        case VMX_EXIT_XSAVES:
+        case VMX_EXIT_RDRAND:
+        case VMX_EXIT_RDSEED:
+            break;
+        default:
+            AssertMsgFailedReturn(("Use instruction-specific handler\n"), VERR_IEM_IPE_5);
+            break;
+    }
+#endif
+
+    /*
+     * Update the Exit qualification field with displacement bytes.
+     * See Intel spec. 27.2.1 "Basic VM-Exit Information".
+     */
+    /* Construct the VM-exit instruction information. */
+    RTGCPTR            GCPtrDisp;
+    uint32_t const     uInstrInfo = iemVmxGetExitInstrInfo(pVCpu, uExitReason, uInstrId, &GCPtrDisp);
+
+    VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO(uExitReason, GCPtrDisp, uInstrInfo, cbInstr);
+    return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to INVLPG.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   The guest-linear address of the page being invalidated.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrInvlpg(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, uint8_t cbInstr) RT_NOEXCEPT
+{
+    VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_INVLPG, GCPtrPage, cbInstr);
+    Assert(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode || !RT_HI_U32(ExitInfo.u64Qual));
+    return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to LMSW.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uGuestCr0       The current guest CR0.
+ * @param   pu16NewMsw      The machine-status word specified in LMSW's source
+ *                          operand. This will be updated depending on the VMX
+ *                          guest/host CR0 mask if LMSW is not intercepted.
+ * @param   GCPtrEffDst     The guest-linear address of the source operand in case
+ *                          of a memory operand. For register operand, pass
+ *                          NIL_RTGCPTR.
+ * @param   cbInstr         The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrLmsw(PVMCPUCC pVCpu, uint32_t uGuestCr0, uint16_t *pu16NewMsw,
+                                   RTGCPTR GCPtrEffDst, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(pu16NewMsw);
+
+    uint16_t const uNewMsw = *pu16NewMsw;
+    if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
+    {
+        Log2(("lmsw: Guest intercept -> VM-exit\n"));
+        bool const fMemOperand = RT_BOOL(GCPtrEffDst != NIL_RTGCPTR);
+        VMXVEXITINFO ExitInfo
+            = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX,
+                                                          RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER,  0) /* CR0 */
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_LMSW_OP,   fMemOperand)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_LMSW_DATA, uNewMsw)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS,  VMX_EXIT_QUAL_CRX_ACCESS_LMSW),
+                                                        cbInstr);
+        if (fMemOperand)
+        {
+            Assert(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode || !RT_HI_U32(GCPtrEffDst));
+            ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
+        }
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    /*
+     * If LMSW did not cause a VM-exit, any CR0 bits in the range 0:3 that is set in the
+     * CR0 guest/host mask must be left unmodified.
+     *
+     * See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation".
+     */
+    uint32_t const fGstHostMask     = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u;
+    uint32_t const fGstHostLmswMask = fGstHostMask & (X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS);
+    *pu16NewMsw = (uGuestCr0 & fGstHostLmswMask) | (uNewMsw & ~fGstHostLmswMask);
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to CLTS.
+ *
+ * @returns Strict VBox status code.
+ * @retval VINF_VMX_MODIFIES_BEHAVIOR if the CLTS instruction did not cause a
+ *         VM-exit but must not modify the guest CR0.TS bit.
+ * @retval VINF_VMX_INTERCEPT_NOT_ACTIVE if the CLTS instruction did not cause a
+ *         VM-exit and modification to the guest CR0.TS bit is allowed (subject to
+ *         CR0 fixed bits in VMX operation).
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrClts(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
+{
+    /*
+     * If CR0.TS is owned by the host:
+     *   - If CR0.TS is set in the read-shadow, we must cause a VM-exit.
+     *   - If CR0.TS is cleared in the read-shadow, no VM-exit is caused and the
+     *     CLTS instruction completes without clearing CR0.TS.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    uint32_t const fGstHostMask = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u;
+    if (fGstHostMask & X86_CR0_TS)
+    {
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0ReadShadow.u & X86_CR0_TS)
+        {
+            Log2(("clts: Guest intercept -> VM-exit\n"));
+            VMXVEXITINFO const ExitInfo
+                = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX,
+                                                              RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 0) /* CR0 */
+                                                            | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS,
+                                                                         VMX_EXIT_QUAL_CRX_ACCESS_CLTS),
+                                                            cbInstr);
+            return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+        }
+        return VINF_VMX_MODIFIES_BEHAVIOR;
+    }
+
+    /*
+     * If CR0.TS is not owned by the host, the CLTS instructions operates normally
+     * and may modify CR0.TS (subject to CR0 fixed bits in VMX operation).
+     */
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov CR0,GReg' and 'Mov CR4,GReg'
+ * (CR0/CR4 write).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iCrReg      The control register (either CR0 or CR4).
+ * @param   uGuestCrX   The current guest CR0/CR4.
+ * @param   puNewCrX    Pointer to the new CR0/CR4 value. Will be updated if no
+ *                      VM-exit is caused.
+ * @param   iGReg       The general register from which the CR0/CR4 value is being
+ *                      loaded.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrMovToCr0Cr4(PVMCPUCC pVCpu, uint8_t iCrReg, uint64_t *puNewCrX,
+                                          uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(puNewCrX);
+    Assert(iCrReg == 0 || iCrReg == 4);
+    Assert(iGReg < X86_GREG_COUNT);
+
+    uint64_t const uNewCrX = *puNewCrX;
+    if (CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX))
+    {
+        Log2(("mov_Cr_Rd: (CR%u) Guest intercept -> VM-exit\n", iCrReg));
+        VMXVEXITINFO const ExitInfo
+            = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX,
+                                                          RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, iCrReg)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG,   iGReg)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, VMX_EXIT_QUAL_CRX_ACCESS_WRITE),
+                                                        cbInstr);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    /*
+     * If the Mov-to-CR0/CR4 did not cause a VM-exit, any bits owned by the host
+     * must not be modified the instruction.
+     *
+     * See Intel Spec. 25.3 "Changes To Instruction Behavior In VMX Non-root Operation".
+     */
+    uint64_t uGuestCrX;
+    uint64_t fGstHostMask;
+    if (iCrReg == 0)
+    {
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+        uGuestCrX    = pVCpu->cpum.GstCtx.cr0;
+        fGstHostMask = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0Mask.u;
+    }
+    else
+    {
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+        uGuestCrX    = pVCpu->cpum.GstCtx.cr4;
+        fGstHostMask = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr4Mask.u;
+    }
+
+    *puNewCrX = (uGuestCrX & fGstHostMask) | (*puNewCrX & ~fGstHostMask);
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov GReg,CR3' (CR3 read).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iGReg       The general register to which the CR3 value is being stored.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrMovFromCr3(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(iGReg < X86_GREG_COUNT);
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+
+    /*
+     * If the CR3-store exiting control is set, we must cause a VM-exit.
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_CR3_STORE_EXIT)
+    {
+        Log2(("mov_Rd_Cr: (CR3) Guest intercept -> VM-exit\n"));
+        VMXVEXITINFO const ExitInfo
+            = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX,
+                                                          RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 3) /* CR3 */
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG,   iGReg)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, VMX_EXIT_QUAL_CRX_ACCESS_READ),
+                                                        cbInstr);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov CR3,GReg' (CR3 write).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uNewCr3     The new CR3 value.
+ * @param   iGReg       The general register from which the CR3 value is being
+ *                      loaded.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrMovToCr3(PVMCPUCC pVCpu, uint64_t uNewCr3, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(iGReg < X86_GREG_COUNT);
+
+    /*
+     * If the CR3-load exiting control is set and the new CR3 value does not
+     * match any of the CR3-target values in the VMCS, we must cause a VM-exit.
+     *
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCr3))
+    {
+        Log2(("mov_Cr_Rd: (CR3) Guest intercept -> VM-exit\n"));
+        VMXVEXITINFO const ExitInfo
+            = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX,
+                                                           RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 3) /* CR3 */
+                                                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG,   iGReg)
+                                                         | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS,
+                                                                      VMX_EXIT_QUAL_CRX_ACCESS_WRITE),
+                                                         cbInstr);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov GReg,CR8' (CR8 read).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iGReg       The general register to which the CR8 value is being stored.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrMovFromCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(iGReg < X86_GREG_COUNT);
+
+    /*
+     * If the CR8-store exiting control is set, we must cause a VM-exit.
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_CR8_STORE_EXIT)
+    {
+        Log2(("mov_Rd_Cr: (CR8) Guest intercept -> VM-exit\n"));
+        VMXVEXITINFO const ExitInfo
+            = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX,
+                                                          RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 8) /* CR8 */
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG,   iGReg)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, VMX_EXIT_QUAL_CRX_ACCESS_READ),
+                                                        cbInstr);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov CR8,GReg' (CR8 write).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iGReg       The general register from which the CR8 value is being
+ *                      loaded.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrMovToCr8(PVMCPUCC pVCpu, uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(iGReg < X86_GREG_COUNT);
+
+    /*
+     * If the CR8-load exiting control is set, we must cause a VM-exit.
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_CR8_LOAD_EXIT)
+    {
+        Log2(("mov_Cr_Rd: (CR8) Guest intercept -> VM-exit\n"));
+        VMXVEXITINFO const ExitInfo
+            = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_CRX,
+                                                          RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_REGISTER, 8) /* CR8 */
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_GENREG,   iGReg)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_CRX_ACCESS, VMX_EXIT_QUAL_CRX_ACCESS_WRITE),
+                                                        cbInstr);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to 'Mov DRx,GReg' (DRx write) and 'Mov
+ * GReg,DRx' (DRx read).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uInstrid    The instruction identity (VMXINSTRID_MOV_TO_DRX or
+ *                      VMXINSTRID_MOV_FROM_DRX).
+ * @param   iDrReg      The debug register being accessed.
+ * @param   iGReg       The general register to/from which the DRx value is being
+ *                      store/loaded.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrMovDrX(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint8_t iDrReg,
+                                     uint8_t iGReg, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(iDrReg <= 7);
+    Assert(uInstrId == VMXINSTRID_MOV_TO_DRX || uInstrId == VMXINSTRID_MOV_FROM_DRX);
+    Assert(iGReg < X86_GREG_COUNT);
+
+    if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT)
+    {
+        VMXVEXITINFO const ExitInfo
+            = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MOV_DRX,
+                                                          RT_BF_MAKE(VMX_BF_EXIT_QUAL_DRX_REGISTER,  iDrReg)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_DRX_GENREG,    iGReg)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_DRX_DIRECTION,
+                                                                     uInstrId == VMXINSTRID_MOV_TO_DRX
+                                                                     ? VMX_EXIT_QUAL_DRX_DIRECTION_WRITE
+                                                                     : VMX_EXIT_QUAL_DRX_DIRECTION_READ),
+                                                        cbInstr);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to I/O instructions (IN and OUT).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uInstrId    The VM-exit instruction identity (VMXINSTRID_IO_IN or
+ *                      VMXINSTRID_IO_OUT).
+ * @param   u16Port     The I/O port being accessed.
+ * @param   fImm        Whether the I/O port was encoded using an immediate operand
+ *                      or the implicit DX register.
+ * @param   cbAccess    The size of the I/O access in bytes (1, 2 or 4 bytes).
+ * @param   cbInstr     The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port,
+                                 bool fImm, uint8_t cbAccess, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(uInstrId == VMXINSTRID_IO_IN || uInstrId == VMXINSTRID_IO_OUT);
+    Assert(cbAccess == 1 || cbAccess == 2 || cbAccess == 4);
+
+    if (CPUMIsGuestVmxIoInterceptSet(pVCpu, u16Port, cbAccess))
+    {
+        VMXVEXITINFO const ExitInfo
+            = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_IO_INSTR,
+                                                          RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_WIDTH,     cbAccess - 1)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_ENCODING,  fImm)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_PORT,      u16Port)
+                                                        | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_DIRECTION,
+                                                                     uInstrId == VMXINSTRID_IO_IN
+                                                                     ? VMX_EXIT_QUAL_IO_DIRECTION_IN
+                                                                     : VMX_EXIT_QUAL_IO_DIRECTION_OUT),
+                                                        cbInstr);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to string I/O instructions (INS and OUTS).
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uInstrId        The VM-exit instruction identity (VMXINSTRID_IO_INS or
+ *                          VMXINSTRID_IO_OUTS).
+ * @param   u16Port         The I/O port being accessed.
+ * @param   cbAccess        The size of the I/O access in bytes (1, 2 or 4 bytes).
+ * @param   fRep            Whether the instruction has a REP prefix or not.
+ * @param   ExitInstrInfo   The VM-exit instruction info. field.
+ * @param   cbInstr         The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrStrIo(PVMCPUCC pVCpu, VMXINSTRID uInstrId, uint16_t u16Port, uint8_t cbAccess,
+                                    bool fRep, VMXEXITINSTRINFO ExitInstrInfo, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(uInstrId == VMXINSTRID_IO_INS || uInstrId == VMXINSTRID_IO_OUTS);
+    Assert(cbAccess == 1 || cbAccess == 2 || cbAccess == 4);
+    Assert(ExitInstrInfo.StrIo.iSegReg < X86_SREG_COUNT);
+    Assert(ExitInstrInfo.StrIo.u3AddrSize == 0 || ExitInstrInfo.StrIo.u3AddrSize == 1 || ExitInstrInfo.StrIo.u3AddrSize == 2);
+    Assert(uInstrId != VMXINSTRID_IO_INS || ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES);
+
+    if (CPUMIsGuestVmxIoInterceptSet(pVCpu, u16Port, cbAccess))
+    {
+        /*
+         * Figure out the guest-linear address and the direction bit (INS/OUTS).
+         */
+        /** @todo r=ramshankar: Is there something in IEM that already does this? */
+        static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
+        uint8_t const  iSegReg       = ExitInstrInfo.StrIo.iSegReg;
+        uint8_t const  uAddrSize     = ExitInstrInfo.StrIo.u3AddrSize;
+        uint64_t const uAddrSizeMask = s_auAddrSizeMasks[uAddrSize];
+
+        uint32_t uDirection;
+        uint64_t uGuestLinearAddr;
+        if (uInstrId == VMXINSTRID_IO_INS)
+        {
+            uDirection       = VMX_EXIT_QUAL_IO_DIRECTION_IN;
+            uGuestLinearAddr = pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base + (pVCpu->cpum.GstCtx.rdi & uAddrSizeMask);
+        }
+        else
+        {
+            uDirection       = VMX_EXIT_QUAL_IO_DIRECTION_OUT;
+            uGuestLinearAddr = pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base + (pVCpu->cpum.GstCtx.rsi & uAddrSizeMask);
+        }
+
+        /*
+         * If the segment is unusable, the guest-linear address in undefined.
+         * We shall clear it for consistency.
+         *
+         * See Intel spec. 27.2.1 "Basic VM-Exit Information".
+         */
+        if (pVCpu->cpum.GstCtx.aSRegs[iSegReg].Attr.n.u1Unusable)
+            uGuestLinearAddr = 0;
+
+        VMXVEXITINFO const ExitInfo
+            = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_AND_LIN_ADDR(VMX_EXIT_IO_INSTR,
+                                                                        RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_WIDTH,   cbAccess - 1)
+                                                                      | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_DIRECTION, uDirection)
+                                                                      | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_IS_STRING, 1)
+                                                                      | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_IS_REP,    fRep)
+                                                                      | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_ENCODING,
+                                                                                   VMX_EXIT_QUAL_IO_ENCODING_DX)
+                                                                      | RT_BF_MAKE(VMX_BF_EXIT_QUAL_IO_PORT,      u16Port),
+                                                                      IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxInsOutInfo
+                                                                      ? ExitInstrInfo.u : 0,
+                                                                      cbInstr,
+                                                                      uGuestLinearAddr);
+        return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to MWAIT.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   fMonitorHwArmed     Whether the address-range monitor hardware is armed.
+ * @param   cbInstr             The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitInstrMwait(PVMCPUCC pVCpu, bool fMonitorHwArmed, uint8_t cbInstr) RT_NOEXCEPT
+{
+    VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_MWAIT, fMonitorHwArmed, cbInstr);
+    return iemVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to PAUSE.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+static VBOXSTRICTRC iemVmxVmexitInstrPause(PVMCPUCC pVCpu, uint8_t cbInstr) RT_NOEXCEPT
+{
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    /*
+     * The PAUSE VM-exit is controlled by the "PAUSE exiting" control and the
+     * "PAUSE-loop exiting" control.
+     *
+     * The PLE-Gap is the maximum number of TSC ticks between two successive executions of
+     * the PAUSE instruction before we cause a VM-exit. The PLE-Window is the maximum amount
+     * of TSC ticks the guest is allowed to execute in a pause loop before we must cause
+     * a VM-exit.
+     *
+     * See Intel spec. 24.6.13 "Controls for PAUSE-Loop Exiting".
+     * See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    bool fIntercept = false;
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
+        fIntercept = true;
+    else if (   (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT)
+             && pVCpu->iem.s.uCpl == 0)
+    {
+        IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+
+        /*
+         * A previous-PAUSE-tick value of 0 is used to identify the first time
+         * execution of a PAUSE instruction after VM-entry at CPL 0. We must
+         * consider this to be the first execution of PAUSE in a loop according
+         * to the Intel.
+         *
+         * All subsequent records for the previous-PAUSE-tick we ensure that it
+         * cannot be zero by OR'ing 1 to rule out the TSC wrap-around cases at 0.
+         */
+        uint64_t *puFirstPauseLoopTick = &pVCpu->cpum.GstCtx.hwvirt.vmx.uFirstPauseLoopTick;
+        uint64_t *puPrevPauseTick      = &pVCpu->cpum.GstCtx.hwvirt.vmx.uPrevPauseTick;
+        uint64_t const  uTick          = TMCpuTickGet(pVCpu);
+        uint32_t const  uPleGap        = pVmcs->u32PleGap;
+        uint32_t const  uPleWindow     = pVmcs->u32PleWindow;
+        if (   *puPrevPauseTick == 0
+            || uTick - *puPrevPauseTick > uPleGap)
+            *puFirstPauseLoopTick = uTick;
+        else if (uTick - *puFirstPauseLoopTick > uPleWindow)
+            fIntercept = true;
+
+        *puPrevPauseTick = uTick | 1;
+    }
+
+    if (fIntercept)
+        return iemVmxVmexitInstr(pVCpu, VMX_EXIT_PAUSE, cbInstr);
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to task switches.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmTaskSwitch   The cause of the task switch.
+ * @param   SelNewTss       The selector of the new TSS.
+ * @param   cbInstr         The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitTaskSwitch(PVMCPUCC pVCpu, IEMTASKSWITCH enmTaskSwitch, RTSEL SelNewTss, uint8_t cbInstr) RT_NOEXCEPT
+{
+    /*
+     * Task-switch VM-exits are unconditional and provide the Exit qualification.
+     *
+     * If the cause of the task switch is due to execution of CALL, IRET or the JMP
+     * instruction or delivery of the exception generated by one of these instructions
+     * lead to a task switch through a task gate in the IDT, we need to provide the
+     * VM-exit instruction length. Any other means of invoking a task switch VM-exit
+     * leaves the VM-exit instruction length field undefined.
+     *
+     * See Intel spec. 25.2 "Other Causes Of VM Exits".
+     * See Intel spec. 27.2.4 "Information for VM Exits Due to Instruction Execution".
+     */
+    Assert(cbInstr <= 15);
+
+    uint8_t uType;
+    switch (enmTaskSwitch)
+    {
+        case IEMTASKSWITCH_CALL:        uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_CALL; break;
+        case IEMTASKSWITCH_IRET:        uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IRET; break;
+        case IEMTASKSWITCH_JUMP:        uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_JMP;  break;
+        case IEMTASKSWITCH_INT_XCPT:    uType = VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT;  break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    uint64_t const u64ExitQual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_TASK_SWITCH_NEW_TSS, SelNewTss)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_TASK_SWITCH_SOURCE,  uType);
+    iemVmxVmcsSetExitInstrLen(pVCpu, cbInstr);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_TASK_SWITCH, u64ExitQual);
+}
+
+
+/**
+ * VMX VM-exit handler for trap-like VM-exits.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ */
+static VBOXSTRICTRC iemVmxVmexitTrapLikeWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    Assert(VMXIsVmexitTrapLike(pExitInfo->uReason));
+    iemVmxVmcsSetGuestPendingDbgXcpts(pVCpu, pExitInfo->u64GuestPendingDbgXcpts);
+    return iemVmxVmexit(pVCpu, pExitInfo->uReason, pExitInfo->u64Qual);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a trap-like VM-exit (MTF, APIC-write,
+ * Virtualized-EOI, TPR-below threshold).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTrapLike(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(pExitInfo);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitTrapLikeWithInfo(pVCpu, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to task switches.
+ *
+ * This is intended for task switches where the caller provides all the relevant
+ * VM-exit information.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ */
+static VBOXSTRICTRC iemVmxVmexitTaskSwitchWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo,
+                                                   PCVMXVEXITEVENTINFO pExitEventInfo) RT_NOEXCEPT
+{
+    Assert(pExitInfo->uReason == VMX_EXIT_TASK_SWITCH);
+    iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr);
+    iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo);
+    iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_TASK_SWITCH, pExitInfo->u64Qual);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a VM-exit due to a task switch.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitTaskSwitch(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    Assert(pExitInfo);
+    Assert(pExitEventInfo);
+    Assert(pExitInfo->uReason == VMX_EXIT_TASK_SWITCH);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitTaskSwitchWithInfo(pVCpu, pExitInfo, pExitEventInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to expiring of the preemption timer.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VBOXSTRICTRC iemVmxVmexitPreemptTimer(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER);
+
+    /* Import the hardware virtualization state (for nested-guest VM-entry TSC-tick). */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+
+    /* Save the VMX-preemption timer value (of 0) back in to the VMCS if the CPU supports this feature. */
+    if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ExitCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER)
+        pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32PreemptTimer = 0;
+
+    /* Cause the VMX-preemption timer VM-exit. The Exit qualification MBZ. */
+    return iemVmxVmexit(pVCpu, VMX_EXIT_PREEMPT_TIMER, 0 /* u64ExitQual */);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to expiry of the preemption timer.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitPreemptTimer(PVMCPUCC pVCpu)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexitPreemptTimer(pVCpu);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to external interrupts.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uVector         The external interrupt vector (pass 0 if the interrupt
+ *                          is still pending since we typically won't know the
+ *                          vector).
+ * @param   fIntPending     Whether the external interrupt is pending or
+ *                          acknowledged in the interrupt controller.
+ */
+static VBOXSTRICTRC iemVmxVmexitExtInt(PVMCPUCC pVCpu, uint8_t uVector, bool fIntPending) RT_NOEXCEPT
+{
+    Assert(!fIntPending || uVector == 0);
+
+    /* The VM-exit is subject to "External interrupt exiting" being set. */
+    if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32PinCtls & VMX_PIN_CTLS_EXT_INT_EXIT)
+    {
+        if (fIntPending)
+        {
+            /*
+             * If the interrupt is pending and we don't need to acknowledge the
+             * interrupt on VM-exit, cause the VM-exit immediately.
+             *
+             * See Intel spec 25.2 "Other Causes Of VM Exits".
+             */
+            if (!(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT))
+                return iemVmxVmexit(pVCpu, VMX_EXIT_EXT_INT, 0 /* u64ExitQual */);
+
+            /*
+             * If the interrupt is pending and we -do- need to acknowledge the interrupt
+             * on VM-exit, postpone VM-exit till after the interrupt controller has been
+             * acknowledged that the interrupt has been consumed. Callers would have to call
+             * us again after getting the vector (and ofc, with fIntPending with false).
+             */
+            return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+        }
+
+        /*
+         * If the interrupt is no longer pending (i.e. it has been acknowledged) and the
+         * "External interrupt exiting" and "Acknowledge interrupt on VM-exit" controls are
+         * all set, we need to record the vector of the external interrupt in the
+         * VM-exit interruption information field. Otherwise, mark this field as invalid.
+         *
+         * See Intel spec. 27.2.2 "Information for VM Exits Due to Vectored Events".
+         */
+        uint32_t uExitIntInfo;
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
+        {
+            bool const fNmiUnblocking = pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret;
+            uExitIntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR,           uVector)
+                         | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE,             VMX_EXIT_INT_INFO_TYPE_EXT_INT)
+                         | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_NMI_UNBLOCK_IRET, fNmiUnblocking)
+                         | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID,            1);
+        }
+        else
+            uExitIntInfo = 0;
+        iemVmxVmcsSetExitIntInfo(pVCpu, uExitIntInfo);
+
+        /*
+         * Cause the VM-exit whether or not the vector has been stored
+         * in the VM-exit interruption-information field.
+         */
+        return iemVmxVmexit(pVCpu, VMX_EXIT_EXT_INT, 0 /* u64ExitQual */);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to external interrupts.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   uVector         The external interrupt vector (pass 0 if the external
+ *                          interrupt is still pending).
+ * @param   fIntPending     Whether the external interrupt is pending or
+ *                          acknowdledged in the interrupt controller.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitExtInt(PVMCPUCC pVCpu, uint8_t uVector, bool fIntPending)
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmexitExtInt(pVCpu, uVector, fIntPending);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to a double fault caused during delivery of
+ * an event.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VBOXSTRICTRC iemVmxVmexitEventDoubleFault(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    uint32_t const fXcptBitmap = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap;
+    if (fXcptBitmap & RT_BIT(X86_XCPT_DF))
+    {
+        /*
+         * The NMI-unblocking due to IRET field need not be set for double faults.
+         * See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
+         */
+        uint32_t const uExitIntInfo   = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR,           X86_XCPT_DF)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE,             VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_ERR_CODE_VALID,   1)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_NMI_UNBLOCK_IRET, 0)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID,            1);
+        iemVmxVmcsSetExitIntInfo(pVCpu, uExitIntInfo);
+        return iemVmxVmexit(pVCpu, VMX_EXIT_XCPT_OR_NMI, 0 /* u64ExitQual */);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exit due to delivery of an events.
+ *
+ * This is intended for VM-exit due to exceptions or NMIs where the caller provides
+ * all the relevant VM-exit information.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ */
+static VBOXSTRICTRC iemVmxVmexitEventWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo) RT_NOEXCEPT
+{
+    Assert(pExitInfo);
+    Assert(pExitEventInfo);
+    Assert(pExitInfo->uReason == VMX_EXIT_XCPT_OR_NMI);
+    Assert(VMX_EXIT_INT_INFO_IS_VALID(pExitEventInfo->uExitIntInfo));
+
+    iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr);
+    iemVmxVmcsSetExitIntInfo(pVCpu, pExitEventInfo->uExitIntInfo);
+    iemVmxVmcsSetExitIntErrCode(pVCpu, pExitEventInfo->uExitIntErrCode);
+    iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo);
+    iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_XCPT_OR_NMI, pExitInfo->u64Qual);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to NMIs.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitXcptNmi(PVMCPUCC pVCpu)
+{
+    VMXVEXITINFO const      ExitInfo      = VMXVEXITINFO_INIT_ONLY_REASON(VMX_EXIT_XCPT_OR_NMI);
+    VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_INT(  RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID, 1)
+                                                                            | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE,
+                                                                                         VMX_EXIT_INT_INFO_TYPE_NMI)
+                                                                            | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR,
+                                                                                         X86_XCPT_NMI),
+                                                                            0);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitEventWithInfo(pVCpu, &ExitInfo, &ExitEventInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Interface for HM and EM to emulate VM-exit due to exceptions.
+ *
+ * Exception includes NMIs, software exceptions (those generated by INT3 or
+ * INTO) and privileged software exceptions (those generated by INT1/ICEBP).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitXcpt(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    Assert(pExitInfo);
+    Assert(pExitEventInfo);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitEventWithInfo(pVCpu, pExitInfo, pExitEventInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMX VM-exit handler for VM-exits due to delivery of an event.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uVector     The interrupt / exception vector.
+ * @param   fFlags      The flags (see IEM_XCPT_FLAGS_XXX).
+ * @param   uErrCode    The error code associated with the event.
+ * @param   uCr2        The CR2 value in case of a \#PF exception.
+ * @param   cbInstr     The instruction length in bytes.
+ */
+VBOXSTRICTRC iemVmxVmexitEvent(PVMCPUCC pVCpu, uint8_t uVector, uint32_t fFlags, uint32_t uErrCode,
+                               uint64_t uCr2, uint8_t cbInstr) RT_NOEXCEPT
+{
+    /*
+     * If the event is being injected as part of VM-entry, it is -not- subject to event
+     * intercepts in the nested-guest. However, secondary exceptions that occur during
+     * injection of any event -are- subject to event interception.
+     *
+     * See Intel spec. 26.5.1.2 "VM Exits During Event Injection".
+     */
+    if (!CPUMIsGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx))
+    {
+        /*
+         * If the event is a virtual-NMI (which is an NMI being inject during VM-entry)
+         * virtual-NMI blocking must be set in effect rather than physical NMI blocking.
+         *
+         * See Intel spec. 24.6.1 "Pin-Based VM-Execution Controls".
+         */
+        if (   uVector == X86_XCPT_NMI
+            && (fFlags & IEM_XCPT_FLAGS_T_CPU_XCPT)
+            && (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
+            pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = true;
+        else
+            Assert(!pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking);
+
+        CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, true);
+        return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+    }
+
+    /*
+     * We are injecting an external interrupt, check if we need to cause a VM-exit now.
+     * If not, the caller will continue delivery of the external interrupt as it would
+     * normally. The interrupt is no longer pending in the interrupt controller at this
+     * point.
+     */
+    if (fFlags & IEM_XCPT_FLAGS_T_EXT_INT)
+    {
+        Assert(!VMX_IDT_VECTORING_INFO_IS_VALID(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32RoIdtVectoringInfo));
+        return iemVmxVmexitExtInt(pVCpu, uVector, false /* fIntPending */);
+    }
+
+    /*
+     * Evaluate intercepts for hardware exceptions, software exceptions (#BP, #OF),
+     * and privileged software exceptions (#DB generated by INT1/ICEBP) and software
+     * interrupts.
+     */
+    Assert(fFlags & (IEM_XCPT_FLAGS_T_CPU_XCPT | IEM_XCPT_FLAGS_T_SOFT_INT));
+    bool fIntercept;
+    if (   !(fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+        ||  (fFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_OF_INSTR | IEM_XCPT_FLAGS_ICEBP_INSTR)))
+        fIntercept = CPUMIsGuestVmxXcptInterceptSet(&pVCpu->cpum.GstCtx, uVector, uErrCode);
+    else
+    {
+        /* Software interrupts cannot be intercepted and therefore do not cause a VM-exit. */
+        fIntercept = false;
+    }
+
+    /*
+     * Now that we've determined whether the event causes a VM-exit, we need to construct the
+     * relevant VM-exit information and cause the VM-exit.
+     */
+    if (fIntercept)
+    {
+        Assert(!(fFlags & IEM_XCPT_FLAGS_T_EXT_INT));
+
+        /* Construct the rest of the event related information fields and cause the VM-exit. */
+        uint64_t u64ExitQual;
+        if (uVector == X86_XCPT_PF)
+        {
+            Assert(fFlags & IEM_XCPT_FLAGS_CR2);
+            u64ExitQual = uCr2;
+        }
+        else if (uVector == X86_XCPT_DB)
+        {
+            IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_DR6);
+            u64ExitQual = pVCpu->cpum.GstCtx.dr[6] & VMX_VMCS_EXIT_QUAL_VALID_MASK;
+        }
+        else
+            u64ExitQual = 0;
+
+        uint8_t  const fNmiUnblocking = pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret;
+        bool     const fErrCodeValid  = RT_BOOL(fFlags & IEM_XCPT_FLAGS_ERR);
+        uint8_t  const uIntInfoType   = iemVmxGetEventType(uVector, fFlags);
+        uint32_t const uExitIntInfo   = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR,           uVector)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_TYPE,             uIntInfoType)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_ERR_CODE_VALID,   fErrCodeValid)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_NMI_UNBLOCK_IRET, fNmiUnblocking)
+                                      | RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VALID,            1);
+        iemVmxVmcsSetExitIntInfo(pVCpu, uExitIntInfo);
+        iemVmxVmcsSetExitIntErrCode(pVCpu, uErrCode);
+
+        /*
+         * For VM-exits due to software exceptions (those generated by INT3 or INTO) or privileged
+         * software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
+         * length.
+         */
+        if (   (fFlags & IEM_XCPT_FLAGS_T_SOFT_INT)
+            || (fFlags & (IEM_XCPT_FLAGS_BP_INSTR | IEM_XCPT_FLAGS_OF_INSTR | IEM_XCPT_FLAGS_ICEBP_INSTR)))
+            iemVmxVmcsSetExitInstrLen(pVCpu, cbInstr);
+        else
+            iemVmxVmcsSetExitInstrLen(pVCpu, 0);
+
+        return iemVmxVmexit(pVCpu, VMX_EXIT_XCPT_OR_NMI, u64ExitQual);
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * VMX VM-exit handler for EPT misconfiguration.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhysAddr      The physical address causing the EPT misconfiguration.
+ *                          This need not be page aligned (e.g. nested-guest in real
+ *                          mode).
+ */
+static VBOXSTRICTRC iemVmxVmexitEptMisconfig(PVMCPUCC pVCpu, RTGCPHYS GCPhysAddr) RT_NOEXCEPT
+{
+    iemVmxVmcsSetExitGuestPhysAddr(pVCpu, GCPhysAddr);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_EPT_MISCONFIG, 0 /* u64ExitQual */);
+}
+
+
+/**
+ * VMX VM-exit handler for EPT misconfiguration.
+ *
+ * This is intended for EPT misconfigurations where the caller provides all the
+ * relevant VM-exit information.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhysAddr      The physical address causing the EPT misconfiguration.
+ *                          This need not be page aligned (e.g. nested-guest in real
+ *                          mode).
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ */
+static VBOXSTRICTRC iemVmxVmexitEptMisconfigWithInfo(PVMCPUCC pVCpu, RTGCPHYS GCPhysAddr, PCVMXVEXITEVENTINFO pExitEventInfo) RT_NOEXCEPT
+{
+    Assert(pExitEventInfo);
+    Assert(!VMX_EXIT_INT_INFO_IS_VALID(pExitEventInfo->uExitIntInfo));
+    iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo);
+    iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode);
+    iemVmxVmcsSetExitGuestPhysAddr(pVCpu, GCPhysAddr);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_EPT_MISCONFIG, 0 /* u64ExitQual */);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a VM-exit due to an EPT misconfiguration.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhysAddr      The nested-guest physical address causing the EPT
+ *                          misconfiguration.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitEptMisconfig(PVMCPUCC pVCpu, RTGCPHYS GCPhysAddr, PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitEptMisconfigWithInfo(pVCpu, GCPhysAddr, pExitEventInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMX VM-exit handler for EPT violation.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   fAccess             The access causing the EPT violation, IEM_ACCESS_XXX.
+ * @param   fSlatFail           The SLAT failure info, IEM_SLAT_FAIL_XXX.
+ * @param   fEptAccess          The EPT paging structure bits.
+ * @param   GCPhysAddr          The physical address causing the EPT violation. This
+ *                              need not be page aligned (e.g. nested-guest in real
+ *                              mode).
+ * @param   fIsLinearAddrValid  Whether translation of a linear address caused this
+ *                              EPT violation. If @c false, GCPtrAddr must be 0.
+ * @param   GCPtrAddr           The linear address causing the EPT violation.
+ * @param   cbInstr             The VM-exit instruction length.
+ */
+static VBOXSTRICTRC iemVmxVmexitEptViolation(PVMCPUCC pVCpu, uint32_t fAccess, uint32_t fSlatFail,
+                                             uint64_t fEptAccess, RTGCPHYS GCPhysAddr, bool fIsLinearAddrValid,
+                                             uint64_t GCPtrAddr, uint8_t cbInstr) RT_NOEXCEPT
+{
+    /*
+     * If the linear address isn't valid (can happen when loading PDPTEs
+     * as part of MOV CR execution) the linear address field is undefined.
+     * While we can leave it this way, it's preferrable to zero it for consistency.
+     */
+    Assert(fIsLinearAddrValid || GCPtrAddr == 0);
+
+    uint64_t const fCaps = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64EptVpidCaps;
+    bool const fSupportsAccessDirty = RT_BOOL(fCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY);
+
+    uint32_t const fDataRdMask     = IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_READ;
+    uint32_t const fDataWrMask     = IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_WRITE;
+    uint32_t const fInstrMask      = IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_EXEC;
+    bool const fDataRead           = ((fAccess & fDataRdMask) == IEM_ACCESS_DATA_R) | fSupportsAccessDirty;
+    bool const fDataWrite          = ((fAccess & fDataWrMask) == IEM_ACCESS_DATA_W) | fSupportsAccessDirty;
+    bool const fInstrFetch         = ((fAccess & fInstrMask)  == IEM_ACCESS_INSTRUCTION);
+    bool const fEptRead            = RT_BOOL(fEptAccess & EPT_E_READ);
+    bool const fEptWrite           = RT_BOOL(fEptAccess & EPT_E_WRITE);
+    bool const fEptExec            = RT_BOOL(fEptAccess & EPT_E_EXECUTE);
+    bool const fNmiUnblocking      = pVCpu->cpum.GstCtx.hwvirt.vmx.fNmiUnblockingIret;
+    bool const fIsLinearToPhysAddr = fIsLinearAddrValid & RT_BOOL(fSlatFail & IEM_SLAT_FAIL_LINEAR_TO_PHYS_ADDR);
+
+    uint64_t const u64ExitQual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ACCESS_READ,         fDataRead)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ACCESS_WRITE,        fDataWrite)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH,  fInstrFetch)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ENTRY_READ,          fEptRead)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ENTRY_WRITE,         fEptWrite)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_ENTRY_EXECUTE,       fEptExec)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_LINEAR_ADDR_VALID,   fIsLinearAddrValid)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_LINEAR_TO_PHYS_ADDR, fIsLinearToPhysAddr)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_EPT_NMI_UNBLOCK_IRET,    fNmiUnblocking);
+
+#ifdef VBOX_STRICT
+    uint64_t const fMiscCaps  = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Misc;
+    uint32_t const fProcCtls2 = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2;
+    Assert(!(fCaps & MSR_IA32_VMX_EPT_VPID_CAP_ADVEXITINFO_EPT_VIOLATION));   /* Advanced VM-exit info. not supported */
+    Assert(!(fCaps & MSR_IA32_VMX_EPT_VPID_CAP_SUPER_SHW_STACK));             /* Supervisor shadow stack control not supported. */
+    Assert(!(RT_BF_GET(fMiscCaps, VMX_BF_MISC_INTEL_PT)));                    /* Intel PT not supported. */
+    Assert(!(fProcCtls2 & VMX_PROC_CTLS2_MODE_BASED_EPT_PERM));               /* Mode-based execute control not supported. */
+#endif
+
+    iemVmxVmcsSetExitGuestPhysAddr(pVCpu, GCPhysAddr);
+    iemVmxVmcsSetExitGuestLinearAddr(pVCpu, GCPtrAddr);
+    iemVmxVmcsSetExitInstrLen(pVCpu, cbInstr);
+
+    return iemVmxVmexit(pVCpu, VMX_EXIT_EPT_VIOLATION, u64ExitQual);
+}
+
+
+/**
+ * VMX VM-exit handler for EPT violation.
+ *
+ * This is intended for EPT violations where the caller provides all the
+ * relevant VM-exit information.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ */
+static VBOXSTRICTRC iemVmxVmexitEptViolationWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo,
+                                                     PCVMXVEXITEVENTINFO pExitEventInfo) RT_NOEXCEPT
+{
+    Assert(pExitInfo);
+    Assert(pExitEventInfo);
+    Assert(pExitInfo->uReason == VMX_EXIT_EPT_VIOLATION);
+    Assert(!VMX_EXIT_INT_INFO_IS_VALID(pExitEventInfo->uExitIntInfo));
+
+    iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo);
+    iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode);
+
+    iemVmxVmcsSetExitGuestPhysAddr(pVCpu, pExitInfo->u64GuestPhysAddr);
+    if (pExitInfo->u64Qual & VMX_BF_EXIT_QUAL_EPT_LINEAR_ADDR_VALID_MASK)
+        iemVmxVmcsSetExitGuestLinearAddr(pVCpu, pExitInfo->u64GuestLinearAddr);
+    else
+        iemVmxVmcsSetExitGuestLinearAddr(pVCpu, 0);
+    iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_EPT_VIOLATION, pExitInfo->u64Qual);
+}
+
+
+/**
+ * Interface for HM and EM to emulate a VM-exit due to an EPT violation.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitEptViolation(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo,
+                                                        PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitEptViolationWithInfo(pVCpu, pExitInfo, pExitEventInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMX VM-exit handler for EPT-induced VM-exits.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pWalk       The page walk info.
+ * @param   fAccess     The access causing the EPT event, IEM_ACCESS_XXX.
+ * @param   fSlatFail   Additional SLAT info, IEM_SLAT_FAIL_XXX.
+ * @param   cbInstr     The VM-exit instruction length if applicable. Pass 0 if not
+ *                      applicable.
+ */
+VBOXSTRICTRC iemVmxVmexitEpt(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint32_t fAccess, uint32_t fSlatFail, uint8_t cbInstr) RT_NOEXCEPT
+{
+    Assert(pWalk->fIsSlat);
+    Assert(pWalk->fFailed & PGM_WALKFAIL_EPT);
+    Assert(!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxEptXcptVe);          /* #VE exceptions not supported. */
+    Assert(!(pWalk->fFailed & PGM_WALKFAIL_EPT_VIOLATION_CONVERTIBLE)); /* Without #VE, convertible violations not possible. */
+
+    if (pWalk->fFailed & PGM_WALKFAIL_EPT_VIOLATION)
+    {
+        LogFlow(("EptViolation: cs:rip=%04x:%08RX64 fAccess=%#RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fAccess));
+        uint64_t const fEptAccess = (pWalk->fEffective & PGM_PTATTRS_EPT_MASK) >> PGM_PTATTRS_EPT_SHIFT;
+        return iemVmxVmexitEptViolation(pVCpu, fAccess, fSlatFail, fEptAccess, pWalk->GCPhysNested, pWalk->fIsLinearAddrValid,
+                                        pWalk->GCPtr, cbInstr);
+    }
+
+    LogFlow(("EptMisconfig: cs:rip=%04x:%08RX64 fAccess=%#RX32\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, fAccess));
+    Assert(pWalk->fFailed & PGM_WALKFAIL_EPT_MISCONFIG);
+    return iemVmxVmexitEptMisconfig(pVCpu, pWalk->GCPhysNested);
+}
+
+
+/**
+ * VMX VM-exit handler for APIC accesses.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offAccess   The offset of the register being accessed.
+ * @param   fAccess     The type of access, see IEM_ACCESS_XXX.
+ */
+static VBOXSTRICTRC iemVmxVmexitApicAccess(PVMCPUCC pVCpu, uint16_t offAccess, uint32_t fAccess) RT_NOEXCEPT
+{
+    VMXAPICACCESS enmAccess;
+    bool const fInEventDelivery = IEMGetCurrentXcpt(pVCpu, NULL, NULL, NULL, NULL);
+    if (fInEventDelivery)
+        enmAccess = VMXAPICACCESS_LINEAR_EVENT_DELIVERY;
+    else if ((fAccess & (IEM_ACCESS_WHAT_MASK | IEM_ACCESS_TYPE_MASK)) == IEM_ACCESS_INSTRUCTION)
+        enmAccess = VMXAPICACCESS_LINEAR_INSTR_FETCH;
+    else if (fAccess & IEM_ACCESS_TYPE_WRITE)
+        enmAccess = VMXAPICACCESS_LINEAR_WRITE;
+    else
+        enmAccess = VMXAPICACCESS_LINEAR_READ;
+
+    uint64_t const u64ExitQual = RT_BF_MAKE(VMX_BF_EXIT_QUAL_APIC_ACCESS_OFFSET, offAccess)
+                               | RT_BF_MAKE(VMX_BF_EXIT_QUAL_APIC_ACCESS_TYPE,   enmAccess);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_APIC_ACCESS, u64ExitQual);
+}
+
+
+/**
+ * VMX VM-exit handler for APIC accesses.
+ *
+ * This is intended for APIC accesses where the caller provides all the
+ * relevant VM-exit information.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ */
+static VBOXSTRICTRC iemVmxVmexitApicAccessWithInfo(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo,
+                                                   PCVMXVEXITEVENTINFO pExitEventInfo) RT_NOEXCEPT
+{
+    /* VM-exit interruption information should not be valid for APIC-access VM-exits. */
+    Assert(!VMX_EXIT_INT_INFO_IS_VALID(pExitEventInfo->uExitIntInfo));
+    Assert(pExitInfo->uReason == VMX_EXIT_APIC_ACCESS);
+    iemVmxVmcsSetExitIntInfo(pVCpu, 0);
+    iemVmxVmcsSetExitIntErrCode(pVCpu, 0);
+    iemVmxVmcsSetExitInstrLen(pVCpu, pExitInfo->cbInstr);
+    iemVmxVmcsSetIdtVectoringInfo(pVCpu, pExitEventInfo->uIdtVectoringInfo);
+    iemVmxVmcsSetIdtVectoringErrCode(pVCpu, pExitEventInfo->uIdtVectoringErrCode);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_APIC_ACCESS, pExitInfo->u64Qual);
+}
+
+
+/**
+ * Interface for HM and EM to virtualize memory-mapped APIC accesses.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_VMX_MODIFIES_BEHAVIOR if the memory access was virtualized.
+ * @retval  VINF_VMX_VMEXIT if the access causes a VM-exit.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @param   pExitEventInfo  Pointer to the VM-exit event information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitApicAccess(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo, PCVMXVEXITEVENTINFO pExitEventInfo)
+{
+    Assert(pExitInfo);
+    Assert(pExitEventInfo);
+    VBOXSTRICTRC rcStrict = iemVmxVmexitApicAccessWithInfo(pVCpu, pExitInfo, pExitEventInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMX VM-exit handler for APIC-write VM-exits.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offApic     The write to the virtual-APIC page offset that caused this
+ *                      VM-exit.
+ */
+static VBOXSTRICTRC iemVmxVmexitApicWrite(PVMCPUCC pVCpu, uint16_t offApic) RT_NOEXCEPT
+{
+    Assert(offApic < XAPIC_OFF_END + 4);
+    /* Write only bits 11:0 of the APIC offset into the Exit qualification field. */
+    offApic &= UINT16_C(0xfff);
+    return iemVmxVmexit(pVCpu, VMX_EXIT_APIC_WRITE, offApic);
+}
+
+
+/**
+ * Clears any pending virtual-APIC write emulation.
+ *
+ * @returns The virtual-APIC offset that was written before clearing it.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(uint16_t) iemVmxVirtApicClearPendingWrite(PVMCPUCC pVCpu)
+{
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+    uint8_t const offVirtApicWrite = pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite;
+    pVCpu->cpum.GstCtx.hwvirt.vmx.offVirtApicWrite = 0;
+    Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE));
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_VMX_APIC_WRITE);
+    return offVirtApicWrite;
+}
+
+
+/**
+ * Reads a 32-bit register from the virtual-APIC page at the given offset.
+ *
+ * @returns The register from the virtual-APIC page.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   offReg  The offset of the register being read.
+ */
+uint32_t iemVmxVirtApicReadRaw32(PVMCPUCC pVCpu, uint16_t offReg) RT_NOEXCEPT
+{
+    Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint32_t));
+
+    uint32_t       uReg           = 0;
+    RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg, sizeof(uReg));
+    AssertMsgStmt(RT_SUCCESS(rc),
+                  ("Failed to read %u bytes at offset %#x of the virtual-APIC page at %#RGp: %Rrc\n",
+                   sizeof(uReg), offReg, GCPhysVirtApic, rc),
+                  uReg = 0);
+    return uReg;
+}
+
+
+/**
+ * Reads a 64-bit register from the virtual-APIC page at the given offset.
+ *
+ * @returns The register from the virtual-APIC page.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   offReg  The offset of the register being read.
+ */
+static uint64_t iemVmxVirtApicReadRaw64(PVMCPUCC pVCpu, uint16_t offReg) RT_NOEXCEPT
+{
+    Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint64_t));
+
+    uint64_t       uReg           = 0;
+    RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg, sizeof(uReg));
+    AssertMsgStmt(RT_SUCCESS(rc),
+                  ("Failed to read %u bytes at offset %#x of the virtual-APIC page at %#RGp: %Rrc\n",
+                   sizeof(uReg), offReg, GCPhysVirtApic, rc),
+                  uReg = 0);
+    return uReg;
+}
+
+
+/**
+ * Writes a 32-bit register to the virtual-APIC page at the given offset.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   offReg  The offset of the register being written.
+ * @param   uReg    The register value to write.
+ */
+void iemVmxVirtApicWriteRaw32(PVMCPUCC pVCpu, uint16_t offReg, uint32_t uReg) RT_NOEXCEPT
+{
+    Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint32_t));
+
+    RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg, &uReg, sizeof(uReg));
+    AssertMsgRC(rc, ("Failed to write %u bytes at offset %#x of the virtual-APIC page at %#RGp: %Rrc\n",
+                     sizeof(uReg), offReg, GCPhysVirtApic, rc));
+}
+
+
+/**
+ * Writes a 64-bit register to the virtual-APIC page at the given offset.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   offReg  The offset of the register being written.
+ * @param   uReg    The register value to write.
+ */
+static void iemVmxVirtApicWriteRaw64(PVMCPUCC pVCpu, uint16_t offReg, uint64_t uReg) RT_NOEXCEPT
+{
+    Assert(offReg <= VMX_V_VIRT_APIC_SIZE - sizeof(uint64_t));
+
+    RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg, &uReg, sizeof(uReg));
+    AssertMsgRC(rc, ("Failed to write %u bytes at offset %#x of the virtual-APIC page at %#RGp: %Rrc\n",
+                     sizeof(uReg), offReg, GCPhysVirtApic, rc));
+}
+
+
+/**
+ * Sets the vector in a virtual-APIC 256-bit sparse register.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offReg      The offset of the 256-bit spare register.
+ * @param   uVector     The vector to set.
+ *
+ * @remarks This is based on our APIC device code.
+ */
+static void iemVmxVirtApicSetVectorInReg(PVMCPUCC pVCpu, uint16_t offReg, uint8_t uVector) RT_NOEXCEPT
+{
+    /* Determine the vector offset within the chunk. */
+    uint16_t const offVector = (uVector & UINT32_C(0xe0)) >> 1;
+
+    /* Read the chunk at the offset. */
+    uint32_t uReg;
+    RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg + offVector, sizeof(uReg));
+    if (RT_SUCCESS(rc))
+    {
+        /* Modify the chunk. */
+        uint16_t const idxVectorBit = uVector & UINT32_C(0x1f);
+        uReg |= RT_BIT(idxVectorBit);
+
+        /* Write the chunk. */
+        rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg + offVector, &uReg, sizeof(uReg));
+        AssertMsgRC(rc, ("Failed to set vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp: %Rrc\n",
+                         uVector, offReg, GCPhysVirtApic, rc));
+    }
+    else
+        AssertMsgFailed(("Failed to get vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp: %Rrc\n",
+                         uVector, offReg, GCPhysVirtApic, rc));
+}
+
+
+/**
+ * Clears the vector in a virtual-APIC 256-bit sparse register.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offReg      The offset of the 256-bit spare register.
+ * @param   uVector     The vector to clear.
+ *
+ * @remarks This is based on our APIC device code.
+ */
+static void iemVmxVirtApicClearVectorInReg(PVMCPUCC pVCpu, uint16_t offReg, uint8_t uVector) RT_NOEXCEPT
+{
+    /* Determine the vector offset within the chunk. */
+    uint16_t const offVector      = (uVector & UINT32_C(0xe0)) >> 1;
+
+    /* Read the chunk at the offset. */
+    uint32_t uReg;
+    RTGCPHYS const GCPhysVirtApic = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrVirtApic.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &uReg, GCPhysVirtApic + offReg + offVector, sizeof(uReg));
+    if (RT_SUCCESS(rc))
+    {
+        /* Modify the chunk. */
+        uint16_t const idxVectorBit = uVector & UINT32_C(0x1f);
+        uReg &= ~RT_BIT(idxVectorBit);
+
+        /* Write the chunk. */
+        rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic + offReg + offVector, &uReg, sizeof(uReg));
+        AssertMsgRC(rc, ("Failed to clear vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp: %Rrc\n",
+                         uVector, offReg, GCPhysVirtApic, rc));
+    }
+    else
+        AssertMsgFailed(("Failed to get vector %#x in 256-bit register at %#x of the virtual-APIC page at %#RGp: %Rrc\n",
+                         uVector, offReg, GCPhysVirtApic, rc));
+}
+
+
+/**
+ * Checks if a memory access to the APIC-access page must causes an APIC-access
+ * VM-exit.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offAccess   The offset of the register being accessed.
+ * @param   cbAccess    The size of the access in bytes.
+ * @param   fAccess     The type of access, see IEM_ACCESS_XXX.
+ *
+ * @remarks This must not be used for MSR-based APIC-access page accesses!
+ * @sa      iemVmxVirtApicAccessMsrWrite, iemVmxVirtApicAccessMsrRead.
+ */
+static bool iemVmxVirtApicIsMemAccessIntercepted(PVMCPUCC pVCpu, uint16_t offAccess, size_t cbAccess, uint32_t fAccess) RT_NOEXCEPT
+{
+    Assert(cbAccess > 0);
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    /*
+     * We must cause a VM-exit if any of the following are true:
+     *   - TPR shadowing isn't active.
+     *   - The access size exceeds 32-bits.
+     *   - The access is not contained within low 4 bytes of a 16-byte aligned offset.
+     *
+     * See Intel spec. 29.4.2 "Virtualizing Reads from the APIC-Access Page".
+     * See Intel spec. 29.4.3.1 "Determining Whether a Write Access is Virtualized".
+     */
+    if (   !(pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+        || cbAccess > sizeof(uint32_t)
+        || ((offAccess + cbAccess - 1) & 0xc)
+        || offAccess >= XAPIC_OFF_END + 4)
+        return true;
+
+    /*
+     * If the access is part of an operation where we have already
+     * virtualized a virtual-APIC write, we must cause a VM-exit.
+     */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
+        return true;
+
+    /*
+     * Check write accesses to the APIC-access page that cause VM-exits.
+     */
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+    {
+        if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+        {
+            /*
+             * With APIC-register virtualization, a write access to any of the
+             * following registers are virtualized. Accessing any other register
+             * causes a VM-exit.
+             */
+            uint16_t const offAlignedAccess = offAccess & 0xfffc;
+            switch (offAlignedAccess)
+            {
+                case XAPIC_OFF_ID:
+                case XAPIC_OFF_TPR:
+                case XAPIC_OFF_EOI:
+                case XAPIC_OFF_LDR:
+                case XAPIC_OFF_DFR:
+                case XAPIC_OFF_SVR:
+                case XAPIC_OFF_ESR:
+                case XAPIC_OFF_ICR_LO:
+                case XAPIC_OFF_ICR_HI:
+                case XAPIC_OFF_LVT_TIMER:
+                case XAPIC_OFF_LVT_THERMAL:
+                case XAPIC_OFF_LVT_PERF:
+                case XAPIC_OFF_LVT_LINT0:
+                case XAPIC_OFF_LVT_LINT1:
+                case XAPIC_OFF_LVT_ERROR:
+                case XAPIC_OFF_TIMER_ICR:
+                case XAPIC_OFF_TIMER_DCR:
+                    break;
+                default:
+                    return true;
+            }
+        }
+        else if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)
+        {
+            /*
+             * With virtual-interrupt delivery, a write access to any of the
+             * following registers are virtualized. Accessing any other register
+             * causes a VM-exit.
+             *
+             * Note! The specification does not allow writing to offsets in-between
+             * these registers (e.g. TPR + 1 byte) unlike read accesses.
+             */
+            switch (offAccess)
+            {
+                case XAPIC_OFF_TPR:
+                case XAPIC_OFF_EOI:
+                case XAPIC_OFF_ICR_LO:
+                    break;
+                default:
+                    return true;
+            }
+        }
+        else
+        {
+            /*
+             * Without APIC-register virtualization or virtual-interrupt delivery,
+             * only TPR accesses are virtualized.
+             */
+            if (offAccess == XAPIC_OFF_TPR)
+            { /* likely */ }
+            else
+                return true;
+        }
+    }
+    else
+    {
+        /*
+         * Check read accesses to the APIC-access page that cause VM-exits.
+         */
+        if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+        {
+            /*
+             * With APIC-register virtualization, a read access to any of the
+             * following registers are virtualized. Accessing any other register
+             * causes a VM-exit.
+             */
+            uint16_t const offAlignedAccess = offAccess & 0xfffc;
+            switch (offAlignedAccess)
+            {
+                /** @todo r=ramshankar: What about XAPIC_OFF_LVT_CMCI? */
+                case XAPIC_OFF_ID:
+                case XAPIC_OFF_VERSION:
+                case XAPIC_OFF_TPR:
+                case XAPIC_OFF_EOI:
+                case XAPIC_OFF_LDR:
+                case XAPIC_OFF_DFR:
+                case XAPIC_OFF_SVR:
+                case XAPIC_OFF_ISR0:    case XAPIC_OFF_ISR1:    case XAPIC_OFF_ISR2:    case XAPIC_OFF_ISR3:
+                case XAPIC_OFF_ISR4:    case XAPIC_OFF_ISR5:    case XAPIC_OFF_ISR6:    case XAPIC_OFF_ISR7:
+                case XAPIC_OFF_TMR0:    case XAPIC_OFF_TMR1:    case XAPIC_OFF_TMR2:    case XAPIC_OFF_TMR3:
+                case XAPIC_OFF_TMR4:    case XAPIC_OFF_TMR5:    case XAPIC_OFF_TMR6:    case XAPIC_OFF_TMR7:
+                case XAPIC_OFF_IRR0:    case XAPIC_OFF_IRR1:    case XAPIC_OFF_IRR2:    case XAPIC_OFF_IRR3:
+                case XAPIC_OFF_IRR4:    case XAPIC_OFF_IRR5:    case XAPIC_OFF_IRR6:    case XAPIC_OFF_IRR7:
+                case XAPIC_OFF_ESR:
+                case XAPIC_OFF_ICR_LO:
+                case XAPIC_OFF_ICR_HI:
+                case XAPIC_OFF_LVT_TIMER:
+                case XAPIC_OFF_LVT_THERMAL:
+                case XAPIC_OFF_LVT_PERF:
+                case XAPIC_OFF_LVT_LINT0:
+                case XAPIC_OFF_LVT_LINT1:
+                case XAPIC_OFF_LVT_ERROR:
+                case XAPIC_OFF_TIMER_ICR:
+                case XAPIC_OFF_TIMER_DCR:
+                    break;
+                default:
+                    return true;
+            }
+        }
+        else
+        {
+            /* Without APIC-register virtualization, only TPR accesses are virtualized. */
+            if (offAccess == XAPIC_OFF_TPR)
+            { /* likely */ }
+            else
+                return true;
+        }
+    }
+
+    /* The APIC access is virtualized, does not cause a VM-exit. */
+    return false;
+}
+
+
+/**
+ * Virtualizes a memory-based APIC access by certain instructions even though they
+ * do not use the address to access memory.
+ *
+ * This is for instructions like MONITOR, CLFLUSH, CLFLUSHOPT, ENTER which may cause
+ * page-faults but do not use the address to access memory.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pGCPhysAccess   Pointer to the guest-physical address accessed.
+ * @param   cbAccess        The size of the access in bytes.
+ * @param   fAccess         The type of access, see IEM_ACCESS_XXX.
+ */
+VBOXSTRICTRC iemVmxVirtApicAccessUnused(PVMCPUCC pVCpu, PRTGCPHYS pGCPhysAccess, size_t cbAccess, uint32_t fAccess) RT_NOEXCEPT
+{
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
+    Assert(pGCPhysAccess);
+
+    RTGCPHYS const GCPhysAccess = *pGCPhysAccess & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    RTGCPHYS const GCPhysApic   = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64AddrApicAccess.u;
+    Assert(!(GCPhysApic & GUEST_PAGE_OFFSET_MASK));
+
+    if (GCPhysAccess == GCPhysApic)
+    {
+        uint16_t const offAccess = *pGCPhysAccess & GUEST_PAGE_OFFSET_MASK;
+        bool const fIntercept = iemVmxVirtApicIsMemAccessIntercepted(pVCpu, offAccess, cbAccess, fAccess);
+        if (fIntercept)
+            return iemVmxVmexitApicAccess(pVCpu, offAccess, fAccess);
+
+        *pGCPhysAccess = GCPhysApic | offAccess;
+        return VINF_VMX_MODIFIES_BEHAVIOR;
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Virtualizes a memory-based APIC access.
+ *
+ * @returns VBox strict status code.
+ * @retval VINF_VMX_MODIFIES_BEHAVIOR if the access was virtualized.
+ * @retval VINF_VMX_VMEXIT if the access causes a VM-exit.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offAccess   The offset of the register being accessed (within the
+ *                      APIC-access page).
+ * @param   cbAccess    The size of the access in bytes.
+ * @param   pvData      Pointer to the data being written or where to store the data
+ *                      being read.
+ * @param   fAccess     The type of access, see IEM_ACCESS_XXX.
+ */
+static VBOXSTRICTRC iemVmxVirtApicAccessMem(PVMCPUCC pVCpu, uint16_t offAccess, size_t cbAccess,
+                                            void *pvData, uint32_t fAccess) RT_NOEXCEPT
+{
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS);
+    Assert(pvData);
+
+    bool const fIntercept = iemVmxVirtApicIsMemAccessIntercepted(pVCpu, offAccess, cbAccess, fAccess);
+    if (fIntercept)
+        return iemVmxVmexitApicAccess(pVCpu, offAccess, fAccess);
+
+    if (fAccess & IEM_ACCESS_TYPE_WRITE)
+    {
+        /*
+         * A write access to the APIC-access page that is virtualized (rather than
+         * causing a VM-exit) writes data to the virtual-APIC page.
+         */
+        uint32_t const u32Data = *(uint32_t *)pvData;
+        iemVmxVirtApicWriteRaw32(pVCpu, offAccess, u32Data);
+
+        /*
+         * Record the currently updated APIC offset, as we need this later for figuring
+         * out whether to perform TPR, EOI or self-IPI virtualization as well as well
+         * as for supplying the exit qualification when causing an APIC-write VM-exit.
+         *
+         * After completion of the current operation, we need to perform TPR virtualization,
+         * EOI virtualization or APIC-write VM-exit depending on which register was written.
+         *
+         * The current operation may be a REP-prefixed string instruction, execution of any
+         * other instruction, or delivery of an event through the IDT.
+         *
+         * Thus things like clearing bytes 3:1 of the VTPR, clearing VEOI are not to be
+         * performed now but later after completion of the current operation.
+         *
+         * See Intel spec. 29.4.3.2 "APIC-Write Emulation".
+         */
+        iemVmxVirtApicSetPendingWrite(pVCpu, offAccess);
+
+        LogFlowFunc(("Write access at offset %#x not intercepted -> Wrote %#RX32\n", offAccess, u32Data));
+    }
+    else
+    {
+        /*
+         * A read access from the APIC-access page that is virtualized (rather than
+         * causing a VM-exit) returns data from the virtual-APIC page.
+         *
+         * See Intel spec. 29.4.2 "Virtualizing Reads from the APIC-Access Page".
+         */
+        Assert(fAccess & IEM_ACCESS_TYPE_READ);
+
+        Assert(cbAccess <= 4);
+        Assert(offAccess < XAPIC_OFF_END + 4);
+        static uint32_t const s_auAccessSizeMasks[] = { 0, 0xff, 0xffff, 0xffffff, 0xffffffff };
+
+        uint32_t u32Data = iemVmxVirtApicReadRaw32(pVCpu, offAccess);
+        u32Data &= s_auAccessSizeMasks[cbAccess];
+        *(uint32_t *)pvData = u32Data;
+
+        LogFlowFunc(("Read access at offset %#x not intercepted -> Read %#RX32\n", offAccess, u32Data));
+    }
+
+    return VINF_VMX_MODIFIES_BEHAVIOR;
+}
+
+
+/**
+ * Virtualizes an MSR-based APIC read access.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_VMX_MODIFIES_BEHAVIOR if the MSR read was virtualized.
+ * @retval  VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR read access must be
+ *          handled by the x2APIC device.
+ * @retval  VERR_OUT_RANGE if the MSR read was supposed to be virtualized but was
+ *          not within the range of valid MSRs, caller must raise \#GP(0).
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The x2APIC MSR being read.
+ * @param   pu64Value   Where to store the read x2APIC MSR value (only valid when
+ *                      VINF_VMX_MODIFIES_BEHAVIOR is returned).
+ */
+static VBOXSTRICTRC iemVmxVirtApicAccessMsrRead(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value) RT_NOEXCEPT
+{
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE);
+    Assert(pu64Value);
+
+    if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+    {
+        if (   idMsr >= MSR_IA32_X2APIC_START
+            && idMsr <= MSR_IA32_X2APIC_END)
+        {
+            uint16_t const offReg   = (idMsr & 0xff) << 4;
+            uint64_t const u64Value = iemVmxVirtApicReadRaw64(pVCpu, offReg);
+            *pu64Value = u64Value;
+            return VINF_VMX_MODIFIES_BEHAVIOR;
+        }
+        return VERR_OUT_OF_RANGE;
+    }
+
+    if (idMsr == MSR_IA32_X2APIC_TPR)
+    {
+        uint16_t const offReg   = (idMsr & 0xff) << 4;
+        uint64_t const u64Value = iemVmxVirtApicReadRaw64(pVCpu, offReg);
+        *pu64Value = u64Value;
+        return VINF_VMX_MODIFIES_BEHAVIOR;
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Virtualizes an MSR-based APIC write access.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_VMX_MODIFIES_BEHAVIOR if the MSR write was virtualized.
+ * @retval  VERR_OUT_RANGE if the MSR read was supposed to be virtualized but was
+ *          not within the range of valid MSRs, caller must raise \#GP(0).
+ * @retval  VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR must be written normally.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idMsr       The x2APIC MSR being written.
+ * @param   u64Value    The value of the x2APIC MSR being written.
+ */
+static VBOXSTRICTRC iemVmxVirtApicAccessMsrWrite(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t u64Value) RT_NOEXCEPT
+{
+    /*
+     * Check if the access is to be virtualized.
+     * See Intel spec. 29.5 "Virtualizing MSR-based APIC Accesses".
+     */
+    if (   idMsr == MSR_IA32_X2APIC_TPR
+        || (   (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)
+            && (   idMsr == MSR_IA32_X2APIC_EOI
+                || idMsr == MSR_IA32_X2APIC_SELF_IPI)))
+    {
+        /* Validate the MSR write depending on the register. */
+        switch (idMsr)
+        {
+            case MSR_IA32_X2APIC_TPR:
+            case MSR_IA32_X2APIC_SELF_IPI:
+            {
+                if (u64Value & UINT64_C(0xffffffffffffff00))
+                    return VERR_OUT_OF_RANGE;
+                break;
+            }
+            case MSR_IA32_X2APIC_EOI:
+            {
+                if (u64Value != 0)
+                    return VERR_OUT_OF_RANGE;
+                break;
+            }
+        }
+
+        /* Write the MSR to the virtual-APIC page. */
+        uint16_t const offReg = (idMsr & 0xff) << 4;
+        iemVmxVirtApicWriteRaw64(pVCpu, offReg, u64Value);
+
+        /*
+         * Record the currently updated APIC offset, as we need this later for figuring
+         * out whether to perform TPR, EOI or self-IPI virtualization as well as well
+         * as for supplying the exit qualification when causing an APIC-write VM-exit.
+         */
+        iemVmxVirtApicSetPendingWrite(pVCpu, offReg);
+
+        return VINF_VMX_MODIFIES_BEHAVIOR;
+    }
+
+    return VINF_VMX_INTERCEPT_NOT_ACTIVE;
+}
+
+
+/**
+ * Interface for HM and EM to virtualize x2APIC MSR accesses.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_VMX_MODIFIES_BEHAVIOR if the MSR access was virtualized.
+ * @retval  VINF_VMX_INTERCEPT_NOT_ACTIVE if the MSR access must be handled by
+ *          the x2APIC device.
+ * @retval  VERR_OUT_RANGE if the caller must raise \#GP(0).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   idMsr       The MSR being read.
+ * @param   pu64Value   Pointer to the value being written or where to store the
+ *                      value being read.
+ * @param   fWrite      Whether this is an MSR write or read access.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVirtApicAccessMsr(PVMCPUCC pVCpu, uint32_t idMsr, uint64_t *pu64Value, bool fWrite)
+{
+    Assert(pu64Value);
+
+    VBOXSTRICTRC rcStrict;
+    if (fWrite)
+        rcStrict = iemVmxVirtApicAccessMsrWrite(pVCpu, idMsr, *pu64Value);
+    else
+        rcStrict = iemVmxVirtApicAccessMsrRead(pVCpu, idMsr, pu64Value);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+
+}
+
+
+/**
+ * Finds the most significant set bit in a virtual-APIC 256-bit sparse register.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when the highest set bit is found.
+ * @retval  VERR_NOT_FOUND when no bit is set.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   offReg          The offset of the APIC 256-bit sparse register.
+ * @param   pidxHighestBit  Where to store the highest bit (most significant bit)
+ *                          set in the register. Only valid when VINF_SUCCESS is
+ *                          returned.
+ *
+ * @remarks The format of the 256-bit sparse register here mirrors that found in
+ *          real APIC hardware.
+ */
+static int iemVmxVirtApicGetHighestSetBitInReg(PVMCPUCC pVCpu, uint16_t offReg, uint8_t *pidxHighestBit)
+{
+    Assert(offReg < XAPIC_OFF_END + 4);
+    Assert(pidxHighestBit);
+
+    /*
+     * There are 8 contiguous fragments (of 16-bytes each) in the sparse register.
+     * However, in each fragment only the first 4 bytes are used.
+     */
+    uint8_t const cFrags = 8;
+    for (int8_t iFrag = cFrags; iFrag >= 0; iFrag--)
+    {
+        uint16_t const offFrag = iFrag * 16;
+        uint32_t const u32Frag = iemVmxVirtApicReadRaw32(pVCpu, offReg + offFrag);
+        if (!u32Frag)
+            continue;
+
+        unsigned idxHighestBit = ASMBitLastSetU32(u32Frag);
+        Assert(idxHighestBit > 0);
+        --idxHighestBit;
+        Assert(idxHighestBit <= UINT8_MAX);
+        *pidxHighestBit = idxHighestBit;
+        return VINF_SUCCESS;
+    }
+    return VERR_NOT_FOUND;
+}
+
+
+/**
+ * Evaluates pending virtual interrupts.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxEvalPendingVirtIntrs(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+
+    if (!(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
+    {
+        uint8_t const uRvi = RT_LO_U8(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u16GuestIntStatus);
+        uint8_t const uPpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_PPR);
+
+        if ((uRvi >> 4) > (uPpr >> 4))
+        {
+            Log2(("eval_virt_intrs: uRvi=%#x uPpr=%#x - Signalling pending interrupt\n", uRvi, uPpr));
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
+        }
+        else
+            Log2(("eval_virt_intrs: uRvi=%#x uPpr=%#x - Nothing to do\n", uRvi, uPpr));
+    }
+}
+
+
+/**
+ * Performs PPR virtualization.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxPprVirtualization(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+
+    /*
+     * PPR virtualization is caused in response to a VM-entry, TPR-virtualization,
+     * or EOI-virtualization.
+     *
+     * See Intel spec. 29.1.3 "PPR Virtualization".
+     */
+    uint8_t const uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
+    uint8_t const uSvi = RT_HI_U8(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u16GuestIntStatus) & 0xf0;
+
+    uint32_t uPpr;
+    if ((uTpr & 0xf0) >= uSvi)
+        uPpr = uTpr;
+    else
+        uPpr = uSvi;
+
+    Log2(("ppr_virt: uTpr=%#x uSvi=%#x uPpr=%#x\n", uTpr, uSvi, uPpr));
+    iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_PPR, uPpr);
+}
+
+
+/**
+ * Performs VMX TPR virtualization.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static VBOXSTRICTRC iemVmxTprVirtualization(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    Assert(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+
+    /*
+     * We should have already performed the virtual-APIC write to the TPR offset
+     * in the virtual-APIC page. We now perform TPR virtualization.
+     *
+     * See Intel spec. 29.1.2 "TPR Virtualization".
+     */
+    if (!(pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY))
+    {
+        uint32_t const uTprThreshold = pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32TprThreshold;
+        uint32_t const uTpr          = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
+
+        /*
+         * If the VTPR falls below the TPR threshold, we must cause a VM-exit.
+         * See Intel spec. 29.1.2 "TPR Virtualization".
+         */
+        if (((uTpr >> 4) & 0xf) < uTprThreshold)
+        {
+            Log2(("tpr_virt: uTpr=%u uTprThreshold=%u -> VM-exit\n", uTpr, uTprThreshold));
+            return iemVmxVmexit(pVCpu, VMX_EXIT_TPR_BELOW_THRESHOLD, 0 /* u64ExitQual */);
+        }
+    }
+    else
+    {
+        iemVmxPprVirtualization(pVCpu);
+        iemVmxEvalPendingVirtIntrs(pVCpu);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks whether an EOI write for the given interrupt vector causes a VM-exit or
+ * not.
+ *
+ * @returns @c true if the EOI write is intercepted, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uVector     The interrupt that was acknowledged using an EOI.
+ */
+static bool iemVmxIsEoiInterceptSet(PCVMCPU pVCpu, uint8_t uVector) RT_NOEXCEPT
+{
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+
+    if (uVector < 64)
+        return RT_BOOL(pVmcs->u64EoiExitBitmap0.u & RT_BIT_64(uVector));
+    if (uVector < 128)
+        return RT_BOOL(pVmcs->u64EoiExitBitmap1.u & RT_BIT_64(uVector));
+    if (uVector < 192)
+        return RT_BOOL(pVmcs->u64EoiExitBitmap2.u & RT_BIT_64(uVector));
+    return RT_BOOL(pVmcs->u64EoiExitBitmap3.u & RT_BIT_64(uVector));
+}
+
+
+/**
+ * Performs EOI virtualization.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+static VBOXSTRICTRC iemVmxEoiVirtualization(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+
+    /*
+     * Clear the interrupt guest-interrupt as no longer in-service (ISR)
+     * and get the next guest-interrupt that's in-service (if any).
+     *
+     * See Intel spec. 29.1.4 "EOI Virtualization".
+     */
+    uint8_t const uRvi = RT_LO_U8(pVmcs->u16GuestIntStatus);
+    uint8_t const uSvi = RT_HI_U8(pVmcs->u16GuestIntStatus);
+    Log2(("eoi_virt: uRvi=%#x uSvi=%#x\n", uRvi, uSvi));
+
+    uint8_t uVector = uSvi;
+    iemVmxVirtApicClearVectorInReg(pVCpu, XAPIC_OFF_ISR0, uVector);
+
+    uVector = 0;
+    iemVmxVirtApicGetHighestSetBitInReg(pVCpu, XAPIC_OFF_ISR0, &uVector);
+
+    if (uVector)
+        Log2(("eoi_virt: next interrupt %#x\n", uVector));
+    else
+        Log2(("eoi_virt: no interrupt pending in ISR\n"));
+
+    /* Update guest-interrupt status SVI (leave RVI portion as it is) in the VMCS. */
+    pVmcs->u16GuestIntStatus = RT_MAKE_U16(uRvi, uVector);
+
+    iemVmxPprVirtualization(pVCpu);
+    if (iemVmxIsEoiInterceptSet(pVCpu, uVector))
+        return iemVmxVmexit(pVCpu, VMX_EXIT_VIRTUALIZED_EOI, uVector);
+    iemVmxEvalPendingVirtIntrs(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs self-IPI virtualization.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static VBOXSTRICTRC iemVmxSelfIpiVirtualization(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+
+    /*
+     * We should have already performed the virtual-APIC write to the self-IPI offset
+     * in the virtual-APIC page. We now perform self-IPI virtualization.
+     *
+     * See Intel spec. 29.1.5 "Self-IPI Virtualization".
+     */
+    uint8_t const uVector = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_ICR_LO);
+    Log2(("self_ipi_virt: uVector=%#x\n", uVector));
+    iemVmxVirtApicSetVectorInReg(pVCpu, XAPIC_OFF_IRR0, uVector);
+    uint8_t const uRvi = RT_LO_U8(pVmcs->u16GuestIntStatus);
+    uint8_t const uSvi = RT_HI_U8(pVmcs->u16GuestIntStatus);
+    if (uVector > uRvi)
+        pVmcs->u16GuestIntStatus = RT_MAKE_U16(uVector, uSvi);
+    iemVmxEvalPendingVirtIntrs(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Performs VMX APIC-write emulation.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VBOXSTRICTRC iemVmxApicWriteEmulation(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    /* Import the virtual-APIC write offset (part of the hardware-virtualization state). */
+    IEM_CTX_IMPORT_RET(pVCpu, CPUMCTX_EXTRN_HWVIRT);
+
+    /*
+     * Perform APIC-write emulation based on the virtual-APIC register written.
+     * See Intel spec. 29.4.3.2 "APIC-Write Emulation".
+     */
+    uint16_t const offApicWrite = iemVmxVirtApicClearPendingWrite(pVCpu);
+    VBOXSTRICTRC rcStrict;
+    switch (offApicWrite)
+    {
+        case XAPIC_OFF_TPR:
+        {
+            /* Clear bytes 3:1 of the VTPR and perform TPR virtualization. */
+            uint32_t uTpr = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
+            uTpr         &= UINT32_C(0x000000ff);
+            iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_TPR, uTpr);
+            Log2(("iemVmxApicWriteEmulation: TPR write %#x\n", uTpr));
+            rcStrict = iemVmxTprVirtualization(pVCpu);
+            break;
+        }
+
+        case XAPIC_OFF_EOI:
+        {
+            if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)
+            {
+                /* Clear VEOI and perform EOI virtualization. */
+                iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_EOI, 0);
+                Log2(("iemVmxApicWriteEmulation: EOI write\n"));
+                rcStrict = iemVmxEoiVirtualization(pVCpu);
+            }
+            else
+                rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite);
+            break;
+        }
+
+        case XAPIC_OFF_ICR_LO:
+        {
+            if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)
+            {
+                /* If the ICR_LO is valid, write it and perform self-IPI virtualization. */
+                uint32_t const uIcrLo    = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_TPR);
+                uint32_t const fIcrLoMb0 = UINT32_C(0xfffbb700);
+                uint32_t const fIcrLoMb1 = UINT32_C(0x000000f0);
+                if (   !(uIcrLo & fIcrLoMb0)
+                    &&  (uIcrLo & fIcrLoMb1))
+                {
+                    Log2(("iemVmxApicWriteEmulation: Self-IPI virtualization with vector %#x\n", (uIcrLo & 0xff)));
+                    rcStrict = iemVmxSelfIpiVirtualization(pVCpu);
+                }
+                else
+                    rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite);
+            }
+            else
+                rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite);
+            break;
+        }
+
+        case XAPIC_OFF_ICR_HI:
+        {
+            /* Clear bytes 2:0 of VICR_HI. No other virtualization or VM-exit must occur. */
+            uint32_t uIcrHi  = iemVmxVirtApicReadRaw32(pVCpu, XAPIC_OFF_ICR_HI);
+            uIcrHi          &= UINT32_C(0xff000000);
+            iemVmxVirtApicWriteRaw32(pVCpu, XAPIC_OFF_ICR_HI, uIcrHi);
+            rcStrict = VINF_SUCCESS;
+            break;
+        }
+
+        default:
+        {
+            /* Writes to any other virtual-APIC register causes an APIC-write VM-exit. */
+            rcStrict = iemVmxVmexitApicWrite(pVCpu, offApicWrite);
+            break;
+        }
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Interface for HM and EM to perform an APIC-write emulation which may cause a
+ * VM-exit.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecVmxVmexitApicWrite(PVMCPUCC pVCpu)
+{
+    VBOXSTRICTRC rcStrict = iemVmxApicWriteEmulation(pVCpu);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+}
+
+
+/**
+ * Checks guest control registers, debug registers and MSRs as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestControlRegsMsrs(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /*
+     * Guest Control Registers, Debug Registers, and MSRs.
+     * See Intel spec. 26.3.1.1 "Checks on Guest Control Registers, Debug Registers, and MSRs".
+     */
+    PCVMXVVMCS const   pVmcs              = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char * const pszFailure         = "VM-exit";
+    bool const         fUnrestrictedGuest = RT_BOOL(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
+
+    /* CR0 reserved bits. */
+    {
+        /* CR0 MB1 bits. */
+        uint64_t const u64Cr0Fixed0 = iemVmxGetCr0Fixed0(pVCpu, true /* fVmxNonRootMode */);
+        if ((pVmcs->u64GuestCr0.u & u64Cr0Fixed0) == u64Cr0Fixed0)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr0Fixed0);
+
+        /* CR0 MBZ bits. */
+        uint64_t const u64Cr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+        if (!(pVmcs->u64GuestCr0.u & ~u64Cr0Fixed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr0Fixed1);
+
+        /* Without unrestricted guest support, VT-x supports does not support unpaged protected mode. */
+        if (   !fUnrestrictedGuest
+            &&  (pVmcs->u64GuestCr0.u & X86_CR0_PG)
+            && !(pVmcs->u64GuestCr0.u & X86_CR0_PE))
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr0PgPe);
+    }
+
+    /* CR4 reserved bits. */
+    {
+        /* CR4 MB1 bits. */
+        uint64_t const u64Cr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+        if ((pVmcs->u64GuestCr4.u & u64Cr4Fixed0) == u64Cr4Fixed0)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr4Fixed0);
+
+        /* CR4 MBZ bits. */
+        uint64_t const u64Cr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+        if (!(pVmcs->u64GuestCr4.u & ~u64Cr4Fixed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr4Fixed1);
+    }
+
+    /* DEBUGCTL MSR. */
+    if (   !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+        || !(pVmcs->u64GuestDebugCtlMsr.u & ~MSR_IA32_DEBUGCTL_VALID_MASK_INTEL))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestDebugCtl);
+
+    /* 64-bit CPU checks. */
+    bool const fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        if (fGstInLongMode)
+        {
+            /* PAE must be set. */
+            if (   (pVmcs->u64GuestCr0.u & X86_CR0_PG)
+                && (pVmcs->u64GuestCr0.u & X86_CR4_PAE))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPae);
+        }
+        else
+        {
+            /* PCIDE should not be set. */
+            if (!(pVmcs->u64GuestCr4.u & X86_CR4_PCIDE))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPcide);
+        }
+
+        /* CR3. */
+        if (!(pVmcs->u64GuestCr3.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestCr3);
+
+        /* DR7. */
+        if (   !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+            || !(pVmcs->u64GuestDr7.u & X86_DR7_MBZ_MASK))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestDr7);
+
+        /* SYSENTER ESP and SYSENTER EIP. */
+        if (   X86_IS_CANONICAL(pVmcs->u64GuestSysenterEsp.u)
+            && X86_IS_CANONICAL(pVmcs->u64GuestSysenterEip.u))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSysenterEspEip);
+    }
+
+    /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */
+    Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR));
+
+    /* PAT MSR. */
+    if (   !(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
+        ||  CPUMIsPatMsrValid(pVmcs->u64GuestPatMsr.u))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPatMsr);
+
+    /* EFER MSR. */
+    if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
+    {
+        uint64_t const uValidEferMask = CPUMGetGuestEferMsrValidMask(pVCpu->CTX_SUFF(pVM));
+        if (!(pVmcs->u64GuestEferMsr.u & ~uValidEferMask))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestEferMsrRsvd);
+
+        bool const fGstLma = RT_BOOL(pVmcs->u64GuestEferMsr.u & MSR_K6_EFER_LMA);
+        bool const fGstLme = RT_BOOL(pVmcs->u64GuestEferMsr.u & MSR_K6_EFER_LME);
+        if (   fGstLma == fGstInLongMode
+            && (   !(pVmcs->u64GuestCr0.u & X86_CR0_PG)
+                || fGstLma == fGstLme))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestEferMsr);
+    }
+
+    /* We don't support IA32_BNDCFGS MSR yet. */
+    Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_BNDCFGS_MSR));
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks guest segment registers, LDTR and TR as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestSegRegs(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /*
+     * Segment registers.
+     * See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
+     */
+    PCVMXVVMCS const   pVmcs              = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char * const pszFailure         = "VM-exit";
+    bool const         fGstInV86Mode      = RT_BOOL(pVmcs->u64GuestRFlags.u & X86_EFL_VM);
+    bool const         fUnrestrictedGuest = RT_BOOL(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST);
+    bool const         fGstInLongMode     = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+
+    /* Selectors. */
+    if (   !fGstInV86Mode
+        && !fUnrestrictedGuest
+        && (pVmcs->GuestSs & X86_SEL_RPL) != (pVmcs->GuestCs & X86_SEL_RPL))
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegSelCsSsRpl);
+
+    for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++)
+    {
+        CPUMSELREG SelReg;
+        int rc = iemVmxVmcsGetGuestSegReg(pVmcs, iSegReg, &SelReg);
+        if (RT_LIKELY(rc == VINF_SUCCESS))
+        { /* likely */ }
+        else
+            return rc;
+
+        /*
+         * Virtual-8086 mode checks.
+         */
+        if (fGstInV86Mode)
+        {
+            /* Base address. */
+            if (SelReg.u64Base == (uint64_t)SelReg.Sel << 4)
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegBaseV86(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* Limit. */
+            if (SelReg.u32Limit == 0xffff)
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegLimitV86(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* Attribute. */
+            if (SelReg.Attr.u == 0xf3)
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrV86(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* We're done; move to checking the next segment. */
+            continue;
+        }
+
+        /* Checks done by 64-bit CPUs. */
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+        {
+            /* Base address. */
+            if (   iSegReg == X86_SREG_FS
+                || iSegReg == X86_SREG_GS)
+            {
+                if (X86_IS_CANONICAL(SelReg.u64Base))
+                { /* likely */ }
+                else
+                {
+                    VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegBase(iSegReg);
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+                }
+            }
+            else if (iSegReg == X86_SREG_CS)
+            {
+                if (!RT_HI_U32(SelReg.u64Base))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegBaseCs);
+            }
+            else
+            {
+                if (   SelReg.Attr.n.u1Unusable
+                    || !RT_HI_U32(SelReg.u64Base))
+                { /* likely */ }
+                else
+                {
+                    VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegBase(iSegReg);
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+                }
+            }
+        }
+
+        /*
+         * Checks outside Virtual-8086 mode.
+         */
+        uint8_t const uSegType     =  SelReg.Attr.n.u4Type;
+        uint8_t const fCodeDataSeg =  SelReg.Attr.n.u1DescType;
+        uint8_t const fUsable      = !SelReg.Attr.n.u1Unusable;
+        uint8_t const uDpl         =  SelReg.Attr.n.u2Dpl;
+        uint8_t const fPresent     =  SelReg.Attr.n.u1Present;
+        uint8_t const uGranularity =  SelReg.Attr.n.u1Granularity;
+        uint8_t const uDefBig      =  SelReg.Attr.n.u1DefBig;
+        uint8_t const fSegLong     =  SelReg.Attr.n.u1Long;
+
+        /* Code or usable segment. */
+        if (   iSegReg == X86_SREG_CS
+            || fUsable)
+        {
+            /* Reserved bits (bits 31:17 and bits 11:8). */
+            if (!(SelReg.Attr.u & 0xfffe0f00))
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrRsvd(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* Descriptor type. */
+            if (fCodeDataSeg)
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrDescType(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* Present. */
+            if (fPresent)
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrPresent(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+
+            /* Granularity. */
+            if (   ((SelReg.u32Limit & 0x00000fff) == 0x00000fff || !uGranularity)
+                && ((SelReg.u32Limit & 0xfff00000) == 0x00000000 ||  uGranularity))
+            { /* likely */ }
+            else
+            {
+                VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrGran(iSegReg);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+        }
+
+        if (iSegReg == X86_SREG_CS)
+        {
+            /* Segment Type and DPL. */
+            if (   uSegType == (X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED)
+                && fUnrestrictedGuest)
+            {
+                if (uDpl == 0)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDplZero);
+            }
+            else if (   uSegType == (X86_SEL_TYPE_CODE | X86_SEL_TYPE_ACCESSED)
+                     || uSegType == (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_ACCESSED))
+            {
+                X86DESCATTR AttrSs; AttrSs.u = pVmcs->u32GuestSsAttr;
+                if (uDpl == AttrSs.n.u2Dpl)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDplEqSs);
+            }
+            else if ((uSegType & (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF | X86_SEL_TYPE_ACCESSED))
+                              == (X86_SEL_TYPE_CODE | X86_SEL_TYPE_CONF | X86_SEL_TYPE_ACCESSED))
+            {
+                X86DESCATTR AttrSs; AttrSs.u = pVmcs->u32GuestSsAttr;
+                if (uDpl <= AttrSs.n.u2Dpl)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDplLtSs);
+            }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsType);
+
+            /* Def/Big. */
+            if (   fGstInLongMode
+                && fSegLong)
+            {
+                if (uDefBig == 0)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsDefBig);
+            }
+        }
+        else if (iSegReg == X86_SREG_SS)
+        {
+            /* Segment Type. */
+            if (   !fUsable
+                || uSegType == (X86_SEL_TYPE_RW   | X86_SEL_TYPE_ACCESSED)
+                || uSegType == (X86_SEL_TYPE_DOWN | X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrSsType);
+
+            /* DPL. */
+            if (!fUnrestrictedGuest)
+            {
+                if (uDpl == (SelReg.Sel & X86_SEL_RPL))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrSsDplEqRpl);
+            }
+            X86DESCATTR AttrCs; AttrCs.u = pVmcs->u32GuestCsAttr;
+            if (   AttrCs.n.u4Type == (X86_SEL_TYPE_RW | X86_SEL_TYPE_ACCESSED)
+                || !(pVmcs->u64GuestCr0.u & X86_CR0_PE))
+            {
+                if (uDpl == 0)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrSsDplZero);
+            }
+        }
+        else
+        {
+            /* DS, ES, FS, GS. */
+            if (fUsable)
+            {
+                /* Segment type. */
+                if (uSegType & X86_SEL_TYPE_ACCESSED)
+                { /* likely */ }
+                else
+                {
+                    VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrTypeAcc(iSegReg);
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+                }
+
+                if (   !(uSegType & X86_SEL_TYPE_CODE)
+                    ||  (uSegType & X86_SEL_TYPE_READ))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrCsTypeRead);
+
+                /* DPL. */
+                if (   !fUnrestrictedGuest
+                    && uSegType <= (X86_SEL_TYPE_CODE | X86_SEL_TYPE_READ | X86_SEL_TYPE_ACCESSED))
+                {
+                    if (uDpl >= (SelReg.Sel & X86_SEL_RPL))
+                    { /* likely */ }
+                    else
+                    {
+                        VMXVDIAG const enmDiag = iemVmxGetDiagVmentrySegAttrDplRpl(iSegReg);
+                        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+                    }
+                }
+            }
+        }
+    }
+
+    /*
+     * LDTR.
+     */
+    {
+        CPUMSELREG Ldtr;
+        Ldtr.Sel      = pVmcs->GuestLdtr;
+        Ldtr.u32Limit = pVmcs->u32GuestLdtrLimit;
+        Ldtr.u64Base  = pVmcs->u64GuestLdtrBase.u;
+        Ldtr.Attr.u   = pVmcs->u32GuestLdtrAttr;
+
+        if (!Ldtr.Attr.n.u1Unusable)
+        {
+            /* Selector. */
+            if (!(Ldtr.Sel & X86_SEL_LDT))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegSelLdtr);
+
+            /* Base. */
+            if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+            {
+                if (X86_IS_CANONICAL(Ldtr.u64Base))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegBaseLdtr);
+            }
+
+            /* Attributes. */
+            /* Reserved bits (bits 31:17 and bits 11:8). */
+            if (!(Ldtr.Attr.u & 0xfffe0f00))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrRsvd);
+
+            if (Ldtr.Attr.n.u4Type == X86_SEL_TYPE_SYS_LDT)
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrType);
+
+            if (!Ldtr.Attr.n.u1DescType)
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrDescType);
+
+            if (Ldtr.Attr.n.u1Present)
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrPresent);
+
+            if (   ((Ldtr.u32Limit & 0x00000fff) == 0x00000fff || !Ldtr.Attr.n.u1Granularity)
+                && ((Ldtr.u32Limit & 0xfff00000) == 0x00000000 ||  Ldtr.Attr.n.u1Granularity))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrLdtrGran);
+        }
+    }
+
+    /*
+     * TR.
+     */
+    {
+        CPUMSELREG Tr;
+        Tr.Sel      = pVmcs->GuestTr;
+        Tr.u32Limit = pVmcs->u32GuestTrLimit;
+        Tr.u64Base  = pVmcs->u64GuestTrBase.u;
+        Tr.Attr.u   = pVmcs->u32GuestTrAttr;
+
+        /* Selector. */
+        if (!(Tr.Sel & X86_SEL_LDT))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegSelTr);
+
+        /* Base. */
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+        {
+            if (X86_IS_CANONICAL(Tr.u64Base))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegBaseTr);
+        }
+
+        /* Attributes. */
+        /* Reserved bits (bits 31:17 and bits 11:8). */
+        if (!(Tr.Attr.u & 0xfffe0f00))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrRsvd);
+
+        if (!Tr.Attr.n.u1Unusable)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrUnusable);
+
+        if (   Tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY
+            || (   !fGstInLongMode
+                && Tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_BUSY))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrType);
+
+        if (!Tr.Attr.n.u1DescType)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrDescType);
+
+        if (Tr.Attr.n.u1Present)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrPresent);
+
+        if (   ((Tr.u32Limit & 0x00000fff) == 0x00000fff || !Tr.Attr.n.u1Granularity)
+            && ((Tr.u32Limit & 0xfff00000) == 0x00000000 ||  Tr.Attr.n.u1Granularity))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestSegAttrTrGran);
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks guest GDTR and IDTR as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestGdtrIdtr(PVMCPUCC pVCpu,  const char *pszInstr)
+{
+    /*
+     * GDTR and IDTR.
+     * See Intel spec. 26.3.1.3 "Checks on Guest Descriptor-Table Registers".
+     */
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char *const pszFailure = "VM-exit";
+
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        /* Base. */
+        if (X86_IS_CANONICAL(pVmcs->u64GuestGdtrBase.u))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestGdtrBase);
+
+        if (X86_IS_CANONICAL(pVmcs->u64GuestIdtrBase.u))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIdtrBase);
+    }
+
+    /* Limit. */
+    if (!RT_HI_U16(pVmcs->u32GuestGdtrLimit))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestGdtrLimit);
+
+    if (!RT_HI_U16(pVmcs->u32GuestIdtrLimit))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIdtrLimit);
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks guest RIP and RFLAGS as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestRipRFlags(PVMCPUCC pVCpu, const char *pszInstr)
+{
+    /*
+     * RIP and RFLAGS.
+     * See Intel spec. 26.3.1.4 "Checks on Guest RIP and RFLAGS".
+     */
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char *const pszFailure = "VM-exit";
+    bool const fGstInLongMode    = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+
+    /* RIP. */
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        X86DESCATTR AttrCs;
+        AttrCs.u = pVmcs->u32GuestCsAttr;
+        if (   !fGstInLongMode
+            || !AttrCs.n.u1Long)
+        {
+            if (!RT_HI_U32(pVmcs->u64GuestRip.u))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRipRsvd);
+        }
+
+        if (   fGstInLongMode
+            && AttrCs.n.u1Long)
+        {
+            Assert(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxLinearAddrWidth == 48);   /* Canonical. */
+            if (   IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxLinearAddrWidth < 64
+                && X86_IS_CANONICAL(pVmcs->u64GuestRip.u))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRip);
+        }
+    }
+
+    /* RFLAGS (bits 63:22 (or 31:22), bits 15, 5, 3 are reserved, bit 1 MB1). */
+    uint64_t const uGuestRFlags = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode ? pVmcs->u64GuestRFlags.u
+                                : pVmcs->u64GuestRFlags.s.Lo;
+    if (   !(uGuestRFlags & ~(X86_EFL_LIVE_MASK | X86_EFL_RA1_MASK))
+        &&  (uGuestRFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK)
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRFlagsRsvd);
+
+    if (!(uGuestRFlags & X86_EFL_VM))
+    { /* likely */ }
+    else
+    {
+        if (   fGstInLongMode
+            || !(pVmcs->u64GuestCr0.u & X86_CR0_PE))
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRFlagsVm);
+    }
+
+    if (VMX_ENTRY_INT_INFO_IS_EXT_INT(pVmcs->u32EntryIntInfo))
+    {
+        if (uGuestRFlags & X86_EFL_IF)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestRFlagsIf);
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks guest non-register state as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+DECLINLINE(int) iemVmxVmentryCheckGuestNonRegState(PVMCPUCC pVCpu,  const char *pszInstr)
+{
+    /*
+     * Guest non-register state.
+     * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
+     */
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char *const pszFailure = "VM-exit";
+
+    /*
+     * Activity state.
+     */
+    uint64_t const u64GuestVmxMiscMsr = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Misc;
+    uint32_t const fActivityStateMask = RT_BF_GET(u64GuestVmxMiscMsr, VMX_BF_MISC_ACTIVITY_STATES);
+    if (!(pVmcs->u32GuestActivityState & fActivityStateMask))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateRsvd);
+
+    X86DESCATTR AttrSs; AttrSs.u = pVmcs->u32GuestSsAttr;
+    if (   !AttrSs.n.u2Dpl
+        || pVmcs->u32GuestActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT)
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateSsDpl);
+
+    if (   pVmcs->u32GuestIntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI
+        || pVmcs->u32GuestIntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
+    {
+        if (pVmcs->u32GuestActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateStiMovSs);
+    }
+
+    if (VMX_ENTRY_INT_INFO_IS_VALID(pVmcs->u32EntryIntInfo))
+    {
+        uint8_t const uType   = VMX_ENTRY_INT_INFO_TYPE(pVmcs->u32EntryIntInfo);
+        uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(pVmcs->u32EntryIntInfo);
+        AssertCompile(VMX_V_GUEST_ACTIVITY_STATE_MASK == (VMX_VMCS_GUEST_ACTIVITY_HLT | VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN));
+        switch (pVmcs->u32GuestActivityState)
+        {
+            case VMX_VMCS_GUEST_ACTIVITY_HLT:
+            {
+                if (   uType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT
+                    || uType == VMX_ENTRY_INT_INFO_TYPE_NMI
+                    || (   uType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
+                        && (   uVector == X86_XCPT_DB
+                            || uVector == X86_XCPT_MC))
+                    || (   uType   == VMX_ENTRY_INT_INFO_TYPE_OTHER_EVENT
+                        && uVector == VMX_ENTRY_INT_INFO_VECTOR_MTF))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateHlt);
+                break;
+            }
+
+            case VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN:
+            {
+                if (   uType == VMX_ENTRY_INT_INFO_TYPE_NMI
+                    || (   uType   == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
+                        && uVector == X86_XCPT_MC))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestActStateShutdown);
+                break;
+            }
+
+            case VMX_VMCS_GUEST_ACTIVITY_ACTIVE:
+            default:
+                break;
+        }
+    }
+
+    /*
+     * Interruptibility state.
+     */
+    if (!(pVmcs->u32GuestIntrState & ~VMX_VMCS_GUEST_INT_STATE_MASK))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateRsvd);
+
+    if ((pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
+                                 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateStiMovSs);
+
+    if (    (pVmcs->u64GuestRFlags.u & X86_EFL_IF)
+        || !(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateRFlagsSti);
+
+    if (VMX_ENTRY_INT_INFO_IS_VALID(pVmcs->u32EntryIntInfo))
+    {
+        uint8_t const uType = VMX_ENTRY_INT_INFO_TYPE(pVmcs->u32EntryIntInfo);
+        if (uType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
+        {
+            if (!(pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateExtInt);
+        }
+        else if (uType == VMX_ENTRY_INT_INFO_TYPE_NMI)
+        {
+            if (!(pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)))
+            { /* likely */ }
+            else
+            {
+                /*
+                 * We don't support injecting NMIs when blocking-by-STI would be in effect.
+                 * We update the Exit qualification only when blocking-by-STI is set
+                 * without blocking-by-MovSS being set. Although in practise it  does not
+                 * make much difference since the order of checks are implementation defined.
+                 */
+                if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
+                    iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_NMI_INJECT);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateNmi);
+            }
+
+            if (   !(pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+                || !(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI))
+            { /* likely */ }
+            else
+               IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateVirtNmi);
+        }
+    }
+
+    /* We don't support SMM yet. So blocking-by-SMIs must not be set. */
+    if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateSmi);
+
+    /* We don't support SGX yet. So enclave-interruption must not be set. */
+    if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_ENCLAVE))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestIntStateEnclave);
+
+    /*
+     * Pending debug exceptions.
+     */
+    uint64_t const uPendingDbgXcpts = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode
+                                    ? pVmcs->u64GuestPendingDbgXcpts.u
+                                    : pVmcs->u64GuestPendingDbgXcpts.s.Lo;
+    if (!(uPendingDbgXcpts & ~VMX_VMCS_GUEST_PENDING_DEBUG_VALID_MASK))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptRsvd);
+
+    if (   (pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
+        || pVmcs->u32GuestActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
+    {
+        if (   (pVmcs->u64GuestRFlags.u & X86_EFL_TF)
+            && !(pVmcs->u64GuestDebugCtlMsr.u & MSR_IA32_DEBUGCTL_BTF)
+            && !(uPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS))
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptBsTf);
+
+        if (   (   !(pVmcs->u64GuestRFlags.u & X86_EFL_TF)
+                ||  (pVmcs->u64GuestDebugCtlMsr.u & MSR_IA32_DEBUGCTL_BTF))
+            && (uPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS))
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptBsNoTf);
+    }
+
+    /* We don't support RTM (Real-time Transactional Memory) yet. */
+    if (!(uPendingDbgXcpts & VMX_VMCS_GUEST_PENDING_DEBUG_RTM))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPndDbgXcptRtm);
+
+    /*
+     * VMCS link pointer.
+     */
+    if (pVmcs->u64VmcsLinkPtr.u != UINT64_C(0xffffffffffffffff))
+    {
+        RTGCPHYS const GCPhysShadowVmcs = pVmcs->u64VmcsLinkPtr.u;
+        /* We don't support SMM yet (so VMCS link pointer cannot be the current VMCS). */
+        if (GCPhysShadowVmcs != IEM_VMX_GET_CURRENT_VMCS(pVCpu))
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrCurVmcs);
+        }
+
+        /* Validate the address. */
+        if (   !(GCPhysShadowVmcs & X86_PAGE_4K_OFFSET_MASK)
+            && !(GCPhysShadowVmcs >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+            &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysShadowVmcs))
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVmcsLinkPtr);
+        }
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * Checks guest PDPTEs as part of VM-entry.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pszInstr        The VMX instruction name (for logging purposes).
+ */
+static int iemVmxVmentryCheckGuestPdptes(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    /*
+     * Guest PDPTEs.
+     * See Intel spec. 26.3.1.5 "Checks on Guest Page-Directory-Pointer-Table Entries".
+     */
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char * const pszFailure = "VM-exit";
+
+    /*
+     * When EPT is used, we need to validate the PAE PDPTEs provided in the VMCS.
+     * Otherwise, we load any PAE PDPTEs referenced by CR3 at a later point.
+     */
+    if (   iemVmxVmcsIsGuestPaePagingEnabled(pVmcs)
+        && (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT))
+    {
+        /* Get PDPTEs from the VMCS. */
+        X86PDPE aPaePdptes[X86_PG_PAE_PDPE_ENTRIES];
+        aPaePdptes[0].u = pVmcs->u64GuestPdpte0.u;
+        aPaePdptes[1].u = pVmcs->u64GuestPdpte1.u;
+        aPaePdptes[2].u = pVmcs->u64GuestPdpte2.u;
+        aPaePdptes[3].u = pVmcs->u64GuestPdpte3.u;
+
+        /* Check validity of the PDPTEs. */
+        if (PGMGstArePaePdpesValid(pVCpu, &aPaePdptes[0]))
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_PDPTE);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPdpte);
+        }
+    }
+
+    NOREF(pszFailure);
+    NOREF(pszInstr);
+    return VINF_SUCCESS;
+}
+#endif  /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+
+/**
+ * Checks guest-state as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pszInstr        The VMX instruction name (for logging purposes).
+ */
+static int iemVmxVmentryCheckGuestState(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    int rc = iemVmxVmentryCheckGuestControlRegsMsrs(pVCpu, pszInstr);
+    if (RT_SUCCESS(rc))
+    {
+        rc = iemVmxVmentryCheckGuestSegRegs(pVCpu, pszInstr);
+        if (RT_SUCCESS(rc))
+        {
+            rc = iemVmxVmentryCheckGuestGdtrIdtr(pVCpu, pszInstr);
+            if (RT_SUCCESS(rc))
+            {
+                rc = iemVmxVmentryCheckGuestRipRFlags(pVCpu, pszInstr);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = iemVmxVmentryCheckGuestNonRegState(pVCpu, pszInstr);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+                    if (RT_SUCCESS(rc))
+                        rc = iemVmxVmentryCheckGuestPdptes(pVCpu, pszInstr);
+#endif
+                }
+            }
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Checks host-state as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+static int iemVmxVmentryCheckHostState(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    /*
+     * Host Control Registers and MSRs.
+     * See Intel spec. 26.2.2 "Checks on Host Control Registers and MSRs".
+     */
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char * const pszFailure = "VMFail";
+
+    /* CR0 reserved bits. */
+    {
+        /* CR0 MB1 bits. */
+        uint64_t const u64Cr0Fixed0 = iemVmxGetCr0Fixed0(pVCpu, true /* fVmxNonRootMode */);
+        if ((pVmcs->u64HostCr0.u & u64Cr0Fixed0) == u64Cr0Fixed0)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr0Fixed0);
+
+        /* CR0 MBZ bits. */
+        uint64_t const u64Cr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+        if (!(pVmcs->u64HostCr0.u & ~u64Cr0Fixed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr0Fixed1);
+    }
+
+    /* CR4 reserved bits. */
+    {
+        /* CR4 MB1 bits. */
+        uint64_t const u64Cr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+        if ((pVmcs->u64HostCr4.u & u64Cr4Fixed0) == u64Cr4Fixed0)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Fixed0);
+
+        /* CR4 MBZ bits. */
+        uint64_t const u64Cr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+        if (!(pVmcs->u64HostCr4.u & ~u64Cr4Fixed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Fixed1);
+    }
+
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        /* CR3 reserved bits. */
+        if (!(pVmcs->u64HostCr3.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr3);
+
+        /* SYSENTER ESP and SYSENTER EIP. */
+        if (   X86_IS_CANONICAL(pVmcs->u64HostSysenterEsp.u)
+            && X86_IS_CANONICAL(pVmcs->u64HostSysenterEip.u))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSysenterEspEip);
+    }
+
+    /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */
+    Assert(!(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_PERF_MSR));
+
+    /* PAT MSR. */
+    if (   !(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_PAT_MSR)
+        ||  CPUMIsPatMsrValid(pVmcs->u64HostPatMsr.u))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostPatMsr);
+
+    /* EFER MSR. */
+    bool const     fHostInLongMode = RT_BOOL(pVmcs->u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE);
+    uint64_t const uValidEferMask  = CPUMGetGuestEferMsrValidMask(pVCpu->CTX_SUFF(pVM));
+    if (pVmcs->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
+    {
+        if (!(pVmcs->u64HostEferMsr.u & ~uValidEferMask))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostEferMsrRsvd);
+
+        bool const fHostLma = RT_BOOL(pVmcs->u64HostEferMsr.u & MSR_K6_EFER_LMA);
+        bool const fHostLme = RT_BOOL(pVmcs->u64HostEferMsr.u & MSR_K6_EFER_LME);
+        if (   fHostInLongMode == fHostLma
+            && fHostInLongMode == fHostLme)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostEferMsr);
+    }
+
+    /*
+     * Host Segment and Descriptor-Table Registers.
+     * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
+     */
+    /* Selector RPL and TI. */
+    if (   !(pVmcs->HostCs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostSs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostDs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostEs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostFs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostGs & (X86_SEL_RPL | X86_SEL_LDT))
+        && !(pVmcs->HostTr & (X86_SEL_RPL | X86_SEL_LDT)))
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSel);
+
+    /* CS and TR selectors cannot be 0. */
+    if (   pVmcs->HostCs
+        && pVmcs->HostTr)
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCsTr);
+
+    /* SS cannot be 0 if 32-bit host. */
+    if (   fHostInLongMode
+        || pVmcs->HostSs)
+    { /* likely */ }
+    else
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSs);
+
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        /* FS, GS, GDTR, IDTR, TR base address. */
+        if (   X86_IS_CANONICAL(pVmcs->u64HostFsBase.u)
+            && X86_IS_CANONICAL(pVmcs->u64HostFsBase.u)
+            && X86_IS_CANONICAL(pVmcs->u64HostGdtrBase.u)
+            && X86_IS_CANONICAL(pVmcs->u64HostIdtrBase.u)
+            && X86_IS_CANONICAL(pVmcs->u64HostTrBase.u))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostSegBase);
+    }
+
+    /*
+     * Host address-space size for 64-bit CPUs.
+     * See Intel spec. 26.2.4 "Checks Related to Address-Space Size".
+     */
+    bool const fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        bool const fCpuInLongMode = CPUMIsGuestInLongMode(pVCpu);
+
+        /* Logical processor in IA-32e mode. */
+        if (fCpuInLongMode)
+        {
+            if (fHostInLongMode)
+            {
+                /* PAE must be set. */
+                if (pVmcs->u64HostCr4.u & X86_CR4_PAE)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Pae);
+
+                /* RIP must be canonical. */
+                if (X86_IS_CANONICAL(pVmcs->u64HostRip.u))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostRip);
+            }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostLongMode);
+        }
+        else
+        {
+            /* Logical processor is outside IA-32e mode. */
+            if (   !fGstInLongMode
+                && !fHostInLongMode)
+            {
+                /* PCIDE should not be set. */
+                if (!(pVmcs->u64HostCr4.u & X86_CR4_PCIDE))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostCr4Pcide);
+
+                /* The high 32-bits of RIP MBZ. */
+                if (!pVmcs->u64HostRip.s.Hi)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostRipRsvd);
+            }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostGuestLongMode);
+        }
+    }
+    else
+    {
+        /* Host address-space size for 32-bit CPUs. */
+        if (   !fGstInLongMode
+            && !fHostInLongMode)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_HostGuestLongModeNoCpu);
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * Checks the EPT pointer VMCS field as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uEptPtr         The EPT pointer to check.
+ * @param   penmVmxDiag     Where to store the diagnostic reason on failure (not
+ *                          updated on success). Optional, can be NULL.
+ */
+static int iemVmxVmentryCheckEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr, VMXVDIAG *penmVmxDiag) RT_NOEXCEPT
+{
+    VMXVDIAG enmVmxDiag;
+
+    /* Reserved bits. */
+    uint8_t const  cMaxPhysAddrWidth = IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cMaxPhysAddrWidth;
+    uint64_t const fValidMask        = VMX_EPTP_VALID_MASK & ~(UINT64_MAX << cMaxPhysAddrWidth);
+    if (uEptPtr & fValidMask)
+    {
+        /* Memory Type. */
+        uint64_t const fCaps    = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64EptVpidCaps;
+        uint8_t const  fMemType = RT_BF_GET(uEptPtr, VMX_BF_EPTP_MEMTYPE);
+        if (   (   fMemType == VMX_EPTP_MEMTYPE_WB
+                && RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_MEMTYPE_WB))
+            || (   fMemType == VMX_EPTP_MEMTYPE_UC
+                && RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_MEMTYPE_UC)))
+        {
+            /*
+             * Page walk length (PML4).
+             * Intel used to specify bit 7 of IA32_VMX_EPT_VPID_CAP as page walk length
+             * of 5 but that seems to be removed from the latest specs. leaving only PML4
+             * as the maximum supported page-walk level hence we hardcode it as 3 (1 less than 4)
+             */
+            Assert(RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_PAGE_WALK_LENGTH_4));
+            if (RT_BF_GET(uEptPtr, VMX_BF_EPTP_PAGE_WALK_LENGTH) == 3)
+            {
+                /* Access and dirty bits support in EPT structures. */
+                if (   !RT_BF_GET(uEptPtr, VMX_BF_EPTP_ACCESS_DIRTY)
+                    ||  RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_ACCESS_DIRTY))
+                    return VINF_SUCCESS;
+
+                enmVmxDiag = kVmxVDiag_Vmentry_EptpAccessDirty;
+            }
+            else
+                enmVmxDiag = kVmxVDiag_Vmentry_EptpPageWalkLength;
+        }
+        else
+            enmVmxDiag = kVmxVDiag_Vmentry_EptpMemType;
+    }
+    else
+        enmVmxDiag = kVmxVDiag_Vmentry_EptpRsvd;
+
+    if (penmVmxDiag)
+        *penmVmxDiag = enmVmxDiag;
+    return VERR_VMX_VMENTRY_FAILED;
+}
+#endif
+
+
+/**
+ * Checks VMCS controls fields as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ *
+ * @remarks This may update secondary-processor based VM-execution control fields
+ *          in the current VMCS if necessary.
+ */
+static int iemVmxVmentryCheckCtls(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char * const pszFailure = "VMFail";
+    bool const fVmxTrueMsrs       = RT_BOOL(pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Basic & VMX_BF_BASIC_TRUE_CTLS_MASK);
+
+    /*
+     * VM-execution controls.
+     * See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
+     */
+    {
+        /* Pin-based VM-execution controls. */
+        {
+            VMXCTLSMSR const PinCtls = fVmxTrueMsrs ? pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.TruePinCtls
+                                                    : pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.PinCtls;
+            if (!(~pVmcs->u32PinCtls & PinCtls.n.allowed0))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_PinCtlsDisallowed0);
+
+            if (!(pVmcs->u32PinCtls & ~PinCtls.n.allowed1))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_PinCtlsAllowed1);
+        }
+
+        /* Processor-based VM-execution controls. */
+        {
+            VMXCTLSMSR const ProcCtls = fVmxTrueMsrs ? pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.TrueProcCtls
+                                                     : pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.ProcCtls;
+            if (!(~pVmcs->u32ProcCtls & ProcCtls.n.allowed0))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtlsDisallowed0);
+
+            if (!(pVmcs->u32ProcCtls & ~ProcCtls.n.allowed1))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtlsAllowed1);
+        }
+
+        /* Secondary processor-based VM-execution controls. */
+        if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
+        {
+            VMXCTLSMSR const ProcCtls2 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.ProcCtls2;
+            if (!(~pVmcs->u32ProcCtls2 & ProcCtls2.n.allowed0))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtls2Disallowed0);
+
+            if (!(pVmcs->u32ProcCtls2 & ~ProcCtls2.n.allowed1))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ProcCtls2Allowed1);
+        }
+        else
+            Assert(!pVmcs->u32ProcCtls2);
+
+        /* CR3-target count. */
+        if (pVmcs->u32Cr3TargetCount <= VMX_V_CR3_TARGET_COUNT)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_Cr3TargetCount);
+
+        /* I/O bitmaps physical addresses. */
+        if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS)
+        {
+            RTGCPHYS const GCPhysIoBitmapA = pVmcs->u64AddrIoBitmapA.u;
+            if (   !(GCPhysIoBitmapA & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysIoBitmapA >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysIoBitmapA))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrIoBitmapA);
+
+            RTGCPHYS const GCPhysIoBitmapB = pVmcs->u64AddrIoBitmapB.u;
+            if (   !(GCPhysIoBitmapB & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysIoBitmapB >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysIoBitmapB))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrIoBitmapB);
+        }
+
+        /* MSR bitmap physical address. */
+        if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+        {
+            RTGCPHYS const GCPhysMsrBitmap = pVmcs->u64AddrMsrBitmap.u;
+            if (   !(GCPhysMsrBitmap & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysMsrBitmap >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysMsrBitmap))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrMsrBitmap);
+        }
+
+        /* TPR shadow related controls. */
+        if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+        {
+            /* Virtual-APIC page physical address. */
+            RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+            if (   !(GCPhysVirtApic & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysVirtApic >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVirtApic))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVirtApicPage);
+
+            /* TPR threshold bits 31:4 MBZ without virtual-interrupt delivery. */
+            if (   !(pVmcs->u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)
+                ||  (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_TprThresholdRsvd);
+
+            /* The rest done XXX document */
+        }
+        else
+        {
+            if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE)
+                && !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+                && !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY))
+            { /* likely */ }
+            else
+            {
+                if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE)
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtX2ApicTprShadow);
+                if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_APIC_REG_VIRT)
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ApicRegVirt);
+                Assert(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtIntDelivery);
+            }
+        }
+
+        /* NMI exiting and virtual-NMIs. */
+        if (    (pVmcs->u32PinCtls & VMX_PIN_CTLS_NMI_EXIT)
+            || !(pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtNmi);
+
+        /* Virtual-NMIs and NMI-window exiting. */
+        if (    (pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+            || !(pVmcs->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_NmiWindowExit);
+
+        /* Virtualize APIC accesses. */
+        if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+        {
+            /* APIC-access physical address. */
+            RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
+            if (   !(GCPhysApicAccess & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysApicAccess >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrApicAccess);
+
+            /*
+             * Disallow APIC-access page and virtual-APIC page from being the same address.
+             * Note! This is not an Intel requirement, but one imposed by our implementation.
+             */
+            /** @todo r=ramshankar: This is done primarily to simplify recursion scenarios while
+             *        redirecting accesses between the APIC-access page and the virtual-APIC
+             *        page. If any nested hypervisor requires this, we can implement it later. */
+            if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+            {
+                RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+                if (GCPhysVirtApic != GCPhysApicAccess)
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrApicAccessEqVirtApic);
+            }
+        }
+
+        /* Virtualize-x2APIC mode is mutually exclusive with virtualize-APIC accesses. */
+        if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_X2APIC_MODE)
+            || !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtX2ApicVirtApic);
+
+        /* Virtual-interrupt delivery requires external interrupt exiting. */
+        if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY)
+            ||  (pVmcs->u32PinCtls & VMX_PIN_CTLS_EXT_INT_EXIT))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtX2ApicVirtApic);
+
+        /* VPID. */
+        if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VPID)
+            || pVmcs->u16Vpid != 0)
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_Vpid);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        /* Extended-Page-Table Pointer (EPTP). */
+        if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT)
+        {
+            VMXVDIAG enmVmxDiag;
+            int const rc = iemVmxVmentryCheckEptPtr(pVCpu, pVmcs->u64EptPtr.u, &enmVmxDiag);
+            if (RT_SUCCESS(rc))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmVmxDiag);
+        }
+#else
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT));
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST));
+#endif
+        Assert(!(pVmcs->u32PinCtls & VMX_PIN_CTLS_POSTED_INT));             /* We don't support posted interrupts yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_PML));                /* We don't support PML yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMFUNC));             /* We don't support VM functions yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT_XCPT_VE));        /* We don't support EPT-violation #VE yet. */
+        Assert(!(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_TSC_SCALING));        /* We don't support TSC-scaling yet. */
+
+        /* VMCS shadowing. */
+        if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+        {
+            /* VMREAD-bitmap physical address. */
+            RTGCPHYS const GCPhysVmreadBitmap = pVmcs->u64AddrVmreadBitmap.u;
+            if (   !(GCPhysVmreadBitmap & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysVmreadBitmap >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmreadBitmap))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVmreadBitmap);
+
+            /* VMWRITE-bitmap physical address. */
+            RTGCPHYS const GCPhysVmwriteBitmap = pVmcs->u64AddrVmreadBitmap.u;
+            if (   !(GCPhysVmwriteBitmap & X86_PAGE_4K_OFFSET_MASK)
+                && !(GCPhysVmwriteBitmap >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmwriteBitmap))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrVmwriteBitmap);
+        }
+    }
+
+    /*
+     * VM-exit controls.
+     * See Intel spec. 26.2.1.2 "VM-Exit Control Fields".
+     */
+    {
+        VMXCTLSMSR const ExitCtls = fVmxTrueMsrs ? pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.TrueExitCtls
+                                                 : pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.ExitCtls;
+        if (!(~pVmcs->u32ExitCtls & ExitCtls.n.allowed0))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ExitCtlsDisallowed0);
+
+        if (!(pVmcs->u32ExitCtls & ~ExitCtls.n.allowed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_ExitCtlsAllowed1);
+
+        /* Save preemption timer without activating it. */
+        if (    (pVmcs->u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER)
+            || !(pVmcs->u32ProcCtls & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_SavePreemptTimer);
+
+        /* VM-exit MSR-store count and VM-exit MSR-store area address. */
+        if (pVmcs->u32ExitMsrStoreCount)
+        {
+            if (   !(pVmcs->u64AddrExitMsrStore.u & VMX_AUTOMSR_OFFSET_MASK)
+                && !(pVmcs->u64AddrExitMsrStore.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), pVmcs->u64AddrExitMsrStore.u))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrExitMsrStore);
+        }
+
+        /* VM-exit MSR-load count and VM-exit MSR-load area address. */
+        if (pVmcs->u32ExitMsrLoadCount)
+        {
+            if (   !(pVmcs->u64AddrExitMsrLoad.u & VMX_AUTOMSR_OFFSET_MASK)
+                && !(pVmcs->u64AddrExitMsrLoad.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), pVmcs->u64AddrExitMsrLoad.u))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrExitMsrLoad);
+        }
+    }
+
+    /*
+     * VM-entry controls.
+     * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
+     */
+    {
+        VMXCTLSMSR const EntryCtls = fVmxTrueMsrs ? pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.TrueEntryCtls
+                                                  : pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.EntryCtls;
+        if (!(~pVmcs->u32EntryCtls & EntryCtls.n.allowed0))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryCtlsDisallowed0);
+
+        if (!(pVmcs->u32EntryCtls & ~EntryCtls.n.allowed1))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryCtlsAllowed1);
+
+        /* Event injection. */
+        uint32_t const uIntInfo = pVmcs->u32EntryIntInfo;
+        if (RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_VALID))
+        {
+            /* Type and vector. */
+            uint8_t const uType   = RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_TYPE);
+            uint8_t const uVector = RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_VECTOR);
+            uint8_t const uRsvd   = RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_RSVD_12_30);
+            if (   !uRsvd
+                && VMXIsEntryIntInfoTypeValid(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxMonitorTrapFlag, uType)
+                && VMXIsEntryIntInfoVectorValid(uVector, uType))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryIntInfoTypeVecRsvd);
+
+            /* Exception error code. */
+            if (RT_BF_GET(uIntInfo, VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID))
+            {
+                /* Delivery possible only in Unrestricted-guest mode when CR0.PE is set. */
+                if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_UNRESTRICTED_GUEST)
+                    ||  (pVmcs->u64GuestCr0.s.Lo & X86_CR0_PE))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryIntInfoErrCodePe);
+
+                /* Exceptions that provide an error code. */
+                if (   uType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
+                    && (   uVector == X86_XCPT_DF
+                        || uVector == X86_XCPT_TS
+                        || uVector == X86_XCPT_NP
+                        || uVector == X86_XCPT_SS
+                        || uVector == X86_XCPT_GP
+                        || uVector == X86_XCPT_PF
+                        || uVector == X86_XCPT_AC))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryIntInfoErrCodeVec);
+
+                /* Exception error-code reserved bits. */
+                if (!(pVmcs->u32EntryXcptErrCode & ~VMX_ENTRY_INT_XCPT_ERR_CODE_VALID_MASK))
+                { /* likely */ }
+                else
+                    IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryXcptErrCodeRsvd);
+
+                /* Injecting a software interrupt, software exception or privileged software exception. */
+                if (   uType == VMX_ENTRY_INT_INFO_TYPE_SW_INT
+                    || uType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT
+                    || uType == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT)
+                {
+                    /* Instruction length must be in the range 0-15. */
+                    if (pVmcs->u32EntryInstrLen <= VMX_ENTRY_INSTR_LEN_MAX)
+                    { /* likely */ }
+                    else
+                        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryInstrLen);
+
+                    /* However, instruction length of 0 is allowed only when its CPU feature is present. */
+                    if (   pVmcs->u32EntryInstrLen != 0
+                        || IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxEntryInjectSoftInt)
+                    { /* likely */ }
+                    else
+                        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_EntryInstrLenZero);
+                }
+            }
+        }
+
+        /* VM-entry MSR-load count and VM-entry MSR-load area address. */
+        if (pVmcs->u32EntryMsrLoadCount)
+        {
+            if (   !(pVmcs->u64AddrEntryMsrLoad.u & VMX_AUTOMSR_OFFSET_MASK)
+                && !(pVmcs->u64AddrEntryMsrLoad.u >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth)
+                &&  PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), pVmcs->u64AddrEntryMsrLoad.u))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrEntryMsrLoad);
+        }
+
+        Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM));           /* We don't support SMM yet. */
+        Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON));    /* We don't support dual-monitor treatment yet. */
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the guest control registers, debug register and some MSRs as part of
+ * VM-entry.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxVmentryLoadGuestControlRegsMsrs(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /*
+     * Load guest control registers, debug registers and MSRs.
+     * See Intel spec. 26.3.2.1 "Loading Guest Control Registers, Debug Registers and MSRs".
+     */
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+    uint64_t const uGstCr0 = (pVmcs->u64GuestCr0.u   & ~VMX_ENTRY_GUEST_CR0_IGNORE_MASK)
+                           | (pVCpu->cpum.GstCtx.cr0 &  VMX_ENTRY_GUEST_CR0_IGNORE_MASK);
+    pVCpu->cpum.GstCtx.cr0 = uGstCr0;
+    pVCpu->cpum.GstCtx.cr4 = pVmcs->u64GuestCr4.u;
+    pVCpu->cpum.GstCtx.cr3 = pVmcs->u64GuestCr3.u;
+
+    if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+        pVCpu->cpum.GstCtx.dr[7] = (pVmcs->u64GuestDr7.u & ~VMX_ENTRY_GUEST_DR7_MBZ_MASK) | VMX_ENTRY_GUEST_DR7_MB1_MASK;
+
+    pVCpu->cpum.GstCtx.SysEnter.eip = pVmcs->u64GuestSysenterEip.s.Lo;
+    pVCpu->cpum.GstCtx.SysEnter.esp = pVmcs->u64GuestSysenterEsp.s.Lo;
+    pVCpu->cpum.GstCtx.SysEnter.cs  = pVmcs->u32GuestSysenterCS;
+
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode)
+    {
+        /* FS base and GS base are loaded  while loading the rest of the guest segment registers. */
+
+        /* EFER MSR. */
+        if (!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR))
+        {
+            IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
+            uint64_t const uHostEfer      = pVCpu->cpum.GstCtx.msrEFER;
+            bool const     fGstInLongMode = RT_BOOL(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+            bool const     fGstPaging     = RT_BOOL(uGstCr0 & X86_CR0_PG);
+            if (fGstInLongMode)
+            {
+                /* If the nested-guest is in long mode, LMA and LME are both set. */
+                Assert(fGstPaging);
+                pVCpu->cpum.GstCtx.msrEFER = uHostEfer | (MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
+            }
+            else
+            {
+                /*
+                 * If the nested-guest is outside long mode:
+                 *   - With paging:    LMA is cleared, LME is cleared.
+                 *   - Without paging: LMA is cleared, LME is left unmodified.
+                 */
+                uint64_t const fLmaLmeMask = MSR_K6_EFER_LMA | (fGstPaging ? MSR_K6_EFER_LME : 0);
+                pVCpu->cpum.GstCtx.msrEFER = uHostEfer & ~fLmaLmeMask;
+            }
+        }
+        /* else: see below. */
+    }
+
+    /* PAT MSR. */
+    if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
+        pVCpu->cpum.GstCtx.msrPAT = pVmcs->u64GuestPatMsr.u;
+
+    /* EFER MSR. */
+    if (pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
+        pVCpu->cpum.GstCtx.msrEFER = pVmcs->u64GuestEferMsr.u;
+
+    /* We don't support IA32_PERF_GLOBAL_CTRL MSR yet. */
+    Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR));
+
+    /* We don't support IA32_BNDCFGS MSR yet. */
+    Assert(!(pVmcs->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_BNDCFGS_MSR));
+
+    /* Nothing to do for SMBASE register - We don't support SMM yet. */
+}
+
+
+/**
+ * Loads the guest segment registers, GDTR, IDTR, LDTR and TR as part of VM-entry.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxVmentryLoadGuestSegRegs(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /*
+     * Load guest segment registers, GDTR, IDTR, LDTR and TR.
+     * See Intel spec. 26.3.2.2 "Loading Guest Segment Registers and Descriptor-Table Registers".
+     */
+    /* CS, SS, ES, DS, FS, GS. */
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    for (unsigned iSegReg = 0; iSegReg < X86_SREG_COUNT; iSegReg++)
+    {
+        PCPUMSELREG pGstSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
+        CPUMSELREG  VmcsSelReg;
+        int rc = iemVmxVmcsGetGuestSegReg(pVmcs, iSegReg, &VmcsSelReg);
+        AssertRC(rc); NOREF(rc);
+        if (!(VmcsSelReg.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            pGstSelReg->Sel      = VmcsSelReg.Sel;
+            pGstSelReg->ValidSel = VmcsSelReg.Sel;
+            pGstSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+            pGstSelReg->u64Base  = VmcsSelReg.u64Base;
+            pGstSelReg->u32Limit = VmcsSelReg.u32Limit;
+            pGstSelReg->Attr.u   = VmcsSelReg.Attr.u;
+        }
+        else
+        {
+            pGstSelReg->Sel      = VmcsSelReg.Sel;
+            pGstSelReg->ValidSel = VmcsSelReg.Sel;
+            pGstSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+            switch (iSegReg)
+            {
+                case X86_SREG_CS:
+                    pGstSelReg->u64Base  = VmcsSelReg.u64Base;
+                    pGstSelReg->u32Limit = VmcsSelReg.u32Limit;
+                    pGstSelReg->Attr.u   = VmcsSelReg.Attr.u;
+                    break;
+
+                case X86_SREG_SS:
+                    pGstSelReg->u64Base  = VmcsSelReg.u64Base & UINT32_C(0xfffffff0);
+                    pGstSelReg->u32Limit = 0;
+                    pGstSelReg->Attr.u   = (VmcsSelReg.Attr.u & X86DESCATTR_DPL) | X86DESCATTR_D | X86DESCATTR_UNUSABLE;
+                    break;
+
+                case X86_SREG_ES:
+                case X86_SREG_DS:
+                    pGstSelReg->u64Base  = 0;
+                    pGstSelReg->u32Limit = 0;
+                    pGstSelReg->Attr.u   = X86DESCATTR_UNUSABLE;
+                    break;
+
+                case X86_SREG_FS:
+                case X86_SREG_GS:
+                    pGstSelReg->u64Base  = VmcsSelReg.u64Base;
+                    pGstSelReg->u32Limit = 0;
+                    pGstSelReg->Attr.u   = X86DESCATTR_UNUSABLE;
+                    break;
+            }
+            Assert(pGstSelReg->Attr.n.u1Unusable);
+        }
+    }
+
+    /* LDTR. */
+    pVCpu->cpum.GstCtx.ldtr.Sel      = pVmcs->GuestLdtr;
+    pVCpu->cpum.GstCtx.ldtr.ValidSel = pVmcs->GuestLdtr;
+    pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    if (!(pVmcs->u32GuestLdtrAttr & X86DESCATTR_UNUSABLE))
+    {
+        pVCpu->cpum.GstCtx.ldtr.u64Base  = pVmcs->u64GuestLdtrBase.u;
+        pVCpu->cpum.GstCtx.ldtr.u32Limit = pVmcs->u32GuestLdtrLimit;
+        pVCpu->cpum.GstCtx.ldtr.Attr.u   = pVmcs->u32GuestLdtrAttr;
+    }
+    else
+    {
+        pVCpu->cpum.GstCtx.ldtr.u64Base  = 0;
+        pVCpu->cpum.GstCtx.ldtr.u32Limit = 0;
+        pVCpu->cpum.GstCtx.ldtr.Attr.u   = X86DESCATTR_UNUSABLE;
+    }
+
+    /* TR. */
+    Assert(!(pVmcs->u32GuestTrAttr & X86DESCATTR_UNUSABLE));
+    pVCpu->cpum.GstCtx.tr.Sel      = pVmcs->GuestTr;
+    pVCpu->cpum.GstCtx.tr.ValidSel = pVmcs->GuestTr;
+    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.tr.u64Base  = pVmcs->u64GuestTrBase.u;
+    pVCpu->cpum.GstCtx.tr.u32Limit = pVmcs->u32GuestTrLimit;
+    pVCpu->cpum.GstCtx.tr.Attr.u   = pVmcs->u32GuestTrAttr;
+
+    /* GDTR. */
+    pVCpu->cpum.GstCtx.gdtr.cbGdt = pVmcs->u32GuestGdtrLimit;
+    pVCpu->cpum.GstCtx.gdtr.pGdt  = pVmcs->u64GuestGdtrBase.u;
+
+    /* IDTR. */
+    pVCpu->cpum.GstCtx.idtr.cbIdt = pVmcs->u32GuestIdtrLimit;
+    pVCpu->cpum.GstCtx.idtr.pIdt  = pVmcs->u64GuestIdtrBase.u;
+}
+
+
+/**
+ * Loads the guest MSRs from the VM-entry MSR-load area as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+static int iemVmxVmentryLoadGuestAutoMsrs(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    /*
+     * Load guest MSRs.
+     * See Intel spec. 26.4 "Loading MSRs".
+     */
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    const char *const pszFailure = "VM-exit";
+
+    /*
+     * The VM-entry MSR-load area address need not be a valid guest-physical address if the
+     * VM-entry MSR load count is 0. If this is the case, bail early without reading it.
+     * See Intel spec. 24.8.2 "VM-Entry Controls for MSRs".
+     */
+    uint32_t const cMsrs = RT_MIN(pVmcs->u32EntryMsrLoadCount, RT_ELEMENTS(pVCpu->cpum.GstCtx.hwvirt.vmx.aEntryMsrLoadArea));
+    if (!cMsrs)
+        return VINF_SUCCESS;
+
+    /*
+     * Verify the MSR auto-load count. Physical CPUs can behave unpredictably if the count is
+     * exceeded including possibly raising #MC exceptions during VMX transition. Our
+     * implementation shall fail VM-entry with an VMX_EXIT_ERR_MSR_LOAD VM-exit.
+     */
+    bool const fIsMsrCountValid = iemVmxIsAutoMsrCountValid(pVCpu, cMsrs);
+    if (fIsMsrCountValid)
+    { /* likely */ }
+    else
+    {
+        iemVmxVmcsSetExitQual(pVCpu, VMX_V_AUTOMSR_AREA_SIZE / sizeof(VMXAUTOMSR));
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrLoadCount);
+    }
+
+    RTGCPHYS const GCPhysVmEntryMsrLoadArea = pVmcs->u64AddrEntryMsrLoad.u;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.aEntryMsrLoadArea[0],
+                                     GCPhysVmEntryMsrLoadArea, cMsrs * sizeof(VMXAUTOMSR));
+    if (RT_SUCCESS(rc))
+    {
+        PCVMXAUTOMSR pMsr = &pVCpu->cpum.GstCtx.hwvirt.vmx.aEntryMsrLoadArea[0];
+        for (uint32_t idxMsr = 0; idxMsr < cMsrs; idxMsr++, pMsr++)
+        {
+            if (   !pMsr->u32Reserved
+                &&  pMsr->u32Msr != MSR_K8_FS_BASE
+                &&  pMsr->u32Msr != MSR_K8_GS_BASE
+                &&  pMsr->u32Msr != MSR_K6_EFER
+                &&  pMsr->u32Msr != MSR_IA32_SMM_MONITOR_CTL
+                &&  pMsr->u32Msr >> 8 != MSR_IA32_X2APIC_START >> 8)
+            {
+                VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, pMsr->u32Msr, pMsr->u64Value);
+                if (rcStrict == VINF_SUCCESS)
+                    continue;
+
+                /*
+                 * If we're in ring-0, we cannot handle returns to ring-3 at this point and continue VM-entry.
+                 * If any nested hypervisor loads MSRs that require ring-3 handling, we cause a VM-entry failure
+                 * recording the MSR index in the Exit qualification (as per the Intel spec.) and indicated
+                 * further by our own, specific diagnostic code. Later, we can try implement handling of the
+                 * MSR in ring-0 if possible, or come up with a better, generic solution.
+                 */
+                iemVmxVmcsSetExitQual(pVCpu, idxMsr);
+                VMXVDIAG const enmDiag = rcStrict == VINF_CPUM_R3_MSR_WRITE
+                                       ? kVmxVDiag_Vmentry_MsrLoadRing3
+                                       : kVmxVDiag_Vmentry_MsrLoad;
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, enmDiag);
+            }
+            else
+            {
+                iemVmxVmcsSetExitQual(pVCpu, idxMsr);
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrLoadRsvd);
+            }
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("%s: Failed to read MSR auto-load area at %#RGp, rc=%Rrc\n", pszInstr, GCPhysVmEntryMsrLoadArea, rc));
+        IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrLoadPtrReadPhys);
+    }
+
+    NOREF(pszInstr);
+    NOREF(pszFailure);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the guest-state non-register state as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ *
+ * @remarks This must be called only after loading the nested-guest register state
+ *          (especially nested-guest RIP).
+ */
+static int iemVmxVmentryLoadGuestNonRegState(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    /*
+     * Load guest non-register state.
+     * See Intel spec. 26.6 "Special Features of VM Entry"
+     */
+    const char *const pszFailure  = "VM-exit";
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    /*
+     * If VM-entry is not vectoring, block-by-STI and block-by-MovSS state must be loaded.
+     * If VM-entry is vectoring, there is no block-by-STI or block-by-MovSS.
+     *
+     * See Intel spec. 26.6.1 "Interruptibility State".
+     */
+    bool const fEntryVectoring = VMXIsVmentryVectoring(pVmcs->u32EntryIntInfo, NULL /* puEntryIntInfoType */);
+    if (   !fEntryVectoring
+        && (pVmcs->u32GuestIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)))
+        CPUMSetInInterruptShadowEx(&pVCpu->cpum.GstCtx, pVmcs->u64GuestRip.u);
+    else
+        Assert(!CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx));
+
+    /* NMI blocking. */
+    if (pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
+    {
+        if (pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+            pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = true;
+        else
+        {
+            pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false;
+            CPUMSetInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx);
+        }
+    }
+    else
+        pVCpu->cpum.GstCtx.hwvirt.vmx.fVirtNmiBlocking = false;
+
+    /* SMI blocking is irrelevant. We don't support SMIs yet. */
+
+    /*
+     * Set PGM's copy of the EPT pointer.
+     * The EPTP has already been validated while checking guest state.
+     *
+     * It is important to do this prior to mapping PAE PDPTEs (below).
+     */
+    if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT)
+        PGMSetGuestEptPtr(pVCpu, pVmcs->u64EptPtr.u);
+
+    /*
+     * Load the guest's PAE PDPTEs.
+     */
+    if (!iemVmxVmcsIsGuestPaePagingEnabled(pVmcs))
+    {
+        /*
+         * When PAE paging is not used we clear the PAE PDPTEs for safety
+         * in case we might be switching from a PAE host to a non-PAE guest.
+         */
+        pVCpu->cpum.GstCtx.aPaePdpes[0].u = 0;
+        pVCpu->cpum.GstCtx.aPaePdpes[1].u = 0;
+        pVCpu->cpum.GstCtx.aPaePdpes[2].u = 0;
+        pVCpu->cpum.GstCtx.aPaePdpes[3].u = 0;
+    }
+    else if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_EPT)
+    {
+        /*
+         * With EPT and the nested-guest using PAE paging, we've already validated the PAE PDPTEs
+         * while checking the guest state. We can load them into the nested-guest CPU state now.
+         * They'll later be used while mapping CR3 and the PAE PDPTEs.
+         */
+        pVCpu->cpum.GstCtx.aPaePdpes[0].u = pVmcs->u64GuestPdpte0.u;
+        pVCpu->cpum.GstCtx.aPaePdpes[1].u = pVmcs->u64GuestPdpte1.u;
+        pVCpu->cpum.GstCtx.aPaePdpes[2].u = pVmcs->u64GuestPdpte2.u;
+        pVCpu->cpum.GstCtx.aPaePdpes[3].u = pVmcs->u64GuestPdpte3.u;
+    }
+    else
+    {
+        /*
+         * Without EPT and the nested-guest using PAE paging, we must load the PAE PDPTEs
+         * referenced by CR3. This involves loading (and mapping) CR3 and validating them now.
+         */
+        int const rc = PGMGstMapPaePdpesAtCr3(pVCpu, pVmcs->u64GuestCr3.u);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_PDPTE);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_GuestPdpte);
+        }
+    }
+
+    /* VPID is irrelevant. We don't support VPID yet. */
+
+    /* Clear address-range monitoring. */
+    EMMonitorWaitClear(pVCpu);
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the guest VMCS referenced state (such as MSR bitmaps, I/O bitmaps etc).
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ *
+ * @remarks This assumes various VMCS related data structure pointers have already
+ *          been verified prior to calling this function.
+ */
+static int iemVmxVmentryLoadGuestVmcsRefState(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    const char *const pszFailure  = "VM-exit";
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    /*
+     * Virtualize APIC accesses.
+     */
+    if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+    {
+        /* APIC-access physical address. */
+        RTGCPHYS const GCPhysApicAccess = pVmcs->u64AddrApicAccess.u;
+
+        /*
+         * Register the handler for the APIC-access page.
+         *
+         * We don't deregister the APIC-access page handler during the VM-exit as a different
+         * nested-VCPU might be using the same guest-physical address for its APIC-access page.
+         *
+         * We leave the page registered until the first access that happens outside VMX non-root
+         * mode. Guest software is allowed to access structures such as the APIC-access page
+         * only when no logical processor with a current VMCS references it in VMX non-root mode,
+         * otherwise it can lead to unpredictable behavior including guest triple-faults.
+         *
+         * See Intel spec. 24.11.4 "Software Access to Related Structures".
+         */
+        if (!PGMHandlerPhysicalIsRegistered(pVCpu->CTX_SUFF(pVM), GCPhysApicAccess))
+        {
+            PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+            int rc = PGMHandlerPhysicalRegister(pVM, GCPhysApicAccess, GCPhysApicAccess | X86_PAGE_4K_OFFSET_MASK,
+                                                pVM->iem.s.hVmxApicAccessPage, 0 /*uUser*/, NULL /*pszDesc*/);
+            if (RT_SUCCESS(rc))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_AddrApicAccessHandlerReg);
+        }
+    }
+
+    /*
+     * VMCS shadowing.
+     */
+    if (pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+    {
+        /* Read the VMREAD-bitmap. */
+        RTGCPHYS const GCPhysVmreadBitmap = pVmcs->u64AddrVmreadBitmap.u;
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.abVmreadBitmap[0],
+                                         GCPhysVmreadBitmap, sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.abVmreadBitmap));
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmreadBitmapPtrReadPhys);
+
+        /* Read the VMWRITE-bitmap. */
+        RTGCPHYS const GCPhysVmwriteBitmap = pVmcs->u64AddrVmwriteBitmap.u;
+        rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.abVmwriteBitmap[0],
+                                     GCPhysVmwriteBitmap, sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.abVmwriteBitmap));
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmwriteBitmapPtrReadPhys);
+    }
+
+    /*
+     * I/O bitmaps.
+     */
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_IO_BITMAPS)
+    {
+        /* Read the IO bitmap A. */
+        RTGCPHYS const GCPhysIoBitmapA = pVmcs->u64AddrIoBitmapA.u;
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.abIoBitmap[0],
+                                         GCPhysIoBitmapA, VMX_V_IO_BITMAP_A_SIZE);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_IoBitmapAPtrReadPhys);
+
+        /* Read the IO bitmap B. */
+        RTGCPHYS const GCPhysIoBitmapB = pVmcs->u64AddrIoBitmapB.u;
+        rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.abIoBitmap[VMX_V_IO_BITMAP_A_SIZE],
+                                     GCPhysIoBitmapB, VMX_V_IO_BITMAP_B_SIZE);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_IoBitmapBPtrReadPhys);
+    }
+
+    /*
+     * TPR shadow and Virtual-APIC page.
+     */
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+    {
+        /* Verify TPR threshold and VTPR when both virtualize-APIC accesses and virtual-interrupt delivery aren't used. */
+        if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
+            && !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_INT_DELIVERY))
+        {
+            /* Read the VTPR from the virtual-APIC page. */
+            RTGCPHYS const GCPhysVirtApic = pVmcs->u64AddrVirtApic.u;
+            uint8_t u8VTpr;
+            int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &u8VTpr, GCPhysVirtApic + XAPIC_OFF_TPR, sizeof(u8VTpr));
+            if (RT_SUCCESS(rc))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VirtApicPagePtrReadPhys);
+
+            /* Bits 3:0 of the TPR-threshold must not be greater than bits 7:4 of VTPR. */
+            if ((uint8_t)RT_BF_GET(pVmcs->u32TprThreshold, VMX_BF_TPR_THRESHOLD_TPR) <= (u8VTpr & 0xf0))
+            { /* likely */ }
+            else
+                IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_TprThresholdVTpr);
+        }
+    }
+
+    /*
+     * VMCS link pointer.
+     */
+    if (pVmcs->u64VmcsLinkPtr.u != UINT64_C(0xffffffffffffffff))
+    {
+        /* Read the VMCS-link pointer from guest memory. */
+        RTGCPHYS const GCPhysShadowVmcs = pVmcs->u64VmcsLinkPtr.u;
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs,
+                                         GCPhysShadowVmcs, sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs));
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrReadPhys);
+        }
+
+        /* Verify the VMCS revision specified by the guest matches what we reported to the guest. */
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs.u32VmcsRevId.n.u31RevisionId == VMX_V_VMCS_REVISION_ID)
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrRevId);
+        }
+
+        /* Verify the shadow bit is set if VMCS shadowing is enabled . */
+        if (   !(pVmcs->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+            || pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs.u32VmcsRevId.n.fIsShadowVmcs)
+        { /* likely */ }
+        else
+        {
+            iemVmxVmcsSetExitQual(pVCpu, VMX_ENTRY_FAIL_QUAL_VMCS_LINK_PTR);
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_VmcsLinkPtrShadow);
+        }
+
+        /* Update our cache of the guest physical address of the shadow VMCS. */
+        pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysShadowVmcs = GCPhysShadowVmcs;
+    }
+
+    /*
+     * MSR bitmap.
+     */
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+    {
+        /* Read the MSR bitmap. */
+        RTGCPHYS const GCPhysMsrBitmap = pVmcs->u64AddrMsrBitmap.u;
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap[0],
+                                         GCPhysMsrBitmap, sizeof(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap));
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            IEM_VMX_VMENTRY_FAILED_RET(pVCpu, pszInstr, pszFailure, kVmxVDiag_Vmentry_MsrBitmapPtrReadPhys);
+    }
+
+    NOREF(pszFailure);
+    NOREF(pszInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Loads the guest-state as part of VM-entry.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ *
+ * @remarks This must be done after all the necessary steps prior to loading of
+ *          guest-state (e.g. checking various VMCS state).
+ */
+static int iemVmxVmentryLoadGuestState(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    /* Load guest control registers, MSRs (that are directly part of the VMCS). */
+    iemVmxVmentryLoadGuestControlRegsMsrs(pVCpu);
+
+    /* Load guest segment registers. */
+    iemVmxVmentryLoadGuestSegRegs(pVCpu);
+
+    /*
+     * Load guest RIP, RSP and RFLAGS.
+     * See Intel spec. 26.3.2.3 "Loading Guest RIP, RSP and RFLAGS".
+     */
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    pVCpu->cpum.GstCtx.rsp      = pVmcs->u64GuestRsp.u;
+    pVCpu->cpum.GstCtx.rip      = pVmcs->u64GuestRip.u;
+    pVCpu->cpum.GstCtx.rflags.u = pVmcs->u64GuestRFlags.u;
+
+    /* Initialize the PAUSE-loop controls as part of VM-entry. */
+    pVCpu->cpum.GstCtx.hwvirt.vmx.uFirstPauseLoopTick = 0;
+    pVCpu->cpum.GstCtx.hwvirt.vmx.uPrevPauseTick      = 0;
+
+    /* Load guest non-register state (such as interrupt shadows, NMI blocking etc). */
+    int rc = iemVmxVmentryLoadGuestNonRegState(pVCpu, pszInstr);
+    if (rc == VINF_SUCCESS)
+    { /* likely */ }
+    else
+        return rc;
+
+    /* Load VMX related structures and state referenced by the VMCS. */
+    rc = iemVmxVmentryLoadGuestVmcsRefState(pVCpu, pszInstr);
+    if (rc == VINF_SUCCESS)
+    { /* likely */ }
+    else
+        return rc;
+
+    NOREF(pszInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Returns whether there are is a pending debug exception on VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+static bool iemVmxVmentryIsPendingDebugXcpt(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    /*
+     * Pending debug exceptions.
+     * See Intel spec. 26.6.3 "Delivery of Pending Debug Exceptions after VM Entry".
+     */
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs);
+
+    bool fPendingDbgXcpt = RT_BOOL(pVmcs->u64GuestPendingDbgXcpts.u & (  VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS
+                                                                       | VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_EN_BP));
+    if (fPendingDbgXcpt)
+    {
+        uint8_t uEntryIntInfoType;
+        bool const fEntryVectoring = VMXIsVmentryVectoring(pVmcs->u32EntryIntInfo, &uEntryIntInfoType);
+        if (fEntryVectoring)
+        {
+            switch (uEntryIntInfoType)
+            {
+                case VMX_ENTRY_INT_INFO_TYPE_EXT_INT:
+                case VMX_ENTRY_INT_INFO_TYPE_NMI:
+                case VMX_ENTRY_INT_INFO_TYPE_HW_XCPT:
+                case VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT:
+                    fPendingDbgXcpt = false;
+                    break;
+
+                case VMX_ENTRY_INT_INFO_TYPE_SW_XCPT:
+                {
+                    /*
+                     * Whether the pending debug exception for software exceptions other than
+                     * #BP and #OF is delivered after injecting the exception or is discard
+                     * is CPU implementation specific. We will discard them (easier).
+                     */
+                    uint8_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(pVmcs->u32EntryIntInfo);
+                    if (   uVector != X86_XCPT_BP
+                        && uVector != X86_XCPT_OF)
+                        fPendingDbgXcpt = false;
+                    RT_FALL_THRU();
+                }
+                case VMX_ENTRY_INT_INFO_TYPE_SW_INT:
+                {
+                    if (!(pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
+                        fPendingDbgXcpt = false;
+                    break;
+                }
+            }
+        }
+        else
+        {
+            /*
+             * When the VM-entry is not vectoring but there is blocking-by-MovSS, whether the
+             * pending debug exception is held pending or is discarded is CPU implementation
+             * specific. We will discard them (easier).
+             */
+            if (pVmcs->u32GuestIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
+                fPendingDbgXcpt = false;
+
+            /* There's no pending debug exception in the shutdown or wait-for-SIPI state. */
+            if (pVmcs->u32GuestActivityState & (VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN | VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT))
+                fPendingDbgXcpt = false;
+        }
+    }
+
+    NOREF(pszInstr);
+    return fPendingDbgXcpt;
+}
+
+
+/**
+ * Set up the monitor-trap flag (MTF).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+static void iemVmxVmentrySetupMtf(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs);
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
+    {
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_MTF);
+        Log(("%s: Monitor-trap flag set on VM-entry\n", pszInstr));
+    }
+    else
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF));
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Sets up NMI-window exiting.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+static void iemVmxVmentrySetupNmiWindow(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs);
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
+    {
+        Assert(pVmcs->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW);
+        Log(("%s: NMI-window set on VM-entry\n", pszInstr));
+    }
+    else
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_NMI_WINDOW));
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Sets up interrupt-window exiting.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+static void iemVmxVmentrySetupIntWindow(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs);
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
+    {
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW);
+        Log(("%s: Interrupt-window set on VM-entry\n", pszInstr));
+    }
+    else
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_INT_WINDOW));
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Set up the VMX-preemption timer.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+static void iemVmxVmentrySetupPreemptTimer(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs);
+    if (pVmcs->u32PinCtls & VMX_PIN_CTLS_PREEMPT_TIMER)
+    {
+        /*
+         * If the timer is 0, we must cause a VM-exit before executing the first
+         * nested-guest instruction. So we can flag as though the timer has already
+         * expired and we will check and cause a VM-exit at the right priority elsewhere
+         * in the code.
+         */
+        uint64_t uEntryTick;
+        uint32_t const uPreemptTimer = pVmcs->u32PreemptTimer;
+        if (uPreemptTimer)
+        {
+            int rc = CPUMStartGuestVmxPremptTimer(pVCpu, uPreemptTimer, VMX_V_PREEMPT_TIMER_SHIFT, &uEntryTick);
+            AssertRC(rc);
+            Log(("%s: VM-entry set up VMX-preemption timer at %#RX64\n", pszInstr, uEntryTick));
+        }
+        else
+        {
+            uEntryTick = TMCpuTickGetNoCheck(pVCpu);
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER);
+            Log(("%s: VM-entry set up VMX-preemption timer at %#RX64 to expire immediately!\n", pszInstr, uEntryTick));
+        }
+
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uEntryTick = uEntryTick;
+    }
+    else
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER));
+
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Injects an event using TRPM given a VM-entry interruption info and related
+ * fields.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pszInstr            The VMX instruction name (for logging purposes).
+ * @param   uEntryIntInfo       The VM-entry interruption info.
+ * @param   uErrCode            The error code associated with the event if any.
+ * @param   cbInstr             The VM-entry instruction length (for software
+ *                              interrupts and software exceptions). Pass 0
+ *                              otherwise.
+ * @param   GCPtrFaultAddress   The guest CR2 if this is a \#PF event.
+ */
+static void iemVmxVmentryInjectTrpmEvent(PVMCPUCC pVCpu, const char *pszInstr, uint32_t uEntryIntInfo, uint32_t uErrCode,
+                                         uint32_t cbInstr, RTGCUINTPTR GCPtrFaultAddress) RT_NOEXCEPT
+{
+    Assert(VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
+
+    uint8_t const   uType        = VMX_ENTRY_INT_INFO_TYPE(uEntryIntInfo);
+    uint8_t const   uVector      = VMX_ENTRY_INT_INFO_VECTOR(uEntryIntInfo);
+    TRPMEVENT const enmTrpmEvent = HMVmxEventTypeToTrpmEventType(uEntryIntInfo);
+
+    Assert(uType != VMX_ENTRY_INT_INFO_TYPE_OTHER_EVENT);
+
+    int rc = TRPMAssertTrap(pVCpu, uVector, enmTrpmEvent);
+    AssertRC(rc);
+    Log(("%s: Injecting: vector=%#x type=%#x (%s)\n", pszInstr, uVector, uType, VMXGetEntryIntInfoTypeDesc(uType)));
+
+    if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(uEntryIntInfo))
+    {
+        TRPMSetErrorCode(pVCpu, uErrCode);
+        Log(("%s: Injecting: err_code=%#x\n", pszInstr, uErrCode));
+    }
+
+    if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(uEntryIntInfo))
+    {
+        TRPMSetFaultAddress(pVCpu, GCPtrFaultAddress);
+        Log(("%s: Injecting: fault_addr=%RGp\n", pszInstr, GCPtrFaultAddress));
+    }
+    else if (   uType == VMX_ENTRY_INT_INFO_TYPE_SW_INT
+             || uType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT
+             || uType == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT)
+    {
+        TRPMSetInstrLength(pVCpu, cbInstr);
+        Log(("%s: Injecting: instr_len=%u\n", pszInstr, cbInstr));
+    }
+
+    if (VMX_ENTRY_INT_INFO_TYPE(uEntryIntInfo) == VMX_ENTRY_INT_INFO_TYPE_PRIV_SW_XCPT)
+    {
+        TRPMSetTrapDueToIcebp(pVCpu);
+        Log(("%s: Injecting: icebp\n", pszInstr));
+    }
+
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Performs event injection (if any) as part of VM-entry.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pszInstr    The VMX instruction name (for logging purposes).
+ */
+static void iemVmxVmentryInjectEvent(PVMCPUCC pVCpu, const char *pszInstr) RT_NOEXCEPT
+{
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    /*
+     * Inject events.
+     * The event that is going to be made pending for injection is not subject to VMX intercepts,
+     * thus we flag ignoring of intercepts. However, recursive exceptions if any during delivery
+     * of the current event -are- subject to intercepts, hence this flag will be flipped during
+     * the actually delivery of this event.
+     *
+     * See Intel spec. 26.5 "Event Injection".
+     */
+    uint32_t const uEntryIntInfo      = pVmcs->u32EntryIntInfo;
+    bool const     fEntryIntInfoValid = VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo);
+
+    CPUMSetGuestVmxInterceptEvents(&pVCpu->cpum.GstCtx, !fEntryIntInfoValid);
+    if (fEntryIntInfoValid)
+    {
+        if (VMX_ENTRY_INT_INFO_TYPE(uEntryIntInfo) == VMX_ENTRY_INT_INFO_TYPE_OTHER_EVENT)
+        {
+            Assert(VMX_ENTRY_INT_INFO_VECTOR(uEntryIntInfo) == VMX_ENTRY_INT_INFO_VECTOR_MTF);
+            VMCPU_FF_SET(pVCpu, VMCPU_FF_VMX_MTF);
+        }
+        else
+            iemVmxVmentryInjectTrpmEvent(pVCpu, pszInstr, uEntryIntInfo, pVmcs->u32EntryXcptErrCode, pVmcs->u32EntryInstrLen,
+                                         pVCpu->cpum.GstCtx.cr2);
+
+        /*
+         * We need to clear the VM-entry interruption information field's valid bit on VM-exit.
+         *
+         * However, we do it here on VM-entry as well because while it isn't visible to guest
+         * software until VM-exit, when and if HM looks at the VMCS to continue nested-guest
+         * execution using hardware-assisted VMX, it will not try to inject the event again.
+         *
+         * See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
+         */
+        pVmcs->u32EntryIntInfo &= ~VMX_ENTRY_INT_INFO_VALID;
+    }
+    else
+    {
+        /*
+         * Inject any pending guest debug exception.
+         * Unlike injecting events, this #DB injection on VM-entry is subject to #DB VMX intercept.
+         * See Intel spec. 26.6.3 "Delivery of Pending Debug Exceptions after VM Entry".
+         */
+        bool const fPendingDbgXcpt = iemVmxVmentryIsPendingDebugXcpt(pVCpu, pszInstr);
+        if (fPendingDbgXcpt)
+        {
+            uint32_t const uDbgXcptInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
+                                        | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,   VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
+                                        | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,  1);
+            iemVmxVmentryInjectTrpmEvent(pVCpu, pszInstr, uDbgXcptInfo, 0 /* uErrCode */, pVmcs->u32EntryInstrLen,
+                                         0 /* GCPtrFaultAddress */);
+        }
+    }
+
+    NOREF(pszInstr);
+}
+
+
+/**
+ * Initializes all read-only VMCS fields as part of VM-entry.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void iemVmxVmentryInitReadOnlyFields(PVMCPUCC pVCpu) RT_NOEXCEPT
+{
+    /*
+     * Any VMCS field which we do not establish on every VM-exit but may potentially
+     * be used on the VM-exit path of a nested hypervisor -and- is not explicitly
+     * specified to be undefined, needs to be initialized here.
+     *
+     * Thus, it is especially important to clear the Exit qualification field
+     * since it must be zero for VM-exits where it is not used. Similarly, the
+     * VM-exit interruption information field's valid bit needs to be cleared for
+     * the same reasons.
+     */
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs);
+
+    /* 16-bit (none currently). */
+    /* 32-bit. */
+    pVmcs->u32RoVmInstrError        = 0;
+    pVmcs->u32RoExitReason          = 0;
+    pVmcs->u32RoExitIntInfo         = 0;
+    pVmcs->u32RoExitIntErrCode      = 0;
+    pVmcs->u32RoIdtVectoringInfo    = 0;
+    pVmcs->u32RoIdtVectoringErrCode = 0;
+    pVmcs->u32RoExitInstrLen        = 0;
+    pVmcs->u32RoExitInstrInfo       = 0;
+
+    /* 64-bit. */
+    pVmcs->u64RoGuestPhysAddr.u     = 0;
+
+    /* Natural-width. */
+    pVmcs->u64RoExitQual.u          = 0;
+    pVmcs->u64RoIoRcx.u             = 0;
+    pVmcs->u64RoIoRsi.u             = 0;
+    pVmcs->u64RoIoRdi.u             = 0;
+    pVmcs->u64RoIoRip.u             = 0;
+    pVmcs->u64RoGuestLinearAddr.u   = 0;
+}
+
+
+/**
+ * VMLAUNCH/VMRESUME instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   uInstrId    The instruction identity (VMXINSTRID_VMLAUNCH or
+ *                      VMXINSTRID_VMRESUME).
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+static VBOXSTRICTRC iemVmxVmlaunchVmresume(PVMCPUCC pVCpu, uint8_t cbInstr, VMXINSTRID uInstrId) RT_NOEXCEPT
+{
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && !defined(IN_RING3)
+    RT_NOREF3(pVCpu, cbInstr, uInstrId);
+    return VINF_EM_RAW_EMULATE_INSTR;
+# else
+    Assert(   uInstrId == VMXINSTRID_VMLAUNCH
+           || uInstrId == VMXINSTRID_VMRESUME);
+    const char * const pszInstr = uInstrId == VMXINSTRID_VMRESUME ? "vmresume" : "vmlaunch";
+
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        return iemVmxVmexitInstr(pVCpu, uInstrId == VMXINSTRID_VMRESUME ? VMX_EXIT_VMRESUME : VMX_EXIT_VMLAUNCH, cbInstr);
+
+    Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
+
+    /*
+     * Basic VM-entry checks.
+     * The order of the CPL, current and shadow VMCS and block-by-MovSS are important.
+     * The checks following that do not have to follow a specific order.
+     *
+     * See Intel spec. 26.1 "Basic VM-entry Checks".
+     */
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("%s: CPL %u -> #GP(0)\n", pszInstr, pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Current VMCS valid. */
+    if (IEM_VMX_HAS_CURRENT_VMCS(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("%s: VMCS pointer %#RGp invalid -> VMFailInvalid\n", pszInstr, IEM_VMX_GET_CURRENT_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_PtrInvalid;
+        iemVmxVmFailInvalid(pVCpu);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* Current VMCS is not a shadow VMCS. */
+    if (!pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32VmcsRevId.n.fIsShadowVmcs)
+    { /* likely */ }
+    else
+    {
+        Log(("%s: VMCS pointer %#RGp is a shadow VMCS -> VMFailInvalid\n", pszInstr, IEM_VMX_GET_CURRENT_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_PtrShadowVmcs;
+        iemVmxVmFailInvalid(pVCpu);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /** @todo Distinguish block-by-MovSS from block-by-STI. Currently we
+     *        use block-by-STI here which is not quite correct. */
+    if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx))
+    { /* likely */ }
+    else
+    {
+        Log(("%s: VM entry with events blocked by MOV SS -> VMFail\n", pszInstr));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_BlocKMovSS;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMENTRY_BLOCK_MOVSS);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    if (uInstrId == VMXINSTRID_VMLAUNCH)
+    {
+        /* VMLAUNCH with non-clear VMCS. */
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.fVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR)
+        { /* likely */ }
+        else
+        {
+            Log(("vmlaunch: VMLAUNCH with non-clear VMCS -> VMFail\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_VmcsClear;
+            iemVmxVmFail(pVCpu, VMXINSTRERR_VMLAUNCH_NON_CLEAR_VMCS);
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+    }
+    else
+    {
+        /* VMRESUME with non-launched VMCS. */
+        if (pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.fVmcsState == VMX_V_VMCS_LAUNCH_STATE_LAUNCHED)
+        { /* likely */ }
+        else
+        {
+            Log(("vmresume: VMRESUME with non-launched VMCS -> VMFail\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmentry_VmcsLaunch;
+            iemVmxVmFail(pVCpu, VMXINSTRERR_VMRESUME_NON_LAUNCHED_VMCS);
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+    }
+
+    /*
+     * We are allowed to cache VMCS related data structures (such as I/O bitmaps, MSR bitmaps)
+     * while entering VMX non-root mode. We do some of this while checking VM-execution
+     * controls. The nested hypervisor should not make assumptions and cannot expect
+     * predictable behavior if changes to these structures are made in guest memory while
+     * executing in VMX non-root mode. As far as VirtualBox is concerned, the guest cannot
+     * modify them anyway as we cache them in host memory.
+     *
+     * See Intel spec. 24.11.4 "Software Access to Related Structures".
+     */
+    PVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs);
+    Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
+
+    int rc = iemVmxVmentryCheckCtls(pVCpu, pszInstr);
+    if (RT_SUCCESS(rc))
+    {
+        rc = iemVmxVmentryCheckHostState(pVCpu, pszInstr);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Initialize read-only VMCS fields before VM-entry since we don't update all of them
+             * for every VM-exit. This needs to be done before invoking a VM-exit (even those
+             * ones that may occur during VM-entry below).
+             */
+            iemVmxVmentryInitReadOnlyFields(pVCpu);
+
+            /*
+             * Blocking of NMIs need to be restored if VM-entry fails due to invalid-guest state.
+             * So we save the VMCPU_FF_BLOCK_NMI force-flag here so we can restore it on
+             * VM-exit when required.
+             * See Intel spec. 26.7 "VM-entry Failures During or After Loading Guest State"
+             */
+            iemVmxVmentrySaveNmiBlockingFF(pVCpu);
+
+            rc = iemVmxVmentryCheckGuestState(pVCpu, pszInstr);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * We've now entered nested-guest execution.
+                 *
+                 * It is important do this prior to loading the guest state because
+                 * as part of loading the guest state, PGM (and perhaps other components
+                 * in the future) relies on detecting whether VMX non-root mode has been
+                 * entered.
+                 */
+                pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode = true;
+
+                rc = iemVmxVmentryLoadGuestState(pVCpu, pszInstr);
+                if (RT_SUCCESS(rc))
+                {
+                    rc = iemVmxVmentryLoadGuestAutoMsrs(pVCpu, pszInstr);
+                    if (RT_SUCCESS(rc))
+                    {
+                        Assert(rc != VINF_CPUM_R3_MSR_WRITE);
+
+                        /* VMLAUNCH instruction must update the VMCS launch state. */
+                        if (uInstrId == VMXINSTRID_VMLAUNCH)
+                            pVmcs->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_LAUNCHED;
+
+                        /* Perform the VMX transition (PGM updates). */
+                        VBOXSTRICTRC rcStrict = iemVmxTransition(pVCpu);
+                        if (rcStrict == VINF_SUCCESS)
+                        { /* likely */ }
+                        else if (RT_SUCCESS(rcStrict))
+                        {
+                            Log3(("%s: iemVmxTransition returns %Rrc -> Setting passup status\n", pszInstr,
+                                  VBOXSTRICTRC_VAL(rcStrict)));
+                            rcStrict = iemSetPassUpStatus(pVCpu, rcStrict);
+                        }
+                        else
+                        {
+                            Log3(("%s: iemVmxTransition failed! rc=%Rrc\n", pszInstr, VBOXSTRICTRC_VAL(rcStrict)));
+                            return rcStrict;
+                        }
+
+                        /* Paranoia. */
+                        Assert(rcStrict == VINF_SUCCESS);
+
+                        /*
+                         * The priority of potential VM-exits during VM-entry is important.
+                         * The priorities of VM-exits and events are listed from highest
+                         * to lowest as follows:
+                         *
+                         * 1.  Event injection.
+                         * 2.  Trap on task-switch (T flag set in TSS).
+                         * 3.  TPR below threshold / APIC-write.
+                         * 4.  SMI, INIT.
+                         * 5.  MTF exit.
+                         * 6.  Debug-trap exceptions (EFLAGS.TF), pending debug exceptions.
+                         * 7.  VMX-preemption timer.
+                         * 9.  NMI-window exit.
+                         * 10. NMI injection.
+                         * 11. Interrupt-window exit.
+                         * 12. Virtual-interrupt injection.
+                         * 13. Interrupt injection.
+                         * 14. Process next instruction (fetch, decode, execute).
+                         */
+
+                        /* Setup VMX-preemption timer. */
+                        iemVmxVmentrySetupPreemptTimer(pVCpu, pszInstr);
+
+                        /* Setup monitor-trap flag. */
+                        iemVmxVmentrySetupMtf(pVCpu, pszInstr);
+
+                        /* Setup NMI-window exiting. */
+                        iemVmxVmentrySetupNmiWindow(pVCpu, pszInstr);
+
+                        /* Setup interrupt-window exiting. */
+                        iemVmxVmentrySetupIntWindow(pVCpu, pszInstr);
+
+                        /*
+                         * Inject any event that the nested hypervisor wants to inject.
+                         * Note! We cannot immediately perform the event injection here as we may have
+                         *       pending PGM operations to perform due to switching page tables and/or
+                         *       mode.
+                         */
+                        iemVmxVmentryInjectEvent(pVCpu, pszInstr);
+
+# if defined(VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM) && defined(IN_RING3)
+                        /* Reschedule to IEM-only execution of the nested-guest. */
+                        LogFlow(("%s: Enabling IEM-only EM execution policy!\n", pszInstr));
+                        int rcSched = EMR3SetExecutionPolicy(pVCpu->CTX_SUFF(pVM)->pUVM, EMEXECPOLICY_IEM_ALL, true);
+                        if (rcSched != VINF_SUCCESS)
+                            iemSetPassUpStatus(pVCpu, rcSched);
+# endif
+
+                        /* Finally, done. */
+                        Log2(("vmentry: %s: cs:rip=%04x:%08RX64 cr0=%#RX64 (%#RX64) cr4=%#RX64 (%#RX64) efer=%#RX64 (%#RX64)\n",
+                              pszInstr, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.cr0,
+                              pVmcs->u64Cr0ReadShadow.u, pVCpu->cpum.GstCtx.cr4, pVmcs->u64Cr4ReadShadow.u,
+                              pVCpu->cpum.GstCtx.msrEFER, pVmcs->u64GuestEferMsr.u));
+                        return VINF_SUCCESS;
+                    }
+                    return iemVmxVmexit(pVCpu, VMX_EXIT_ERR_MSR_LOAD | VMX_EXIT_REASON_ENTRY_FAILED, pVmcs->u64RoExitQual.u);
+                }
+            }
+            return iemVmxVmexit(pVCpu, VMX_EXIT_ERR_INVALID_GUEST_STATE | VMX_EXIT_REASON_ENTRY_FAILED, pVmcs->u64RoExitQual.u);
+        }
+
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMENTRY_INVALID_HOST_STATE);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    iemVmxVmFail(pVCpu, VMXINSTRERR_VMENTRY_INVALID_CTLS);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+# endif
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMLAUNCH/VMRESUME instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   uInstrId        The instruction ID (VMXINSTRID_VMLAUNCH or
+ *                          VMXINSTRID_VMRESUME).
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmlaunchVmresume(PVMCPUCC pVCpu, uint8_t cbInstr, VMXINSTRID uInstrId)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = iemVmxVmlaunchVmresume(pVCpu, cbInstr, uInstrId);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Checks whether an RDMSR or WRMSR instruction for the given MSR is intercepted
+ * (causes a VM-exit)  or not.
+ *
+ * @returns @c true if the instruction is intercepted, @c false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason (VMX_EXIT_RDMSR or
+ *                          VMX_EXIT_WRMSR).
+ * @param   idMsr           The MSR.
+ */
+bool iemVmxIsRdmsrWrmsrInterceptSet(PCVMCPU pVCpu, uint32_t uExitReason, uint32_t idMsr) RT_NOEXCEPT
+{
+    Assert(IEM_VMX_IS_NON_ROOT_MODE(pVCpu));
+    Assert(   uExitReason == VMX_EXIT_RDMSR
+           || uExitReason == VMX_EXIT_WRMSR);
+
+    /* Consult the MSR bitmap if the feature is supported. */
+    PCVMXVVMCS const pVmcs = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+    Assert(pVmcs);
+    if (pVmcs->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+    {
+        uint32_t const fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, idMsr);
+        if (uExitReason == VMX_EXIT_RDMSR)
+            return RT_BOOL(fMsrpm & VMXMSRPM_EXIT_RD);
+        return RT_BOOL(fMsrpm & VMXMSRPM_EXIT_WR);
+    }
+
+    /* Without MSR bitmaps, all MSR accesses are intercepted. */
+    return true;
+}
+
+
+/**
+ * VMREAD instruction execution worker that does not perform any validation checks.
+ *
+ * Callers are expected to have performed the necessary checks and to ensure the
+ * VMREAD will succeed.
+ *
+ * @param   pVmcs           Pointer to the virtual VMCS.
+ * @param   pu64Dst         Where to write the VMCS value.
+ * @param   u64VmcsField    The VMCS field.
+ *
+ * @remarks May be called with interrupts disabled.
+ */
+static void iemVmxVmreadNoCheck(PCVMXVVMCS pVmcs, uint64_t *pu64Dst, uint64_t u64VmcsField) RT_NOEXCEPT
+{
+    VMXVMCSFIELD VmcsField;
+    VmcsField.u = u64VmcsField;
+    uint8_t  const uWidth     = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_WIDTH);
+    uint8_t  const uType      = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_TYPE);
+    uint8_t  const uWidthType = (uWidth << 2) | uType;
+    uint8_t  const uIndex     = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_INDEX);
+    Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+    uint16_t const offField   = g_aoffVmcsMap[uWidthType][uIndex];
+    AssertMsg(offField < VMX_V_VMCS_SIZE, ("off=%u field=%#RX64 width=%#x type=%#x index=%#x (%u)\n", offField, u64VmcsField,
+                                           uWidth, uType, uIndex, uIndex));
+    AssertCompile(VMX_V_SHADOW_VMCS_SIZE == VMX_V_VMCS_SIZE);
+
+    /*
+     * Read the VMCS component based on the field's effective width.
+     *
+     * The effective width is 64-bit fields adjusted to 32-bits if the access-type
+     * indicates high bits (little endian).
+     *
+     * Note! The caller is responsible to trim the result and update registers
+     * or memory locations are required. Here we just zero-extend to the largest
+     * type (i.e. 64-bits).
+     */
+    uint8_t const *pbVmcs    = (uint8_t const *)pVmcs;
+    uint8_t const *pbField   = pbVmcs + offField;
+    uint8_t const  uEffWidth = VMXGetVmcsFieldWidthEff(VmcsField.u);
+    switch (uEffWidth)
+    {
+        case VMX_VMCSFIELD_WIDTH_64BIT:
+        case VMX_VMCSFIELD_WIDTH_NATURAL: *pu64Dst = *(uint64_t const *)pbField; break;
+        case VMX_VMCSFIELD_WIDTH_32BIT:   *pu64Dst = *(uint32_t const *)pbField; break;
+        case VMX_VMCSFIELD_WIDTH_16BIT:   *pu64Dst = *(uint16_t const *)pbField; break;
+    }
+}
+
+
+/**
+ * Interface for HM and EM to read a VMCS field from the nested-guest VMCS.
+ *
+ * It is ASSUMED the caller knows what they're doing. No VMREAD instruction checks
+ * are performed. Bounds checks are strict builds only.
+ *
+ * @param   pVmcs           Pointer to the virtual VMCS.
+ * @param   u64VmcsField    The VMCS field.
+ * @param   pu64Dst         Where to store the VMCS value.
+ *
+ * @remarks May be called with interrupts disabled.
+ * @todo    This should probably be moved to CPUM someday.
+ */
+VMM_INT_DECL(void) IEMReadVmxVmcsField(PCVMXVVMCS pVmcs, uint64_t u64VmcsField, uint64_t *pu64Dst)
+{
+    AssertPtr(pVmcs);
+    AssertPtr(pu64Dst);
+    iemVmxVmreadNoCheck(pVmcs, pu64Dst, u64VmcsField);
+}
+
+
+/**
+ * VMREAD common (memory/register) instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   pu64Dst         Where to write the VMCS value (only updated when
+ *                          VINF_SUCCESS is returned).
+ * @param   u64VmcsField    The VMCS field.
+ * @param   pExitInfo       Pointer to the VM-exit information. Optional, can be
+ *                          NULL.
+ */
+static VBOXSTRICTRC iemVmxVmreadCommon(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t *pu64Dst,
+                                       uint64_t u64VmcsField, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    /* Nested-guest intercept. */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, VMX_EXIT_VMREAD, u64VmcsField))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMREAD, VMXINSTRID_VMREAD, cbInstr);
+    }
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmread: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    pVCpu->iem.s.cPotentialExits++;
+
+    /* VMCS pointer in root mode. */
+    if (   !IEM_VMX_IS_ROOT_MODE(pVCpu)
+        ||  IEM_VMX_HAS_CURRENT_VMCS(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("vmread: VMCS pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_CURRENT_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_PtrInvalid;
+        iemVmxVmFailInvalid(pVCpu);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* VMCS-link pointer in non-root mode. */
+    if (   !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        ||  IEM_VMX_HAS_SHADOW_VMCS(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("vmread: VMCS-link pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_SHADOW_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmread_LinkPtrInvalid;
+        iemVmxVmFailInvalid(pVCpu);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* Supported VMCS field. */
+    if (CPUMIsGuestVmxVmcsFieldValid(pVCpu->CTX_SUFF(pVM), u64VmcsField))
+    { /* likely */ }
+    else
+    {
+        Log(("vmread: VMCS field %#RX64 invalid -> VMFail\n", u64VmcsField));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmread_FieldInvalid;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64VmcsField;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMREAD_INVALID_COMPONENT);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /*
+     * Reading from the current or shadow VMCS.
+     */
+    PCVMXVVMCS pVmcs = !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+                     ? &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs
+                     : &pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs;
+    iemVmxVmreadNoCheck(pVmcs, pu64Dst, u64VmcsField);
+    Log4(("vmread %#RX64 => %#RX64\n", u64VmcsField, *pu64Dst));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VMREAD (64-bit register) instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   pu64Dst         Where to store the VMCS field's value.
+ * @param   u64VmcsField    The VMCS field.
+ * @param   pExitInfo       Pointer to the VM-exit information. Optional, can be
+ *                          NULL.
+ */
+static VBOXSTRICTRC iemVmxVmreadReg64(PVMCPUCC pVCpu, uint8_t cbInstr, uint64_t *pu64Dst,
+                                      uint64_t u64VmcsField, PCVMXVEXITINFO pExitInfo)  RT_NOEXCEPT
+{
+    VBOXSTRICTRC rcStrict = iemVmxVmreadCommon(pVCpu, cbInstr, pu64Dst, u64VmcsField, pExitInfo);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        iemVmxVmSucceed(pVCpu);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    Log(("vmread/reg: iemVmxVmreadCommon failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VMREAD (32-bit register) instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   pu32Dst         Where to store the VMCS field's value.
+ * @param   u32VmcsField    The VMCS field.
+ * @param   pExitInfo       Pointer to the VM-exit information. Optional, can be
+ *                          NULL.
+ */
+static VBOXSTRICTRC iemVmxVmreadReg32(PVMCPUCC pVCpu, uint8_t cbInstr, uint32_t *pu32Dst,
+                                      uint64_t u32VmcsField, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    uint64_t u64Dst;
+    VBOXSTRICTRC rcStrict = iemVmxVmreadCommon(pVCpu, cbInstr, &u64Dst, u32VmcsField, pExitInfo);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        *pu32Dst = u64Dst;
+        iemVmxVmSucceed(pVCpu);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    Log(("vmread/reg: iemVmxVmreadCommon failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VMREAD (memory) instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   iEffSeg         The effective segment register to use with @a u64Val.
+ *                          Pass UINT8_MAX if it is a register access.
+ * @param   GCPtrDst        The guest linear address to store the VMCS field's
+ *                          value.
+ * @param   u64VmcsField    The VMCS field.
+ * @param   pExitInfo       Pointer to the VM-exit information. Optional, can be
+ *                          NULL.
+ */
+static VBOXSTRICTRC iemVmxVmreadMem(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrDst,
+                                    uint64_t u64VmcsField, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    uint64_t u64Dst;
+    VBOXSTRICTRC rcStrict = iemVmxVmreadCommon(pVCpu, cbInstr, &u64Dst, u64VmcsField, pExitInfo);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * Write the VMCS field's value to the location specified in guest-memory.
+         */
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+            rcStrict = iemMemStoreDataU64(pVCpu, iEffSeg, GCPtrDst, u64Dst);
+        else
+            rcStrict = iemMemStoreDataU32(pVCpu, iEffSeg, GCPtrDst, u64Dst);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            iemVmxVmSucceed(pVCpu);
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+
+        Log(("vmread/mem: Failed to write to memory operand at %#RGv, rc=%Rrc\n", GCPtrDst, VBOXSTRICTRC_VAL(rcStrict)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmread_PtrMap;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrDst;
+        return rcStrict;
+    }
+
+    Log(("vmread/mem: iemVmxVmreadCommon failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMREAD instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmread(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+    Assert(pExitInfo);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    VBOXSTRICTRC   rcStrict;
+    uint8_t const  cbInstr       = pExitInfo->cbInstr;
+    bool const     fIs64BitMode  = RT_BOOL(pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT);
+    uint64_t const u64FieldEnc   = fIs64BitMode
+                                 ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2)
+                                 : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2);
+    if (pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand)
+    {
+        if (fIs64BitMode)
+        {
+            uint64_t *pu64Dst = iemGRegRefU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
+            rcStrict = iemVmxVmreadReg64(pVCpu, cbInstr, pu64Dst, u64FieldEnc, pExitInfo);
+        }
+        else
+        {
+            uint32_t *pu32Dst = iemGRegRefU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
+            rcStrict = iemVmxVmreadReg32(pVCpu, cbInstr, pu32Dst, u64FieldEnc, pExitInfo);
+        }
+    }
+    else
+    {
+        RTGCPTR const GCPtrDst = pExitInfo->GCPtrEffAddr;
+        uint8_t const iEffSeg  = pExitInfo->InstrInfo.VmreadVmwrite.iSegReg;
+        rcStrict = iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u64FieldEnc, pExitInfo);
+    }
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMWRITE instruction execution worker that does not perform any validation
+ * checks.
+ *
+ * Callers are expected to have performed the necessary checks and to ensure the
+ * VMWRITE will succeed.
+ *
+ * @param   pVmcs           Pointer to the virtual VMCS.
+ * @param   u64Val          The value to write.
+ * @param   u64VmcsField    The VMCS field.
+ *
+ * @remarks May be called with interrupts disabled.
+ */
+static void iemVmxVmwriteNoCheck(PVMXVVMCS pVmcs, uint64_t u64Val, uint64_t u64VmcsField) RT_NOEXCEPT
+{
+    VMXVMCSFIELD VmcsField;
+    VmcsField.u = u64VmcsField;
+    uint8_t  const uWidth     = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_WIDTH);
+    uint8_t  const uType      = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_TYPE);
+    uint8_t  const uWidthType = (uWidth << 2) | uType;
+    uint8_t  const uIndex     = RT_BF_GET(VmcsField.u, VMX_BF_VMCSFIELD_INDEX);
+    Assert(uIndex <= VMX_V_VMCS_MAX_INDEX);
+    uint16_t const offField   = g_aoffVmcsMap[uWidthType][uIndex];
+    Assert(offField < VMX_V_VMCS_SIZE);
+    AssertCompile(VMX_V_SHADOW_VMCS_SIZE == VMX_V_VMCS_SIZE);
+
+    /*
+     * Write the VMCS component based on the field's effective width.
+     *
+     * The effective width is 64-bit fields adjusted to 32-bits if the access-type
+     * indicates high bits (little endian).
+     */
+    uint8_t      *pbVmcs    = (uint8_t *)pVmcs;
+    uint8_t      *pbField   = pbVmcs + offField;
+    uint8_t const uEffWidth = VMXGetVmcsFieldWidthEff(VmcsField.u);
+    switch (uEffWidth)
+    {
+        case VMX_VMCSFIELD_WIDTH_64BIT:
+        case VMX_VMCSFIELD_WIDTH_NATURAL: *(uint64_t *)pbField = u64Val; break;
+        case VMX_VMCSFIELD_WIDTH_32BIT:   *(uint32_t *)pbField = u64Val; break;
+        case VMX_VMCSFIELD_WIDTH_16BIT:   *(uint16_t *)pbField = u64Val; break;
+    }
+}
+
+
+/**
+ * Interface for HM and EM to write a VMCS field in the nested-guest VMCS.
+ *
+ * It is ASSUMED the caller knows what they're doing. No VMWRITE instruction checks
+ * are performed. Bounds checks are strict builds only.
+ *
+ * @param   pVmcs           Pointer to the virtual VMCS.
+ * @param   u64VmcsField    The VMCS field.
+ * @param   u64Val          The value to write.
+ *
+ * @remarks May be called with interrupts disabled.
+ * @todo    This should probably be moved to CPUM someday.
+ */
+VMM_INT_DECL(void) IEMWriteVmxVmcsField(PVMXVVMCS pVmcs, uint64_t u64VmcsField, uint64_t u64Val)
+{
+    AssertPtr(pVmcs);
+    iemVmxVmwriteNoCheck(pVmcs, u64Val, u64VmcsField);
+}
+
+
+/**
+ * VMWRITE instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cbInstr         The instruction length in bytes.
+ * @param   iEffSeg         The effective segment register to use with @a u64Val.
+ *                          Pass UINT8_MAX if it is a register access.
+ * @param   u64Val          The value to write (or guest linear address to the
+ *                          value), @a iEffSeg will indicate if it's a memory
+ *                          operand.
+ * @param   u64VmcsField    The VMCS field.
+ * @param   pExitInfo       Pointer to the VM-exit information. Optional, can be
+ *                          NULL.
+ */
+static VBOXSTRICTRC iemVmxVmwrite(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, uint64_t u64Val,
+                                  uint64_t u64VmcsField, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    /* Nested-guest intercept. */
+    if (   IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        && CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, VMX_EXIT_VMWRITE, u64VmcsField))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMWRITE, VMXINSTRID_VMWRITE, cbInstr);
+    }
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmwrite: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    pVCpu->iem.s.cPotentialExits++;
+
+    /* VMCS pointer in root mode. */
+    if (   !IEM_VMX_IS_ROOT_MODE(pVCpu)
+        ||  IEM_VMX_HAS_CURRENT_VMCS(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("vmwrite: VMCS pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_CURRENT_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_PtrInvalid;
+        iemVmxVmFailInvalid(pVCpu);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* VMCS-link pointer in non-root mode. */
+    if (   !IEM_VMX_IS_NON_ROOT_MODE(pVCpu)
+        ||  IEM_VMX_HAS_SHADOW_VMCS(pVCpu))
+    { /* likely */ }
+    else
+    {
+        Log(("vmwrite: VMCS-link pointer %#RGp invalid -> VMFailInvalid\n", IEM_VMX_GET_SHADOW_VMCS(pVCpu)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmwrite_LinkPtrInvalid;
+        iemVmxVmFailInvalid(pVCpu);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* If the VMWRITE instruction references memory, access the specified memory operand. */
+    bool const fIsRegOperand = iEffSeg == UINT8_MAX;
+    if (!fIsRegOperand)
+    {
+        /* Read the value from the specified guest memory location. */
+        VBOXSTRICTRC  rcStrict;
+        RTGCPTR const GCPtrVal = u64Val;
+        if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+            rcStrict = iemMemFetchDataU64(pVCpu, &u64Val, iEffSeg, GCPtrVal);
+        else
+            rcStrict = iemMemFetchDataU32_ZX_U64(pVCpu, &u64Val, iEffSeg, GCPtrVal);
+        if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
+        {
+            Log(("vmwrite: Failed to read value from memory operand at %#RGv, rc=%Rrc\n", GCPtrVal, VBOXSTRICTRC_VAL(rcStrict)));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmwrite_PtrMap;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVal;
+            return rcStrict;
+        }
+    }
+    else
+        Assert(!pExitInfo || pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand);
+
+    /* Supported VMCS field. */
+    if (CPUMIsGuestVmxVmcsFieldValid(pVCpu->CTX_SUFF(pVM), u64VmcsField))
+    { /* likely */ }
+    else
+    {
+        Log(("vmwrite: VMCS field %#RX64 invalid -> VMFail\n", u64VmcsField));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmwrite_FieldInvalid;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64VmcsField;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMWRITE_INVALID_COMPONENT);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* Read-only VMCS field. */
+    bool const fIsFieldReadOnly = VMXIsVmcsFieldReadOnly(u64VmcsField);
+    if (   !fIsFieldReadOnly
+        ||  IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxVmwriteAll)
+    { /* likely */ }
+    else
+    {
+        Log(("vmwrite: Write to read-only VMCS component %#RX64 -> VMFail\n", u64VmcsField));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmwrite_FieldRo;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64VmcsField;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMWRITE_RO_COMPONENT);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /*
+     * Write to the current or shadow VMCS.
+     */
+    bool const fInVmxNonRootMode = IEM_VMX_IS_NON_ROOT_MODE(pVCpu);
+    PVMXVVMCS pVmcs = !fInVmxNonRootMode
+                    ? &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs
+                    : &pVCpu->cpum.GstCtx.hwvirt.vmx.ShadowVmcs;
+    iemVmxVmwriteNoCheck(pVmcs, u64Val, u64VmcsField);
+    Log4(("vmwrite %#RX64 <= %#RX64\n", u64VmcsField, u64Val));
+
+    if (   !fInVmxNonRootMode
+        && VM_IS_HM_ENABLED(pVCpu->CTX_SUFF(pVM)))
+    {
+        /* Notify HM that the VMCS content might have changed. */
+        HMNotifyVmxNstGstCurrentVmcsChanged(pVCpu);
+    }
+
+    iemVmxVmSucceed(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMWRITE instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmwrite(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+    Assert(pExitInfo);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint64_t u64Val;
+    uint8_t  iEffSeg;
+    if (pExitInfo->InstrInfo.VmreadVmwrite.fIsRegOperand)
+    {
+        u64Val  = iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg1);
+        iEffSeg = UINT8_MAX;
+    }
+    else
+    {
+        u64Val  = pExitInfo->GCPtrEffAddr;
+        iEffSeg = pExitInfo->InstrInfo.VmreadVmwrite.iSegReg;
+    }
+    uint8_t const  cbInstr     = pExitInfo->cbInstr;
+    uint64_t const u64FieldEnc = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+                               ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2)
+                               : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.VmreadVmwrite.iReg2);
+    VBOXSTRICTRC rcStrict = iemVmxVmwrite(pVCpu, cbInstr, iEffSeg, u64Val, u64FieldEnc, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMCLEAR instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iEffSeg     The effective segment register to use with @a GCPtrVmcs.
+ * @param   GCPtrVmcs   The linear address of the VMCS pointer.
+ * @param   pExitInfo   Pointer to the VM-exit information. Optional, can be NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+static VBOXSTRICTRC iemVmxVmclear(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg,
+                                  RTGCPHYS GCPtrVmcs, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMCLEAR, VMXINSTRID_NONE, cbInstr);
+    }
+
+    Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmclear_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Get the VMCS pointer from the location specified by the source memory operand. */
+    RTGCPHYS GCPhysVmcs;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU64(pVCpu, &GCPhysVmcs, iEffSeg, GCPtrVmcs);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: Failed to read VMCS physaddr from %#RGv, rc=%Rrc\n", GCPtrVmcs, VBOXSTRICTRC_VAL(rcStrict)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmclear_PtrMap;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmcs;
+        return rcStrict;
+    }
+
+    /* VMCS pointer alignment. */
+    if (!(GCPhysVmcs & X86_PAGE_4K_OFFSET_MASK))
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: VMCS pointer not page-aligned -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmclear_PtrAlign;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_INVALID_PHYSADDR);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* VMCS physical-address width limits. */
+    if (!(GCPhysVmcs >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth))
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: VMCS pointer extends beyond physical-address width -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmclear_PtrWidth;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_INVALID_PHYSADDR);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* VMCS is not the VMXON region. */
+    if (GCPhysVmcs != pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon)
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: VMCS pointer cannot be identical to VMXON region pointer -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmclear_PtrVmxon;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_VMXON_PTR);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* Ensure VMCS is not MMIO, ROM etc. This is not an Intel requirement but a
+       restriction imposed by our implementation. */
+    if (PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcs))
+    { /* likely */ }
+    else
+    {
+        Log(("vmclear: VMCS not normal memory -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmclear_PtrAbnormal;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMCLEAR_INVALID_PHYSADDR);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /*
+     * VMCLEAR allows committing and clearing any valid VMCS pointer.
+     *
+     * If the current VMCS is the one being cleared, set its state to 'clear' and commit
+     * to guest memory. Otherwise, set the state of the VMCS referenced in guest memory
+     * to 'clear'.
+     */
+    uint8_t const fVmcsLaunchStateClear = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
+    if (   IEM_VMX_HAS_CURRENT_VMCS(pVCpu)
+        && IEM_VMX_GET_CURRENT_VMCS(pVCpu) == GCPhysVmcs)
+    {
+        pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.fVmcsState = fVmcsLaunchStateClear;
+        iemVmxWriteCurrentVmcsToGstMem(pVCpu);
+        IEM_VMX_CLEAR_CURRENT_VMCS(pVCpu);
+    }
+    else
+    {
+        AssertCompileMemberSize(VMXVVMCS, fVmcsState, sizeof(fVmcsLaunchStateClear));
+        rcStrict = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), GCPhysVmcs + RT_UOFFSETOF(VMXVVMCS, fVmcsState),
+                                            (const void *)&fVmcsLaunchStateClear, sizeof(fVmcsLaunchStateClear));
+        if (RT_FAILURE(rcStrict))
+            return rcStrict;
+    }
+
+    iemVmxVmSucceed(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMCLEAR instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmclear(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(pExitInfo);
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const iEffSeg   = pExitInfo->InstrInfo.VmxXsave.iSegReg;
+    uint8_t const cbInstr   = pExitInfo->cbInstr;
+    RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
+    VBOXSTRICTRC rcStrict = iemVmxVmclear(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMPTRST instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iEffSeg     The effective segment register to use with @a GCPtrVmcs.
+ * @param   GCPtrVmcs   The linear address of where to store the current VMCS
+ *                      pointer.
+ * @param   pExitInfo   Pointer to the VM-exit information. Optional, can be NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+static VBOXSTRICTRC iemVmxVmptrst(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg,
+                                  RTGCPHYS GCPtrVmcs, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMPTRST, VMXINSTRID_NONE, cbInstr);
+    }
+
+    Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrst: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrst_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Set the VMCS pointer to the location specified by the destination memory operand. */
+    AssertCompile(NIL_RTGCPHYS == ~(RTGCPHYS)0U);
+    VBOXSTRICTRC rcStrict = iemMemStoreDataU64(pVCpu, iEffSeg, GCPtrVmcs, IEM_VMX_GET_CURRENT_VMCS(pVCpu));
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        iemVmxVmSucceed(pVCpu);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    Log(("vmptrst: Failed to store VMCS pointer to memory at destination operand %#Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrst_PtrMap;
+    pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmcs;
+    return rcStrict;
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMPTRST instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmptrst(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(pExitInfo);
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const iEffSeg   = pExitInfo->InstrInfo.VmxXsave.iSegReg;
+    uint8_t const cbInstr   = pExitInfo->cbInstr;
+    RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
+    VBOXSTRICTRC rcStrict = iemVmxVmptrst(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * VMPTRLD instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   GCPtrVmcs   The linear address of the current VMCS pointer.
+ * @param   pExitInfo   Pointer to the VM-exit information. Optional, can be NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+static VBOXSTRICTRC iemVmxVmptrld(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg,
+                                  RTGCPHYS GCPtrVmcs, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMPTRLD, VMXINSTRID_NONE, cbInstr);
+    }
+
+    Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Get the VMCS pointer from the location specified by the source memory operand. */
+    RTGCPHYS GCPhysVmcs;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU64(pVCpu, &GCPhysVmcs, iEffSeg, GCPtrVmcs);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: Failed to read VMCS physaddr from %#RGv, rc=%Rrc\n", GCPtrVmcs, VBOXSTRICTRC_VAL(rcStrict)));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrMap;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmcs;
+        return rcStrict;
+    }
+
+    /* VMCS pointer alignment. */
+    if (!(GCPhysVmcs & X86_PAGE_4K_OFFSET_MASK))
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: VMCS pointer not page-aligned -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrAlign;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INVALID_PHYSADDR);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* VMCS physical-address width limits. */
+    if (!(GCPhysVmcs >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth))
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: VMCS pointer extends beyond physical-address width -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrWidth;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INVALID_PHYSADDR);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* VMCS is not the VMXON region. */
+    if (GCPhysVmcs != pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon)
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: VMCS pointer cannot be identical to VMXON region pointer -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrVmxon;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_VMXON_PTR);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* Ensure VMCS is not MMIO, ROM etc. This is not an Intel requirement but a
+       restriction imposed by our implementation. */
+    if (PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmcs))
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: VMCS not normal memory -> VMFail()\n"));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrAbnormal;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INVALID_PHYSADDR);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* Read just the VMCS revision from the VMCS. */
+    VMXVMCSREVID VmcsRevId;
+    int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcsRevId, GCPhysVmcs, sizeof(VmcsRevId));
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+    {
+        Log(("vmptrld: Failed to read revision identifier from VMCS at %#RGp, rc=%Rrc\n", GCPhysVmcs, rc));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_RevPtrReadPhys;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+        return rc;
+    }
+
+    /*
+     * Verify the VMCS revision specified by the guest matches what we reported to the guest.
+     * Verify the VMCS is not a shadow VMCS, if the VMCS shadowing feature is supported.
+     */
+    if (   VmcsRevId.n.u31RevisionId == VMX_V_VMCS_REVISION_ID
+        && (   !VmcsRevId.n.fIsShadowVmcs
+            ||  IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxVmcsShadowing))
+    { /* likely */ }
+    else
+    {
+        if (VmcsRevId.n.u31RevisionId != VMX_V_VMCS_REVISION_ID)
+        {
+            Log(("vmptrld: VMCS revision mismatch, expected %#RX32 got %#RX32, GCPtrVmcs=%#RGv GCPhysVmcs=%#RGp -> VMFail()\n",
+                 VMX_V_VMCS_REVISION_ID, VmcsRevId.n.u31RevisionId, GCPtrVmcs, GCPhysVmcs));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_VmcsRevId;
+        }
+        else
+        {
+            Log(("vmptrld: Shadow VMCS -> VMFail()\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmptrld_ShadowVmcs;
+        }
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMPTRLD_INCORRECT_VMCS_REV);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /*
+     * We cache only the current VMCS in CPUMCTX. Therefore, VMPTRLD should always flush
+     * the cache of an existing, current VMCS back to guest memory before loading a new,
+     * different current VMCS.
+     */
+    if (IEM_VMX_GET_CURRENT_VMCS(pVCpu) != GCPhysVmcs)
+    {
+        if (IEM_VMX_HAS_CURRENT_VMCS(pVCpu))
+        {
+            iemVmxWriteCurrentVmcsToGstMem(pVCpu);
+            IEM_VMX_CLEAR_CURRENT_VMCS(pVCpu);
+        }
+
+        /* Set the new VMCS as the current VMCS and read it from guest memory. */
+        IEM_VMX_SET_CURRENT_VMCS(pVCpu, GCPhysVmcs);
+        rc = iemVmxReadCurrentVmcsFromGstMem(pVCpu);
+        if (RT_SUCCESS(rc))
+        {
+            /* Notify HM that a new, current VMCS is loaded. */
+            if (VM_IS_HM_ENABLED(pVCpu->CTX_SUFF(pVM)))
+                HMNotifyVmxNstGstCurrentVmcsChanged(pVCpu);
+        }
+        else
+        {
+            Log(("vmptrld: Failed to read VMCS at %#RGp, rc=%Rrc\n", GCPhysVmcs, rc));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmptrld_PtrReadPhys;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmcs;
+            return rc;
+        }
+    }
+
+    Assert(IEM_VMX_HAS_CURRENT_VMCS(pVCpu));
+    iemVmxVmSucceed(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMPTRLD instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmptrld(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(pExitInfo);
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const iEffSeg   = pExitInfo->InstrInfo.VmxXsave.iSegReg;
+    uint8_t const cbInstr   = pExitInfo->cbInstr;
+    RTGCPTR const GCPtrVmcs = pExitInfo->GCPtrEffAddr;
+    VBOXSTRICTRC rcStrict = iemVmxVmptrld(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * INVVPID instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ * @param   iEffSeg             The segment of the invvpid descriptor.
+ * @param   GCPtrInvvpidDesc    The address of invvpid descriptor.
+ * @param   u64InvvpidType      The invalidation type.
+ * @param   pExitInfo           Pointer to the VM-exit information. Optional, can be
+ *                              NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+VBOXSTRICTRC iemVmxInvvpid(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrInvvpidDesc,
+                           uint64_t u64InvvpidType, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    /* Check if INVVPID instruction is supported, otherwise raise #UD. */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxVpid)
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_INVVPID, VMXINSTRID_NONE, cbInstr);
+    }
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("invvpid: CPL != 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Validate INVVPID invalidation type.
+     *
+     * The instruction specifies exactly ONE of the supported invalidation types.
+     *
+     * Each of the types has a bit in IA32_VMX_EPT_VPID_CAP MSR specifying if it is
+     * supported. In theory, it's possible for a CPU to not support flushing individual
+     * addresses but all the other types or any other combination. We do not take any
+     * shortcuts here by assuming the types we currently expose to the guest.
+     */
+    uint64_t const fCaps = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64EptVpidCaps;
+    bool const fInvvpidSupported           = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID);
+    bool const fTypeIndivAddr              = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
+    bool const fTypeSingleCtx              = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_SINGLE_CTX);
+    bool const fTypeAllCtx                 = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_ALL_CTX);
+    bool const fTypeSingleCtxRetainGlobals = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVVPID_SINGLE_CTX_RETAIN_GLOBALS);
+
+    bool afSupportedTypes[4];
+    afSupportedTypes[0] = fTypeIndivAddr;
+    afSupportedTypes[1] = fTypeSingleCtx;
+    afSupportedTypes[2] = fTypeAllCtx;
+    afSupportedTypes[3] = fTypeSingleCtxRetainGlobals;
+
+    if (   fInvvpidSupported
+        && !(u64InvvpidType & ~(uint64_t)VMX_INVVPID_VALID_MASK)
+        && afSupportedTypes[u64InvvpidType & 3])
+    { /* likely */ }
+    else
+    {
+        Log(("invvpid: invalid/unsupported invvpid type %#x -> VMFail\n", u64InvvpidType));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_TypeInvalid;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InvvpidType;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /*
+     * Fetch the invvpid descriptor from guest memory.
+     */
+    RTUINT128U uDesc;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU128(pVCpu, &uDesc, iEffSeg, GCPtrInvvpidDesc);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * Validate the descriptor.
+         */
+        if (uDesc.s.Lo <= 0xffff)
+        { /* likely */ }
+        else
+        {
+            Log(("invvpid: reserved bits set in invvpid descriptor %#RX64 -> #GP(0)\n", uDesc.s.Lo));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_DescRsvd;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = uDesc.s.Lo;
+            iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+        RTGCUINTPTR64 const GCPtrInvAddr = uDesc.s.Hi;
+        uint16_t      const uVpid        = uDesc.Words.w0;
+        uint64_t      const uCr3         = pVCpu->cpum.GstCtx.cr3;
+        switch (u64InvvpidType)
+        {
+            case VMXTLBFLUSHVPID_INDIV_ADDR:
+            {
+                if (uVpid != 0)
+                {
+                    if (IEM_IS_CANONICAL(GCPtrInvAddr))
+                    {
+                        /* Invalidate mappings for the linear address tagged with VPID. */
+                        /** @todo PGM support for VPID? Currently just flush everything. */
+                        PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+                        iemVmxVmSucceed(pVCpu);
+                    }
+                    else
+                    {
+                        Log(("invvpid: invalidation address %#RGP is not canonical -> VMFail\n", GCPtrInvAddr));
+                        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_Type0InvalidAddr;
+                        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrInvAddr;
+                        iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+                    }
+                }
+                else
+                {
+                    Log(("invvpid: invalid VPID %#x for invalidation type %u -> VMFail\n", uVpid, u64InvvpidType));
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_Type0InvalidVpid;
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InvvpidType;
+                    iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+                }
+                break;
+            }
+
+            case VMXTLBFLUSHVPID_SINGLE_CONTEXT:
+            {
+                if (uVpid != 0)
+                {
+                    /* Invalidate all mappings with VPID. */
+                    /** @todo PGM support for VPID? Currently just flush everything. */
+                    PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+                    iemVmxVmSucceed(pVCpu);
+                }
+                else
+                {
+                    Log(("invvpid: invalid VPID %#x for invalidation type %u -> VMFail\n", uVpid, u64InvvpidType));
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_Type1InvalidVpid;
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InvvpidType;
+                    iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+                }
+                break;
+            }
+
+            case VMXTLBFLUSHVPID_ALL_CONTEXTS:
+            {
+                /* Invalidate all mappings with non-zero VPIDs. */
+                /** @todo PGM support for VPID? Currently just flush everything. */
+                PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+                iemVmxVmSucceed(pVCpu);
+                break;
+            }
+
+            case VMXTLBFLUSHVPID_SINGLE_CONTEXT_RETAIN_GLOBALS:
+            {
+                if (uVpid != 0)
+                {
+                    /* Invalidate all mappings with VPID except global translations. */
+                    /** @todo PGM support for VPID? Currently just flush everything. */
+                    PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+                    iemVmxVmSucceed(pVCpu);
+                }
+                else
+                {
+                    Log(("invvpid: invalid VPID %#x for invalidation type %u -> VMFail\n", uVpid, u64InvvpidType));
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invvpid_Type3InvalidVpid;
+                    pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = uVpid;
+                    iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+                }
+                break;
+            }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVVPID instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvvpid(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 4);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+    Assert(pExitInfo);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const  iEffSeg          = pExitInfo->InstrInfo.Inv.iSegReg;
+    uint8_t const  cbInstr          = pExitInfo->cbInstr;
+    RTGCPTR const  GCPtrInvvpidDesc = pExitInfo->GCPtrEffAddr;
+    uint64_t const u64InvvpidType   = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+                                    ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.Inv.iReg2)
+                                    : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.Inv.iReg2);
+    VBOXSTRICTRC rcStrict = iemVmxInvvpid(pVCpu, cbInstr, iEffSeg, GCPtrInvvpidDesc, u64InvvpidType, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+
+/**
+ * INVEPT instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length in bytes.
+ * @param   iEffSeg             The segment of the invept descriptor.
+ * @param   GCPtrInveptDesc     The address of invept descriptor.
+ * @param   u64InveptType       The invalidation type.
+ * @param   pExitInfo           Pointer to the VM-exit information. Optional, can be
+ *                              NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. VMX operation, CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+static VBOXSTRICTRC iemVmxInvept(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg, RTGCPTR GCPtrInveptDesc,
+                                 uint64_t u64InveptType, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    /* Check if EPT is supported, otherwise raise #UD. */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmxEpt)
+        return iemRaiseUndefinedOpcode(pVCpu);
+
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_INVEPT, VMXINSTRID_NONE, cbInstr);
+    }
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl != 0)
+    {
+        Log(("invept: CPL != 0 -> #GP(0)\n"));
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /*
+     * Validate INVEPT invalidation type.
+     *
+     * The instruction specifies exactly ONE of the supported invalidation types.
+     *
+     * Each of the types has a bit in IA32_VMX_EPT_VPID_CAP MSR specifying if it is
+     * supported. In theory, it's possible for a CPU to not support flushing individual
+     * addresses but all the other types or any other combination. We do not take any
+     * shortcuts here by assuming the types we currently expose to the guest.
+     */
+    uint64_t const fCaps = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64EptVpidCaps;
+    bool const fInveptSupported = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVEPT);
+    bool const fTypeSingleCtx   = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVEPT_SINGLE_CTX);
+    bool const fTypeAllCtx      = RT_BF_GET(fCaps, VMX_BF_EPT_VPID_CAP_INVEPT_ALL_CTX);
+
+    bool afSupportedTypes[4];
+    afSupportedTypes[0] = false;
+    afSupportedTypes[1] = fTypeSingleCtx;
+    afSupportedTypes[2] = fTypeAllCtx;
+    afSupportedTypes[3] = false;
+
+    if (   fInveptSupported
+        && !(u64InveptType & ~(uint64_t)VMX_INVEPT_VALID_MASK)
+        && afSupportedTypes[u64InveptType & 3])
+    { /* likely */ }
+    else
+    {
+        Log(("invept: invalid/unsupported invvpid type %#x -> VMFail\n", u64InveptType));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invept_TypeInvalid;
+        pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = u64InveptType;
+        iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /*
+     * Fetch the invept descriptor from guest memory.
+     */
+    RTUINT128U uDesc;
+    VBOXSTRICTRC rcStrict = iemMemFetchDataU128(pVCpu, &uDesc, iEffSeg, GCPtrInveptDesc);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * Validate the descriptor.
+         *
+         * The Intel spec. does not explicit say the INVEPT instruction fails when reserved
+         * bits in the descriptor are set, but it -does- for INVVPID. Until we test on real
+         * hardware, it's assumed INVEPT behaves the same as INVVPID in this regard. It's
+         * better to be strict in our emulation until proven otherwise.
+         */
+        if (uDesc.s.Hi)
+        {
+            Log(("invept: reserved bits set in invept descriptor %#RX64 -> VMFail\n", uDesc.s.Hi));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invept_DescRsvd;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = uDesc.s.Hi;
+            iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+
+        /*
+         * Flush TLB mappings based on the EPT type.
+         */
+        if (u64InveptType == VMXTLBFLUSHEPT_SINGLE_CONTEXT)
+        {
+            uint64_t const GCPhysEptPtr = uDesc.s.Lo;
+            int const rc = iemVmxVmentryCheckEptPtr(pVCpu, GCPhysEptPtr, NULL /* enmDiag */);
+            if (RT_SUCCESS(rc))
+            { /* likely */ }
+            else
+            {
+                Log(("invept: EPTP invalid %#RX64 -> VMFail\n", GCPhysEptPtr));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Invept_EptpInvalid;
+                pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysEptPtr;
+                iemVmxVmFail(pVCpu, VMXINSTRERR_INVEPT_INVVPID_INVALID_OPERAND);
+                return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            }
+        }
+
+        /** @todo PGM support for EPT tags? Currently just flush everything. */
+        IEM_CTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+        uint64_t const uCr3 = pVCpu->cpum.GstCtx.cr3;
+        PGMFlushTLB(pVCpu, uCr3, true /* fGlobal */);
+
+        iemVmxVmSucceed(pVCpu);
+        rcStrict = iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Interface for HM and EM to emulate the INVEPT instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedInvept(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 4);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+    Assert(pExitInfo);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const  iEffSeg          = pExitInfo->InstrInfo.Inv.iSegReg;
+    uint8_t const  cbInstr          = pExitInfo->cbInstr;
+    RTGCPTR const  GCPtrInveptDesc  = pExitInfo->GCPtrEffAddr;
+    uint64_t const u64InveptType    = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+                                    ? iemGRegFetchU64(pVCpu, pExitInfo->InstrInfo.Inv.iReg2)
+                                    : iemGRegFetchU32(pVCpu, pExitInfo->InstrInfo.Inv.iReg2);
+    VBOXSTRICTRC rcStrict = iemVmxInvept(pVCpu, cbInstr, iEffSeg, GCPtrInveptDesc, u64InveptType, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+/**
+ * VMXON instruction execution worker.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     The instruction length in bytes.
+ * @param   iEffSeg     The effective segment register to use with @a
+ *                      GCPtrVmxon.
+ * @param   GCPtrVmxon  The linear address of the VMXON pointer.
+ * @param   pExitInfo   Pointer to the VM-exit information. Optional, can be NULL.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+static VBOXSTRICTRC iemVmxVmxon(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iEffSeg,
+                                RTGCPHYS GCPtrVmxon, PCVMXVEXITINFO pExitInfo) RT_NOEXCEPT
+{
+    if (!IEM_VMX_IS_ROOT_MODE(pVCpu))
+    {
+        /* CPL. */
+        if (pVCpu->iem.s.uCpl == 0)
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cpl;
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        /* A20M (A20 Masked) mode. */
+        if (PGMPhysIsA20Enabled(pVCpu))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: A20M mode -> #GP(0)\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_A20M;
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        /* CR0. */
+        {
+            /*
+             * CR0 MB1 bits.
+             *
+             * We use VMX_V_CR0_FIXED0 below to ensure CR0.PE and CR0.PG are always set
+             * while executing VMXON. CR0.PE and CR0.PG are only allowed to be clear
+             * when the guest running in VMX non-root mode with unrestricted-guest control
+             * enabled in the VMCS.
+             */
+            uint64_t const uCr0Fixed0 = VMX_V_CR0_FIXED0;
+            if ((pVCpu->cpum.GstCtx.cr0 & uCr0Fixed0) == uCr0Fixed0)
+            { /* likely */ }
+            else
+            {
+                Log(("vmxon: CR0 fixed0 bits cleared -> #GP(0)\n"));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr0Fixed0;
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            /* CR0 MBZ bits. */
+            uint64_t const uCr0Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr0Fixed1;
+            if (!(pVCpu->cpum.GstCtx.cr0 & ~uCr0Fixed1))
+            { /* likely */ }
+            else
+            {
+                Log(("vmxon: CR0 fixed1 bits set -> #GP(0)\n"));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr0Fixed1;
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+        }
+
+        /* CR4. */
+        {
+            /* CR4 MB1 bits. */
+            uint64_t const uCr4Fixed0 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed0;
+            if ((pVCpu->cpum.GstCtx.cr4 & uCr4Fixed0) == uCr4Fixed0)
+            { /* likely */ }
+            else
+            {
+                Log(("vmxon: CR4 fixed0 bits cleared -> #GP(0)\n"));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr4Fixed0;
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+
+            /* CR4 MBZ bits. */
+            uint64_t const uCr4Fixed1 = pVCpu->cpum.GstCtx.hwvirt.vmx.Msrs.u64Cr4Fixed1;
+            if (!(pVCpu->cpum.GstCtx.cr4 & ~uCr4Fixed1))
+            { /* likely */ }
+            else
+            {
+                Log(("vmxon: CR4 fixed1 bits set -> #GP(0)\n"));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_Cr4Fixed1;
+                return iemRaiseGeneralProtectionFault0(pVCpu);
+            }
+        }
+
+        /* Feature control MSR's LOCK and VMXON bits. */
+        uint64_t const uMsrFeatCtl = CPUMGetGuestIa32FeatCtrl(pVCpu);
+        if ((uMsrFeatCtl & (MSR_IA32_FEATURE_CONTROL_LOCK | MSR_IA32_FEATURE_CONTROL_VMXON))
+                        == (MSR_IA32_FEATURE_CONTROL_LOCK | MSR_IA32_FEATURE_CONTROL_VMXON))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: Feature control lock bit or VMXON bit cleared -> #GP(0)\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_MsrFeatCtl;
+            return iemRaiseGeneralProtectionFault0(pVCpu);
+        }
+
+        /* Get the VMXON pointer from the location specified by the source memory operand. */
+        RTGCPHYS GCPhysVmxon;
+        VBOXSTRICTRC rcStrict = iemMemFetchDataU64(pVCpu, &GCPhysVmxon, iEffSeg, GCPtrVmxon);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: Failed to read VMXON region physaddr from %#RGv, rc=%Rrc\n", GCPtrVmxon, VBOXSTRICTRC_VAL(rcStrict)));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmxon_PtrMap;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPtrVmxon;
+            return rcStrict;
+        }
+
+        /* VMXON region pointer alignment. */
+        if (!(GCPhysVmxon & X86_PAGE_4K_OFFSET_MASK))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: VMXON region pointer not page-aligned -> VMFailInvalid\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmxon_PtrAlign;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmxon;
+            iemVmxVmFailInvalid(pVCpu);
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+
+        /* VMXON physical-address width limits. */
+        if (!(GCPhysVmxon >> IEM_GET_GUEST_CPU_FEATURES(pVCpu)->cVmxMaxPhysAddrWidth))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: VMXON region pointer extends beyond physical-address width -> VMFailInvalid\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmxon_PtrWidth;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmxon;
+            iemVmxVmFailInvalid(pVCpu);
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+
+        /* Ensure VMXON region is not MMIO, ROM etc. This is not an Intel requirement but a
+           restriction imposed by our implementation. */
+        if (PGMPhysIsGCPhysNormal(pVCpu->CTX_SUFF(pVM), GCPhysVmxon))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: VMXON region not normal memory -> VMFailInvalid\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag  = kVmxVDiag_Vmxon_PtrAbnormal;
+            pVCpu->cpum.GstCtx.hwvirt.vmx.uDiagAux = GCPhysVmxon;
+            iemVmxVmFailInvalid(pVCpu);
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+
+        /* Read the VMCS revision ID from the VMXON region. */
+        VMXVMCSREVID VmcsRevId;
+        int rc = PGMPhysSimpleReadGCPhys(pVCpu->CTX_SUFF(pVM), &VmcsRevId, GCPhysVmxon, sizeof(VmcsRevId));
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+        {
+            Log(("vmxon: Failed to read VMXON region at %#RGp, rc=%Rrc\n", GCPhysVmxon, rc));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_PtrReadPhys;
+            return rc;
+        }
+
+        /* Verify the VMCS revision specified by the guest matches what we reported to the guest. */
+        if (RT_LIKELY(VmcsRevId.u == VMX_V_VMCS_REVISION_ID))
+        { /* likely */ }
+        else
+        {
+            /* Revision ID mismatch. */
+            if (!VmcsRevId.n.fIsShadowVmcs)
+            {
+                Log(("vmxon: VMCS revision mismatch, expected %#RX32 got %#RX32 -> VMFailInvalid\n", VMX_V_VMCS_REVISION_ID,
+                     VmcsRevId.n.u31RevisionId));
+                pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_VmcsRevId;
+                iemVmxVmFailInvalid(pVCpu);
+                return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+            }
+
+            /* Shadow VMCS disallowed. */
+            Log(("vmxon: Shadow VMCS -> VMFailInvalid\n"));
+            pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_ShadowVmcs;
+            iemVmxVmFailInvalid(pVCpu);
+            return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+        }
+
+        /*
+         * Record that we're in VMX operation, block INIT, block and disable A20M.
+         */
+        pVCpu->cpum.GstCtx.hwvirt.vmx.GCPhysVmxon    = GCPhysVmxon;
+        IEM_VMX_CLEAR_CURRENT_VMCS(pVCpu);
+        pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxRootMode = true;
+
+        /* Clear address-range monitoring. */
+        EMMonitorWaitClear(pVCpu);
+        /** @todo NSTVMX: Intel PT. */
+
+        iemVmxVmSucceed(pVCpu);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        /* Nested-guest intercept. */
+        if (pExitInfo)
+            return iemVmxVmexitInstrWithInfo(pVCpu, pExitInfo);
+        return iemVmxVmexitInstrNeedsInfo(pVCpu, VMX_EXIT_VMXON, VMXINSTRID_NONE, cbInstr);
+    }
+
+    Assert(IEM_VMX_IS_ROOT_MODE(pVCpu));
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl > 0)
+    {
+        Log(("vmxon: In VMX root mode: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_VmxRootCpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* VMXON when already in VMX root mode. */
+    iemVmxVmFail(pVCpu, VMXINSTRERR_VMXON_IN_VMXROOTMODE);
+    pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxon_VmxAlreadyRoot;
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMXON instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pExitInfo       Pointer to the VM-exit information.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxon(PVMCPUCC pVCpu, PCVMXVEXITINFO pExitInfo)
+{
+    Assert(pExitInfo);
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(pExitInfo->cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+
+    uint8_t const iEffSeg    = pExitInfo->InstrInfo.VmxXsave.iSegReg;
+    uint8_t const cbInstr    = pExitInfo->cbInstr;
+    RTGCPTR const GCPtrVmxon = pExitInfo->GCPtrEffAddr;
+    VBOXSTRICTRC rcStrict = iemVmxVmxon(pVCpu, cbInstr, iEffSeg, GCPtrVmxon, pExitInfo);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Implements 'VMXOFF'.
+ *
+ * @remarks Common VMX instruction checks are already expected to by the caller,
+ *          i.e. CR4.VMXE, Real/V86 mode, EFER/CS.L checks.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmxoff)
+{
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        return iemVmxVmexitInstr(pVCpu, VMX_EXIT_VMXOFF, cbInstr);
+
+    /* CPL. */
+    if (pVCpu->iem.s.uCpl == 0)
+    { /* likely */ }
+    else
+    {
+        Log(("vmxoff: CPL %u -> #GP(0)\n", pVCpu->iem.s.uCpl));
+        pVCpu->cpum.GstCtx.hwvirt.vmx.enmDiag = kVmxVDiag_Vmxoff_Cpl;
+        return iemRaiseGeneralProtectionFault0(pVCpu);
+    }
+
+    /* Dual monitor treatment of SMIs and SMM. */
+    uint64_t const fSmmMonitorCtl = CPUMGetGuestIa32SmmMonitorCtl(pVCpu);
+    if (!(fSmmMonitorCtl & MSR_IA32_SMM_MONITOR_VALID))
+    { /* likely */ }
+    else
+    {
+        iemVmxVmFail(pVCpu, VMXINSTRERR_VMXOFF_DUAL_MON);
+        return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+    }
+
+    /* Record that we're no longer in VMX root operation, block INIT, block and disable A20M. */
+    pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxRootMode = false;
+    Assert(!pVCpu->cpum.GstCtx.hwvirt.vmx.fInVmxNonRootMode);
+
+    if (fSmmMonitorCtl & MSR_IA32_SMM_MONITOR_VMXOFF_UNBLOCK_SMI)
+    { /** @todo NSTVMX: Unblock SMI. */ }
+
+    EMMonitorWaitClear(pVCpu);
+    /** @todo NSTVMX: Unblock and enable A20M. */
+
+    iemVmxVmSucceed(pVCpu);
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+
+/**
+ * Interface for HM and EM to emulate the VMXOFF instruction.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   cbInstr     The instruction length in bytes.
+ * @thread  EMT(pVCpu)
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IEMExecDecodedVmxoff(PVMCPUCC pVCpu, uint8_t cbInstr)
+{
+    IEMEXEC_ASSERT_INSTR_LEN_RETURN(cbInstr, 3);
+    IEM_CTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+
+    iemInitExec(pVCpu, false /*fBypassHandlers*/);
+    VBOXSTRICTRC rcStrict = IEM_CIMPL_CALL_0(iemCImpl_vmxoff);
+    Assert(!pVCpu->iem.s.cActiveMappings);
+    return iemUninitExecAndFiddleStatusAndMaybeReenter(pVCpu, rcStrict);
+}
+
+
+/**
+ * Implements 'VMXON'.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmxon, uint8_t, iEffSeg, RTGCPTR, GCPtrVmxon)
+{
+    return iemVmxVmxon(pVCpu, cbInstr, iEffSeg, GCPtrVmxon, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMLAUNCH'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmlaunch)
+{
+    return iemVmxVmlaunchVmresume(pVCpu, cbInstr, VMXINSTRID_VMLAUNCH);
+}
+
+
+/**
+ * Implements 'VMRESUME'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmresume)
+{
+    return iemVmxVmlaunchVmresume(pVCpu, cbInstr, VMXINSTRID_VMRESUME);
+}
+
+
+/**
+ * Implements 'VMPTRLD'.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmptrld, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs)
+{
+    return iemVmxVmptrld(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMPTRST'.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmptrst, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs)
+{
+    return iemVmxVmptrst(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMCLEAR'.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmclear, uint8_t, iEffSeg, RTGCPTR, GCPtrVmcs)
+{
+    return iemVmxVmclear(pVCpu, cbInstr, iEffSeg, GCPtrVmcs, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMWRITE' register.
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmwrite_reg, uint64_t, u64Val, uint64_t, u64VmcsField)
+{
+    return iemVmxVmwrite(pVCpu, cbInstr, UINT8_MAX /* iEffSeg */, u64Val, u64VmcsField, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMWRITE' memory.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_vmwrite_mem, uint8_t, iEffSeg, RTGCPTR, GCPtrVal, uint32_t, u64VmcsField)
+{
+    return iemVmxVmwrite(pVCpu, cbInstr, iEffSeg, GCPtrVal, u64VmcsField,  NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMREAD' register (64-bit).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmread_reg64, uint64_t *, pu64Dst, uint64_t, u64VmcsField)
+{
+    return iemVmxVmreadReg64(pVCpu, cbInstr, pu64Dst, u64VmcsField, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMREAD' register (32-bit).
+ */
+IEM_CIMPL_DEF_2(iemCImpl_vmread_reg32, uint32_t *, pu32Dst, uint32_t, u32VmcsField)
+{
+    return iemVmxVmreadReg32(pVCpu, cbInstr, pu32Dst, u32VmcsField, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMREAD' memory, 64-bit register.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_vmread_mem_reg64, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u64VmcsField)
+{
+    return iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u64VmcsField, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'VMREAD' memory, 32-bit register.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_vmread_mem_reg32, uint8_t, iEffSeg, RTGCPTR, GCPtrDst, uint32_t, u32VmcsField)
+{
+    return iemVmxVmreadMem(pVCpu, cbInstr, iEffSeg, GCPtrDst, u32VmcsField, NULL /* pExitInfo */);
+}
+
+
+/**
+ * Implements 'INVVPID'.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_invvpid, uint8_t, iEffSeg, RTGCPTR, GCPtrInvvpidDesc, uint64_t, uInvvpidType)
+{
+    return iemVmxInvvpid(pVCpu, cbInstr, iEffSeg, GCPtrInvvpidDesc, uInvvpidType, NULL /* pExitInfo */);
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * Implements 'INVEPT'.
+ */
+IEM_CIMPL_DEF_3(iemCImpl_invept, uint8_t, iEffSeg, RTGCPTR, GCPtrInveptDesc, uint64_t, uInveptType)
+{
+    return iemVmxInvept(pVCpu, cbInstr, iEffSeg, GCPtrInveptDesc, uInveptType, NULL /* pExitInfo */);
+}
+#endif
+
+
+/**
+ * Implements VMX's implementation of PAUSE.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmx_pause)
+{
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+    {
+        VBOXSTRICTRC rcStrict = iemVmxVmexitInstrPause(pVCpu, cbInstr);
+        if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+            return rcStrict;
+    }
+
+    /*
+     * Outside VMX non-root operation or if the PAUSE instruction does not cause
+     * a VM-exit, the instruction operates normally.
+     */
+    return iemRegAddToRipAndFinishingClearingRF(pVCpu, cbInstr);
+}
+
+#endif  /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * Implements 'VMCALL'.
+ */
+IEM_CIMPL_DEF_0(iemCImpl_vmcall)
+{
+    pVCpu->iem.s.cPotentialExits++;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* Nested-guest intercept. */
+    if (IEM_VMX_IS_NON_ROOT_MODE(pVCpu))
+        return iemVmxVmexitInstr(pVCpu, VMX_EXIT_VMCALL, cbInstr);
+#endif
+
+    /* Join forces with vmmcall. */
+    return IEM_CIMPL_CALL_1(iemCImpl_Hypercall, OP_VMCALL);
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER, VMX APIC-access page accesses}
+ *
+ * @remarks The @a uUser argument is currently unused.
+ */
+DECLCALLBACK(VBOXSTRICTRC) iemVmxApicAccessPageHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys,
+                                                       void *pvBuf, size_t cbBuf, PGMACCESSTYPE enmAccessType,
+                                                       PGMACCESSORIGIN enmOrigin, uint64_t uUser)
+{
+    RT_NOREF3(pvPhys, enmOrigin, uUser);
+
+    RTGCPHYS const GCPhysAccessBase = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
+    {
+        Assert(CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(pVCpu), VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
+        Assert(CPUMGetGuestVmxApicAccessPageAddrEx(IEM_GET_CTX(pVCpu)) == GCPhysAccessBase);
+
+        uint32_t const fAccess   = enmAccessType == PGMACCESSTYPE_WRITE ? IEM_ACCESS_DATA_W : IEM_ACCESS_DATA_R;
+        uint16_t const offAccess = GCPhysFault & GUEST_PAGE_OFFSET_MASK;
+
+        LogFlowFunc(("Fault at %#RGp (cbBuf=%u fAccess=%#x)\n", GCPhysFault, cbBuf, fAccess));
+        VBOXSTRICTRC rcStrict = iemVmxVirtApicAccessMem(pVCpu, offAccess, cbBuf, pvBuf, fAccess);
+        if (RT_FAILURE(rcStrict))
+            return rcStrict;
+
+        /* Any access on this APIC-access page has been handled, caller should not carry out the access. */
+        return VINF_SUCCESS;
+    }
+
+    LogFunc(("Accessed outside VMX non-root mode, deregistering page handler for %#RGp\n", GCPhysAccessBase));
+    int rc = PGMHandlerPhysicalDeregister(pVM, GCPhysAccessBase);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* Instruct the caller of this handler to perform the read/write as normal memory. */
+    return VINF_PGM_HANDLER_DO_DEFAULT;
+}
+
+
+# ifndef IN_RING3
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      \#PF access handler callback for guest VMX APIC-access page.}
+ */
+DECLCALLBACK(VBOXSTRICTRC) iemVmxApicAccessPagePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
+                                                         RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
+
+{
+    RT_NOREF3(pVM, pCtx, uUser);
+
+    /*
+     * Handle the VMX APIC-access page only when the guest is in VMX non-root mode.
+     * Otherwise we must deregister the page and allow regular RAM access.
+     * Failing to do so lands us with endless EPT VM-exits.
+     */
+    RTGCPHYS const GCPhysPage = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    if (CPUMIsGuestInVmxNonRootMode(IEM_GET_CTX(pVCpu)))
+    {
+        Assert(CPUMIsGuestVmxProcCtls2Set(IEM_GET_CTX(pVCpu), VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
+        Assert(CPUMGetGuestVmxApicAccessPageAddrEx(IEM_GET_CTX(pVCpu)) == GCPhysPage);
+
+        /*
+         * Check if the access causes an APIC-access VM-exit.
+         */
+        uint32_t fAccess;
+        if (uErr & X86_TRAP_PF_ID)
+            fAccess = IEM_ACCESS_INSTRUCTION;
+        else if (uErr & X86_TRAP_PF_RW)
+            fAccess = IEM_ACCESS_DATA_W;
+        else
+            fAccess = IEM_ACCESS_DATA_R;
+
+        RTGCPHYS const GCPhysNestedFault = (RTGCPHYS)pvFault;
+        uint16_t const offAccess         = GCPhysNestedFault & GUEST_PAGE_OFFSET_MASK;
+        bool const fIntercept = iemVmxVirtApicIsMemAccessIntercepted(pVCpu, offAccess, 1 /* cbAccess */, fAccess);
+        LogFlowFunc(("#PF at %#RGp (GCPhysNestedFault=%#RGp offAccess=%#x)\n", GCPhysFault, GCPhysNestedFault, offAccess));
+        if (fIntercept)
+        {
+            /*
+             * Query the source VM-exit (from the execution engine) that caused this access
+             * within the APIC-access page. Currently only HM is supported.
+             */
+            AssertMsg(VM_IS_HM_ENABLED(pVM),
+                      ("VM-exit auxiliary info. fetching not supported for execution engine %d\n", pVM->bMainExecutionEngine));
+
+            HMEXITAUX HmExitAux;
+            RT_ZERO(HmExitAux);
+            int const rc = HMR0GetExitAuxInfo(pVCpu, &HmExitAux, HMVMX_READ_EXIT_INSTR_LEN
+                                                               | HMVMX_READ_EXIT_QUALIFICATION
+                                                               | HMVMX_READ_IDT_VECTORING_INFO
+                                                               | HMVMX_READ_IDT_VECTORING_ERROR_CODE);
+            AssertRC(rc);
+
+            /*
+             * Verify the VM-exit reason must be an EPT violation.
+             * Other accesses should go through the other handler (iemVmxApicAccessPageHandler).
+             * Refer to @bugref{10092#c33s} for a more detailed explanation.
+             */
+            AssertMsgReturn(HmExitAux.Vmx.uReason == VMX_EXIT_EPT_VIOLATION,
+                            ("Unexpected call to APIC-access page #PF handler for %#RGp offAcesss=%u uErr=%#RGx uReason=%u\n",
+                             GCPhysPage, offAccess, uErr, HmExitAux.Vmx.uReason), VERR_IEM_IPE_7);
+
+            /*
+             * Construct the virtual APIC-access VM-exit.
+             */
+            VMXAPICACCESS enmAccess;
+            if (HmExitAux.Vmx.u64Qual & VMX_EXIT_QUAL_EPT_LINEAR_ADDR_VALID)
+            {
+                if (VMX_IDT_VECTORING_INFO_IS_VALID(HmExitAux.Vmx.uIdtVectoringInfo))
+                    enmAccess = VMXAPICACCESS_LINEAR_EVENT_DELIVERY;
+                else if (fAccess == IEM_ACCESS_INSTRUCTION)
+                    enmAccess = VMXAPICACCESS_LINEAR_INSTR_FETCH;
+                else if (fAccess & IEM_ACCESS_TYPE_WRITE)
+                    enmAccess = VMXAPICACCESS_LINEAR_WRITE;
+                else
+                    enmAccess = VMXAPICACCESS_LINEAR_READ;
+
+                /* For linear-address accesss the instruction length must be valid. */
+                AssertMsg(HmExitAux.Vmx.cbInstr > 0,
+                          ("Invalid APIC-access VM-exit instruction length. cbInstr=%u\n", HmExitAux.Vmx.cbInstr));
+            }
+            else
+            {
+                if (VMX_IDT_VECTORING_INFO_IS_VALID(HmExitAux.Vmx.uIdtVectoringInfo))
+                    enmAccess = VMXAPICACCESS_PHYSICAL_EVENT_DELIVERY;
+                else
+                {
+                    /** @todo How to distinguish between monitoring/trace vs other instructions
+                     *        here? */
+                    enmAccess = VMXAPICACCESS_PHYSICAL_INSTR;
+                }
+
+                /* For physical accesses the instruction length is undefined, we zero it for safety and consistency. */
+                HmExitAux.Vmx.cbInstr = 0;
+            }
+
+            /*
+             * Raise the APIC-access VM-exit.
+             */
+            LogFlowFunc(("Raising APIC-access VM-exit from #PF handler at offset %#x\n", offAccess));
+            VMXVEXITINFO const ExitInfo
+                = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN(VMX_EXIT_APIC_ACCESS,
+                                                              RT_BF_MAKE(VMX_BF_EXIT_QUAL_APIC_ACCESS_OFFSET, offAccess)
+                                                            | RT_BF_MAKE(VMX_BF_EXIT_QUAL_APIC_ACCESS_TYPE,   enmAccess),
+                                                            HmExitAux.Vmx.cbInstr);
+            VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(HmExitAux.Vmx.uIdtVectoringInfo,
+                                                                                    HmExitAux.Vmx.uIdtVectoringErrCode);
+            VBOXSTRICTRC const      rcStrict      = iemVmxVmexitApicAccessWithInfo(pVCpu, &ExitInfo, &ExitEventInfo);
+            return iemExecStatusCodeFiddling(pVCpu, rcStrict);
+        }
+
+        /*
+         * The access isn't intercepted, which means it needs to be virtualized.
+         *
+         * This requires emulating the instruction because we need the bytes being
+         * read/written by the instruction not just the offset being accessed within
+         * the APIC-access page (which we derive from the faulting address).
+         */
+        LogFlowFunc(("Access at offset %#x not intercepted -> VINF_EM_RAW_EMULATE_INSTR\n", offAccess));
+        return VINF_EM_RAW_EMULATE_INSTR;
+    }
+
+    /** @todo This isn't ideal but works for now as nested-hypervisors generally play
+     *        nice because the spec states that this page should be modified only when
+     *        no CPU refers to it VMX non-root mode. Nonetheless, we could use an atomic
+     *        reference counter to ensure the aforementioned condition before
+     *        de-registering the page.  */
+    LogFunc(("Accessed outside VMX non-root mode, deregistering page handler for %#RGp\n", GCPhysPage));
+    int const rc = PGMHandlerPhysicalDeregister(pVM, GCPhysPage);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    return VINF_SUCCESS;
+}
+# endif /* !IN_RING3 */
+
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h
new file mode 100644
index 00000000..bf186b94
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructions3DNow.cpp.h
@@ -0,0 +1,143 @@
+/* $Id: IEMAllInstructions3DNow.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation, 3DNow!.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/** @name 3DNow! instructions (0x0f 0x0f)
+ *
+ * @{
+ */
+
+/** Opcode 0x0f 0x0f 0x0c. */
+FNIEMOP_STUB(iemOp_3Dnow_pi2fw_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x0d. */
+FNIEMOP_STUB(iemOp_3Dnow_pi2fd_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x1c. */
+FNIEMOP_STUB(iemOp_3Dnow_pf2fw_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x1d. */
+FNIEMOP_STUB(iemOp_3Dnow_pf2fd_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x8a. */
+FNIEMOP_STUB(iemOp_3Dnow_pfnacc_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x8e. */
+FNIEMOP_STUB(iemOp_3Dnow_pfpnacc_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x90. */
+FNIEMOP_STUB(iemOp_3Dnow_pfcmpge_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x94. */
+FNIEMOP_STUB(iemOp_3Dnow_pfmin_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x96. */
+FNIEMOP_STUB(iemOp_3Dnow_pfrcp_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x97. */
+FNIEMOP_STUB(iemOp_3Dnow_pfrsqrt_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x9a. */
+FNIEMOP_STUB(iemOp_3Dnow_pfsub_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0x9e. */
+FNIEMOP_STUB(iemOp_3Dnow_pfadd_PQ_Qq);
+
+/** Opcode 0x0f 0x0f 0xa0. */
+FNIEMOP_STUB(iemOp_3Dnow_pfcmpgt_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xa4. */
+FNIEMOP_STUB(iemOp_3Dnow_pfmax_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xa6. */
+FNIEMOP_STUB(iemOp_3Dnow_pfrcpit1_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xa7. */
+FNIEMOP_STUB(iemOp_3Dnow_pfrsqit1_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xaa. */
+FNIEMOP_STUB(iemOp_3Dnow_pfsubr_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xae. */
+FNIEMOP_STUB(iemOp_3Dnow_pfacc_PQ_Qq);
+
+/** Opcode 0x0f 0x0f 0xb0. */
+FNIEMOP_STUB(iemOp_3Dnow_pfcmpeq_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xb4. */
+FNIEMOP_STUB(iemOp_3Dnow_pfmul_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xb6. */
+FNIEMOP_STUB(iemOp_3Dnow_pfrcpit2_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xb7. */
+FNIEMOP_STUB(iemOp_3Dnow_pmulhrw_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xbb. */
+FNIEMOP_STUB(iemOp_3Dnow_pswapd_Pq_Qq);
+
+/** Opcode 0x0f 0x0f 0xbf. */
+FNIEMOP_STUB(iemOp_3Dnow_pavgusb_PQ_Qq);
+
+
+/** Opcode 0x0f 0x0f. */
+FNIEMOP_DEF_1(iemOp_3DNowDispatcher, uint8_t, b)
+{
+    /* This is pretty sparse, use switch instead of table. */
+    switch (b)
+    {
+        case 0x0c: return FNIEMOP_CALL(iemOp_3Dnow_pi2fw_Pq_Qq);
+        case 0x0d: return FNIEMOP_CALL(iemOp_3Dnow_pi2fd_Pq_Qq);
+        case 0x1c: return FNIEMOP_CALL(iemOp_3Dnow_pf2fw_Pq_Qq);
+        case 0x1d: return FNIEMOP_CALL(iemOp_3Dnow_pf2fd_Pq_Qq);
+        case 0x8a: return FNIEMOP_CALL(iemOp_3Dnow_pfnacc_Pq_Qq);
+        case 0x8e: return FNIEMOP_CALL(iemOp_3Dnow_pfpnacc_Pq_Qq);
+        case 0x90: return FNIEMOP_CALL(iemOp_3Dnow_pfcmpge_Pq_Qq);
+        case 0x94: return FNIEMOP_CALL(iemOp_3Dnow_pfmin_Pq_Qq);
+        case 0x96: return FNIEMOP_CALL(iemOp_3Dnow_pfrcp_Pq_Qq);
+        case 0x97: return FNIEMOP_CALL(iemOp_3Dnow_pfrsqrt_Pq_Qq);
+        case 0x9a: return FNIEMOP_CALL(iemOp_3Dnow_pfsub_Pq_Qq);
+        case 0x9e: return FNIEMOP_CALL(iemOp_3Dnow_pfadd_PQ_Qq);
+        case 0xa0: return FNIEMOP_CALL(iemOp_3Dnow_pfcmpgt_Pq_Qq);
+        case 0xa4: return FNIEMOP_CALL(iemOp_3Dnow_pfmax_Pq_Qq);
+        case 0xa6: return FNIEMOP_CALL(iemOp_3Dnow_pfrcpit1_Pq_Qq);
+        case 0xa7: return FNIEMOP_CALL(iemOp_3Dnow_pfrsqit1_Pq_Qq);
+        case 0xaa: return FNIEMOP_CALL(iemOp_3Dnow_pfsubr_Pq_Qq);
+        case 0xae: return FNIEMOP_CALL(iemOp_3Dnow_pfacc_PQ_Qq);
+        case 0xb0: return FNIEMOP_CALL(iemOp_3Dnow_pfcmpeq_Pq_Qq);
+        case 0xb4: return FNIEMOP_CALL(iemOp_3Dnow_pfmul_Pq_Qq);
+        case 0xb6: return FNIEMOP_CALL(iemOp_3Dnow_pfrcpit2_Pq_Qq);
+        case 0xb7: return FNIEMOP_CALL(iemOp_3Dnow_pmulhrw_Pq_Qq);
+        case 0xbb: return FNIEMOP_CALL(iemOp_3Dnow_pswapd_Pq_Qq);
+        case 0xbf: return FNIEMOP_CALL(iemOp_3Dnow_pavgusb_PQ_Qq);
+        default:
+            return IEMOP_RAISE_INVALID_OPCODE();
+    }
+}
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsInterpretOnly.cpp b/src/VBox/VMM/VMMAll/IEMAllInstructionsInterpretOnly.cpp
new file mode 100644
index 00000000..06a2ded9
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsInterpretOnly.cpp
@@ -0,0 +1,1678 @@
+/* $Id: IEMAllInstructionsInterpretOnly.cpp $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#ifndef LOG_GROUP /* defined when included by tstIEMCheckMc.cpp */
+# define LOG_GROUP LOG_GROUP_IEM
+#endif
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/apic.h>
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/gim.h>
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+# include <VBox/vmm/em.h>
+# include <VBox/vmm/hm_svm.h>
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+# include <VBox/vmm/hmvmxinline.h>
+#endif
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/dbgftrace.h>
+#ifndef TST_IEM_CHECK_MC
+# include "IEMInternal.h"
+#endif
+#include <VBox/vmm/vmcc.h>
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <iprt/asm-math.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/x86.h>
+
+#ifndef TST_IEM_CHECK_MC
+# include "IEMInline.h"
+# include "IEMOpHlp.h"
+# include "IEMMc.h"
+#endif
+
+
+#ifdef _MSC_VER
+# pragma warning(push)
+# pragma warning(disable: 4702) /* Unreachable code like return in iemOp_Grp6_lldt. */
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifndef TST_IEM_CHECK_MC
+/** Function table for the ADD instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_add =
+{
+    iemAImpl_add_u8,  iemAImpl_add_u8_locked,
+    iemAImpl_add_u16, iemAImpl_add_u16_locked,
+    iemAImpl_add_u32, iemAImpl_add_u32_locked,
+    iemAImpl_add_u64, iemAImpl_add_u64_locked
+};
+
+/** Function table for the ADC instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_adc =
+{
+    iemAImpl_adc_u8,  iemAImpl_adc_u8_locked,
+    iemAImpl_adc_u16, iemAImpl_adc_u16_locked,
+    iemAImpl_adc_u32, iemAImpl_adc_u32_locked,
+    iemAImpl_adc_u64, iemAImpl_adc_u64_locked
+};
+
+/** Function table for the SUB instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sub =
+{
+    iemAImpl_sub_u8,  iemAImpl_sub_u8_locked,
+    iemAImpl_sub_u16, iemAImpl_sub_u16_locked,
+    iemAImpl_sub_u32, iemAImpl_sub_u32_locked,
+    iemAImpl_sub_u64, iemAImpl_sub_u64_locked
+};
+
+/** Function table for the SBB instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_sbb =
+{
+    iemAImpl_sbb_u8,  iemAImpl_sbb_u8_locked,
+    iemAImpl_sbb_u16, iemAImpl_sbb_u16_locked,
+    iemAImpl_sbb_u32, iemAImpl_sbb_u32_locked,
+    iemAImpl_sbb_u64, iemAImpl_sbb_u64_locked
+};
+
+/** Function table for the OR instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_or =
+{
+    iemAImpl_or_u8,  iemAImpl_or_u8_locked,
+    iemAImpl_or_u16, iemAImpl_or_u16_locked,
+    iemAImpl_or_u32, iemAImpl_or_u32_locked,
+    iemAImpl_or_u64, iemAImpl_or_u64_locked
+};
+
+/** Function table for the XOR instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_xor =
+{
+    iemAImpl_xor_u8,  iemAImpl_xor_u8_locked,
+    iemAImpl_xor_u16, iemAImpl_xor_u16_locked,
+    iemAImpl_xor_u32, iemAImpl_xor_u32_locked,
+    iemAImpl_xor_u64, iemAImpl_xor_u64_locked
+};
+
+/** Function table for the AND instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_and =
+{
+    iemAImpl_and_u8,  iemAImpl_and_u8_locked,
+    iemAImpl_and_u16, iemAImpl_and_u16_locked,
+    iemAImpl_and_u32, iemAImpl_and_u32_locked,
+    iemAImpl_and_u64, iemAImpl_and_u64_locked
+};
+
+/** Function table for the CMP instruction.
+ * @remarks Making operand order ASSUMPTIONS.
+ */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_cmp =
+{
+    iemAImpl_cmp_u8,  NULL,
+    iemAImpl_cmp_u16, NULL,
+    iemAImpl_cmp_u32, NULL,
+    iemAImpl_cmp_u64, NULL
+};
+
+/** Function table for the TEST instruction.
+ * @remarks Making operand order ASSUMPTIONS.
+ */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_test =
+{
+    iemAImpl_test_u8,  NULL,
+    iemAImpl_test_u16, NULL,
+    iemAImpl_test_u32, NULL,
+    iemAImpl_test_u64, NULL
+};
+
+
+/** Function table for the BT instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bt =
+{
+    NULL,  NULL,
+    iemAImpl_bt_u16, NULL,
+    iemAImpl_bt_u32, NULL,
+    iemAImpl_bt_u64, NULL
+};
+
+/** Function table for the BTC instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btc =
+{
+    NULL,  NULL,
+    iemAImpl_btc_u16, iemAImpl_btc_u16_locked,
+    iemAImpl_btc_u32, iemAImpl_btc_u32_locked,
+    iemAImpl_btc_u64, iemAImpl_btc_u64_locked
+};
+
+/** Function table for the BTR instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_btr =
+{
+    NULL,  NULL,
+    iemAImpl_btr_u16, iemAImpl_btr_u16_locked,
+    iemAImpl_btr_u32, iemAImpl_btr_u32_locked,
+    iemAImpl_btr_u64, iemAImpl_btr_u64_locked
+};
+
+/** Function table for the BTS instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bts =
+{
+    NULL,  NULL,
+    iemAImpl_bts_u16, iemAImpl_bts_u16_locked,
+    iemAImpl_bts_u32, iemAImpl_bts_u32_locked,
+    iemAImpl_bts_u64, iemAImpl_bts_u64_locked
+};
+
+/** Function table for the BSF instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsf =
+{
+    NULL,  NULL,
+    iemAImpl_bsf_u16, NULL,
+    iemAImpl_bsf_u32, NULL,
+    iemAImpl_bsf_u64, NULL
+};
+
+/** Function table for the BSF instruction, AMD EFLAGS variant. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsf_amd =
+{
+    NULL,  NULL,
+    iemAImpl_bsf_u16_amd, NULL,
+    iemAImpl_bsf_u32_amd, NULL,
+    iemAImpl_bsf_u64_amd, NULL
+};
+
+/** Function table for the BSF instruction, Intel EFLAGS variant. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsf_intel =
+{
+    NULL,  NULL,
+    iemAImpl_bsf_u16_intel, NULL,
+    iemAImpl_bsf_u32_intel, NULL,
+    iemAImpl_bsf_u64_intel, NULL
+};
+
+/** EFLAGS variation selection table for the BSF instruction. */
+IEM_STATIC const IEMOPBINSIZES * const g_iemAImpl_bsf_eflags[] =
+{
+    &g_iemAImpl_bsf,
+    &g_iemAImpl_bsf_intel,
+    &g_iemAImpl_bsf_amd,
+    &g_iemAImpl_bsf,
+};
+
+/** Function table for the BSR instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsr =
+{
+    NULL,  NULL,
+    iemAImpl_bsr_u16, NULL,
+    iemAImpl_bsr_u32, NULL,
+    iemAImpl_bsr_u64, NULL
+};
+
+/** Function table for the BSR instruction, AMD EFLAGS variant. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsr_amd =
+{
+    NULL,  NULL,
+    iemAImpl_bsr_u16_amd, NULL,
+    iemAImpl_bsr_u32_amd, NULL,
+    iemAImpl_bsr_u64_amd, NULL
+};
+
+/** Function table for the BSR instruction, Intel EFLAGS variant. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_bsr_intel =
+{
+    NULL,  NULL,
+    iemAImpl_bsr_u16_intel, NULL,
+    iemAImpl_bsr_u32_intel, NULL,
+    iemAImpl_bsr_u64_intel, NULL
+};
+
+/** EFLAGS variation selection table for the BSR instruction. */
+IEM_STATIC const IEMOPBINSIZES * const g_iemAImpl_bsr_eflags[] =
+{
+    &g_iemAImpl_bsr,
+    &g_iemAImpl_bsr_intel,
+    &g_iemAImpl_bsr_amd,
+    &g_iemAImpl_bsr,
+};
+
+/** Function table for the IMUL instruction. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_imul_two =
+{
+    NULL,  NULL,
+    iemAImpl_imul_two_u16, NULL,
+    iemAImpl_imul_two_u32, NULL,
+    iemAImpl_imul_two_u64, NULL
+};
+
+/** Function table for the IMUL instruction, AMD EFLAGS variant. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_imul_two_amd =
+{
+    NULL,  NULL,
+    iemAImpl_imul_two_u16_amd, NULL,
+    iemAImpl_imul_two_u32_amd, NULL,
+    iemAImpl_imul_two_u64_amd, NULL
+};
+
+/** Function table for the IMUL instruction, Intel EFLAGS variant. */
+IEM_STATIC const IEMOPBINSIZES g_iemAImpl_imul_two_intel =
+{
+    NULL,  NULL,
+    iemAImpl_imul_two_u16_intel, NULL,
+    iemAImpl_imul_two_u32_intel, NULL,
+    iemAImpl_imul_two_u64_intel, NULL
+};
+
+/** EFLAGS variation selection table for the IMUL instruction. */
+IEM_STATIC const IEMOPBINSIZES * const g_iemAImpl_imul_two_eflags[] =
+{
+    &g_iemAImpl_imul_two,
+    &g_iemAImpl_imul_two_intel,
+    &g_iemAImpl_imul_two_amd,
+    &g_iemAImpl_imul_two,
+};
+
+/** EFLAGS variation selection table for the 16-bit IMUL instruction. */
+IEM_STATIC PFNIEMAIMPLBINU16 const g_iemAImpl_imul_two_u16_eflags[] =
+{
+    iemAImpl_imul_two_u16,
+    iemAImpl_imul_two_u16_intel,
+    iemAImpl_imul_two_u16_amd,
+    iemAImpl_imul_two_u16,
+};
+
+/** EFLAGS variation selection table for the 32-bit IMUL instruction. */
+IEM_STATIC PFNIEMAIMPLBINU32 const g_iemAImpl_imul_two_u32_eflags[] =
+{
+    iemAImpl_imul_two_u32,
+    iemAImpl_imul_two_u32_intel,
+    iemAImpl_imul_two_u32_amd,
+    iemAImpl_imul_two_u32,
+};
+
+/** EFLAGS variation selection table for the 64-bit IMUL instruction. */
+IEM_STATIC PFNIEMAIMPLBINU64 const g_iemAImpl_imul_two_u64_eflags[] =
+{
+    iemAImpl_imul_two_u64,
+    iemAImpl_imul_two_u64_intel,
+    iemAImpl_imul_two_u64_amd,
+    iemAImpl_imul_two_u64,
+};
+
+/** Group 1 /r lookup table. */
+IEM_STATIC const PCIEMOPBINSIZES g_apIemImplGrp1[8] =
+{
+    &g_iemAImpl_add,
+    &g_iemAImpl_or,
+    &g_iemAImpl_adc,
+    &g_iemAImpl_sbb,
+    &g_iemAImpl_and,
+    &g_iemAImpl_sub,
+    &g_iemAImpl_xor,
+    &g_iemAImpl_cmp
+};
+
+/** Function table for the INC instruction. */
+IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_inc =
+{
+    iemAImpl_inc_u8,  iemAImpl_inc_u8_locked,
+    iemAImpl_inc_u16, iemAImpl_inc_u16_locked,
+    iemAImpl_inc_u32, iemAImpl_inc_u32_locked,
+    iemAImpl_inc_u64, iemAImpl_inc_u64_locked
+};
+
+/** Function table for the DEC instruction. */
+IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_dec =
+{
+    iemAImpl_dec_u8,  iemAImpl_dec_u8_locked,
+    iemAImpl_dec_u16, iemAImpl_dec_u16_locked,
+    iemAImpl_dec_u32, iemAImpl_dec_u32_locked,
+    iemAImpl_dec_u64, iemAImpl_dec_u64_locked
+};
+
+/** Function table for the NEG instruction. */
+IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_neg =
+{
+    iemAImpl_neg_u8,  iemAImpl_neg_u8_locked,
+    iemAImpl_neg_u16, iemAImpl_neg_u16_locked,
+    iemAImpl_neg_u32, iemAImpl_neg_u32_locked,
+    iemAImpl_neg_u64, iemAImpl_neg_u64_locked
+};
+
+/** Function table for the NOT instruction. */
+IEM_STATIC const IEMOPUNARYSIZES g_iemAImpl_not =
+{
+    iemAImpl_not_u8,  iemAImpl_not_u8_locked,
+    iemAImpl_not_u16, iemAImpl_not_u16_locked,
+    iemAImpl_not_u32, iemAImpl_not_u32_locked,
+    iemAImpl_not_u64, iemAImpl_not_u64_locked
+};
+
+
+/** Function table for the ROL instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rol =
+{
+    iemAImpl_rol_u8,
+    iemAImpl_rol_u16,
+    iemAImpl_rol_u32,
+    iemAImpl_rol_u64
+};
+
+/** Function table for the ROL instruction, AMD EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rol_amd =
+{
+    iemAImpl_rol_u8_amd,
+    iemAImpl_rol_u16_amd,
+    iemAImpl_rol_u32_amd,
+    iemAImpl_rol_u64_amd
+};
+
+/** Function table for the ROL instruction, Intel EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rol_intel =
+{
+    iemAImpl_rol_u8_intel,
+    iemAImpl_rol_u16_intel,
+    iemAImpl_rol_u32_intel,
+    iemAImpl_rol_u64_intel
+};
+
+/** EFLAGS variation selection table for the ROL instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_rol_eflags[] =
+{
+    &g_iemAImpl_rol,
+    &g_iemAImpl_rol_intel,
+    &g_iemAImpl_rol_amd,
+    &g_iemAImpl_rol,
+};
+
+
+/** Function table for the ROR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_ror =
+{
+    iemAImpl_ror_u8,
+    iemAImpl_ror_u16,
+    iemAImpl_ror_u32,
+    iemAImpl_ror_u64
+};
+
+/** Function table for the ROR instruction, AMD EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_ror_amd =
+{
+    iemAImpl_ror_u8_amd,
+    iemAImpl_ror_u16_amd,
+    iemAImpl_ror_u32_amd,
+    iemAImpl_ror_u64_amd
+};
+
+/** Function table for the ROR instruction, Intel EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_ror_intel =
+{
+    iemAImpl_ror_u8_intel,
+    iemAImpl_ror_u16_intel,
+    iemAImpl_ror_u32_intel,
+    iemAImpl_ror_u64_intel
+};
+
+/** EFLAGS variation selection table for the ROR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_ror_eflags[] =
+{
+    &g_iemAImpl_ror,
+    &g_iemAImpl_ror_intel,
+    &g_iemAImpl_ror_amd,
+    &g_iemAImpl_ror,
+};
+
+
+/** Function table for the RCL instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcl =
+{
+    iemAImpl_rcl_u8,
+    iemAImpl_rcl_u16,
+    iemAImpl_rcl_u32,
+    iemAImpl_rcl_u64
+};
+
+/** Function table for the RCL instruction, AMD EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcl_amd =
+{
+    iemAImpl_rcl_u8_amd,
+    iemAImpl_rcl_u16_amd,
+    iemAImpl_rcl_u32_amd,
+    iemAImpl_rcl_u64_amd
+};
+
+/** Function table for the RCL instruction, Intel EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcl_intel =
+{
+    iemAImpl_rcl_u8_intel,
+    iemAImpl_rcl_u16_intel,
+    iemAImpl_rcl_u32_intel,
+    iemAImpl_rcl_u64_intel
+};
+
+/** EFLAGS variation selection table for the RCL instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_rcl_eflags[] =
+{
+    &g_iemAImpl_rcl,
+    &g_iemAImpl_rcl_intel,
+    &g_iemAImpl_rcl_amd,
+    &g_iemAImpl_rcl,
+};
+
+
+/** Function table for the RCR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcr =
+{
+    iemAImpl_rcr_u8,
+    iemAImpl_rcr_u16,
+    iemAImpl_rcr_u32,
+    iemAImpl_rcr_u64
+};
+
+/** Function table for the RCR instruction, AMD EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcr_amd =
+{
+    iemAImpl_rcr_u8_amd,
+    iemAImpl_rcr_u16_amd,
+    iemAImpl_rcr_u32_amd,
+    iemAImpl_rcr_u64_amd
+};
+
+/** Function table for the RCR instruction, Intel EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_rcr_intel =
+{
+    iemAImpl_rcr_u8_intel,
+    iemAImpl_rcr_u16_intel,
+    iemAImpl_rcr_u32_intel,
+    iemAImpl_rcr_u64_intel
+};
+
+/** EFLAGS variation selection table for the RCR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_rcr_eflags[] =
+{
+    &g_iemAImpl_rcr,
+    &g_iemAImpl_rcr_intel,
+    &g_iemAImpl_rcr_amd,
+    &g_iemAImpl_rcr,
+};
+
+
+/** Function table for the SHL instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shl =
+{
+    iemAImpl_shl_u8,
+    iemAImpl_shl_u16,
+    iemAImpl_shl_u32,
+    iemAImpl_shl_u64
+};
+
+/** Function table for the SHL instruction, AMD EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shl_amd =
+{
+    iemAImpl_shl_u8_amd,
+    iemAImpl_shl_u16_amd,
+    iemAImpl_shl_u32_amd,
+    iemAImpl_shl_u64_amd
+};
+
+/** Function table for the SHL instruction, Intel EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shl_intel =
+{
+    iemAImpl_shl_u8_intel,
+    iemAImpl_shl_u16_intel,
+    iemAImpl_shl_u32_intel,
+    iemAImpl_shl_u64_intel
+};
+
+/** EFLAGS variation selection table for the SHL instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_shl_eflags[] =
+{
+    &g_iemAImpl_shl,
+    &g_iemAImpl_shl_intel,
+    &g_iemAImpl_shl_amd,
+    &g_iemAImpl_shl,
+};
+
+
+/** Function table for the SHR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shr =
+{
+    iemAImpl_shr_u8,
+    iemAImpl_shr_u16,
+    iemAImpl_shr_u32,
+    iemAImpl_shr_u64
+};
+
+/** Function table for the SHR instruction, AMD EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shr_amd =
+{
+    iemAImpl_shr_u8_amd,
+    iemAImpl_shr_u16_amd,
+    iemAImpl_shr_u32_amd,
+    iemAImpl_shr_u64_amd
+};
+
+/** Function table for the SHR instruction, Intel EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_shr_intel =
+{
+    iemAImpl_shr_u8_intel,
+    iemAImpl_shr_u16_intel,
+    iemAImpl_shr_u32_intel,
+    iemAImpl_shr_u64_intel
+};
+
+/** EFLAGS variation selection table for the SHR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_shr_eflags[] =
+{
+    &g_iemAImpl_shr,
+    &g_iemAImpl_shr_intel,
+    &g_iemAImpl_shr_amd,
+    &g_iemAImpl_shr,
+};
+
+
+/** Function table for the SAR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_sar =
+{
+    iemAImpl_sar_u8,
+    iemAImpl_sar_u16,
+    iemAImpl_sar_u32,
+    iemAImpl_sar_u64
+};
+
+/** Function table for the SAR instruction, AMD EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_sar_amd =
+{
+    iemAImpl_sar_u8_amd,
+    iemAImpl_sar_u16_amd,
+    iemAImpl_sar_u32_amd,
+    iemAImpl_sar_u64_amd
+};
+
+/** Function table for the SAR instruction, Intel EFLAGS variant. */
+IEM_STATIC const IEMOPSHIFTSIZES g_iemAImpl_sar_intel =
+{
+    iemAImpl_sar_u8_intel,
+    iemAImpl_sar_u16_intel,
+    iemAImpl_sar_u32_intel,
+    iemAImpl_sar_u64_intel
+};
+
+/** EFLAGS variation selection table for the SAR instruction. */
+IEM_STATIC const IEMOPSHIFTSIZES * const g_iemAImpl_sar_eflags[] =
+{
+    &g_iemAImpl_sar,
+    &g_iemAImpl_sar_intel,
+    &g_iemAImpl_sar_amd,
+    &g_iemAImpl_sar,
+};
+
+
+/** Function table for the MUL instruction. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_mul =
+{
+    iemAImpl_mul_u8,
+    iemAImpl_mul_u16,
+    iemAImpl_mul_u32,
+    iemAImpl_mul_u64
+};
+
+/** Function table for the MUL instruction, AMD EFLAGS variation. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_mul_amd =
+{
+    iemAImpl_mul_u8_amd,
+    iemAImpl_mul_u16_amd,
+    iemAImpl_mul_u32_amd,
+    iemAImpl_mul_u64_amd
+};
+
+/** Function table for the MUL instruction, Intel EFLAGS variation. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_mul_intel =
+{
+    iemAImpl_mul_u8_intel,
+    iemAImpl_mul_u16_intel,
+    iemAImpl_mul_u32_intel,
+    iemAImpl_mul_u64_intel
+};
+
+/** EFLAGS variation selection table for the MUL instruction. */
+IEM_STATIC const IEMOPMULDIVSIZES * const g_iemAImpl_mul_eflags[] =
+{
+    &g_iemAImpl_mul,
+    &g_iemAImpl_mul_intel,
+    &g_iemAImpl_mul_amd,
+    &g_iemAImpl_mul,
+};
+
+/** EFLAGS variation selection table for the 8-bit MUL instruction. */
+IEM_STATIC PFNIEMAIMPLMULDIVU8 const g_iemAImpl_mul_u8_eflags[] =
+{
+    iemAImpl_mul_u8,
+    iemAImpl_mul_u8_intel,
+    iemAImpl_mul_u8_amd,
+    iemAImpl_mul_u8
+};
+
+
+/** Function table for the IMUL instruction working implicitly on rAX. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_imul =
+{
+    iemAImpl_imul_u8,
+    iemAImpl_imul_u16,
+    iemAImpl_imul_u32,
+    iemAImpl_imul_u64
+};
+
+/** Function table for the IMUL instruction working implicitly on rAX, AMD EFLAGS variation. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_imul_amd =
+{
+    iemAImpl_imul_u8_amd,
+    iemAImpl_imul_u16_amd,
+    iemAImpl_imul_u32_amd,
+    iemAImpl_imul_u64_amd
+};
+
+/** Function table for the IMUL instruction working implicitly on rAX, Intel EFLAGS variation. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_imul_intel =
+{
+    iemAImpl_imul_u8_intel,
+    iemAImpl_imul_u16_intel,
+    iemAImpl_imul_u32_intel,
+    iemAImpl_imul_u64_intel
+};
+
+/** EFLAGS variation selection table for the IMUL instruction. */
+IEM_STATIC const IEMOPMULDIVSIZES * const g_iemAImpl_imul_eflags[] =
+{
+    &g_iemAImpl_imul,
+    &g_iemAImpl_imul_intel,
+    &g_iemAImpl_imul_amd,
+    &g_iemAImpl_imul,
+};
+
+/** EFLAGS variation selection table for the 8-bit IMUL instruction. */
+IEM_STATIC PFNIEMAIMPLMULDIVU8 const g_iemAImpl_imul_u8_eflags[] =
+{
+    iemAImpl_imul_u8,
+    iemAImpl_imul_u8_intel,
+    iemAImpl_imul_u8_amd,
+    iemAImpl_imul_u8
+};
+
+
+/** Function table for the DIV instruction. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_div =
+{
+    iemAImpl_div_u8,
+    iemAImpl_div_u16,
+    iemAImpl_div_u32,
+    iemAImpl_div_u64
+};
+
+/** Function table for the DIV instruction, AMD EFLAGS variation. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_div_amd =
+{
+    iemAImpl_div_u8_amd,
+    iemAImpl_div_u16_amd,
+    iemAImpl_div_u32_amd,
+    iemAImpl_div_u64_amd
+};
+
+/** Function table for the DIV instruction, Intel EFLAGS variation. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_div_intel =
+{
+    iemAImpl_div_u8_intel,
+    iemAImpl_div_u16_intel,
+    iemAImpl_div_u32_intel,
+    iemAImpl_div_u64_intel
+};
+
+/** EFLAGS variation selection table for the DIV instruction. */
+IEM_STATIC const IEMOPMULDIVSIZES * const g_iemAImpl_div_eflags[] =
+{
+    &g_iemAImpl_div,
+    &g_iemAImpl_div_intel,
+    &g_iemAImpl_div_amd,
+    &g_iemAImpl_div,
+};
+
+/** EFLAGS variation selection table for the 8-bit DIV instruction. */
+IEM_STATIC PFNIEMAIMPLMULDIVU8 const g_iemAImpl_div_u8_eflags[] =
+{
+    iemAImpl_div_u8,
+    iemAImpl_div_u8_intel,
+    iemAImpl_div_u8_amd,
+    iemAImpl_div_u8
+};
+
+
+/** Function table for the IDIV instruction. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_idiv =
+{
+    iemAImpl_idiv_u8,
+    iemAImpl_idiv_u16,
+    iemAImpl_idiv_u32,
+    iemAImpl_idiv_u64
+};
+
+/** Function table for the IDIV instruction, AMD EFLAGS variation. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_idiv_amd =
+{
+    iemAImpl_idiv_u8_amd,
+    iemAImpl_idiv_u16_amd,
+    iemAImpl_idiv_u32_amd,
+    iemAImpl_idiv_u64_amd
+};
+
+/** Function table for the IDIV instruction, Intel EFLAGS variation. */
+IEM_STATIC const IEMOPMULDIVSIZES g_iemAImpl_idiv_intel =
+{
+    iemAImpl_idiv_u8_intel,
+    iemAImpl_idiv_u16_intel,
+    iemAImpl_idiv_u32_intel,
+    iemAImpl_idiv_u64_intel
+};
+
+/** EFLAGS variation selection table for the IDIV instruction. */
+IEM_STATIC const IEMOPMULDIVSIZES * const g_iemAImpl_idiv_eflags[] =
+{
+    &g_iemAImpl_idiv,
+    &g_iemAImpl_idiv_intel,
+    &g_iemAImpl_idiv_amd,
+    &g_iemAImpl_idiv,
+};
+
+/** EFLAGS variation selection table for the 8-bit IDIV instruction. */
+IEM_STATIC PFNIEMAIMPLMULDIVU8 const g_iemAImpl_idiv_u8_eflags[] =
+{
+    iemAImpl_idiv_u8,
+    iemAImpl_idiv_u8_intel,
+    iemAImpl_idiv_u8_amd,
+    iemAImpl_idiv_u8
+};
+
+
+/** Function table for the SHLD instruction. */
+IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shld =
+{
+    iemAImpl_shld_u16,
+    iemAImpl_shld_u32,
+    iemAImpl_shld_u64,
+};
+
+/** Function table for the SHLD instruction, AMD EFLAGS variation. */
+IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shld_amd =
+{
+    iemAImpl_shld_u16_amd,
+    iemAImpl_shld_u32_amd,
+    iemAImpl_shld_u64_amd
+};
+
+/** Function table for the SHLD instruction, Intel EFLAGS variation. */
+IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shld_intel =
+{
+    iemAImpl_shld_u16_intel,
+    iemAImpl_shld_u32_intel,
+    iemAImpl_shld_u64_intel
+};
+
+/** EFLAGS variation selection table for the SHLD instruction. */
+IEM_STATIC const IEMOPSHIFTDBLSIZES * const g_iemAImpl_shld_eflags[] =
+{
+    &g_iemAImpl_shld,
+    &g_iemAImpl_shld_intel,
+    &g_iemAImpl_shld_amd,
+    &g_iemAImpl_shld
+};
+
+/** Function table for the SHRD instruction. */
+IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd =
+{
+    iemAImpl_shrd_u16,
+    iemAImpl_shrd_u32,
+    iemAImpl_shrd_u64
+};
+
+/** Function table for the SHRD instruction, AMD EFLAGS variation. */
+IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd_amd =
+{
+    iemAImpl_shrd_u16_amd,
+    iemAImpl_shrd_u32_amd,
+    iemAImpl_shrd_u64_amd
+};
+
+/** Function table for the SHRD instruction, Intel EFLAGS variation. */
+IEM_STATIC const IEMOPSHIFTDBLSIZES g_iemAImpl_shrd_intel =
+{
+    iemAImpl_shrd_u16_intel,
+    iemAImpl_shrd_u32_intel,
+    iemAImpl_shrd_u64_intel
+};
+
+/** EFLAGS variation selection table for the SHRD instruction. */
+IEM_STATIC const IEMOPSHIFTDBLSIZES * const g_iemAImpl_shrd_eflags[] =
+{
+    &g_iemAImpl_shrd,
+    &g_iemAImpl_shrd_intel,
+    &g_iemAImpl_shrd_amd,
+    &g_iemAImpl_shrd
+};
+
+
+# ifndef IEM_WITHOUT_ASSEMBLY
+/** Function table for the VPXOR instruction */
+IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpand          = { iemAImpl_vpand_u128,   iemAImpl_vpand_u256 };
+/** Function table for the VPXORN instruction */
+IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpandn         = { iemAImpl_vpandn_u128,  iemAImpl_vpandn_u256 };
+/** Function table for the VPOR instruction */
+IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpor           = { iemAImpl_vpor_u128,    iemAImpl_vpor_u256 };
+/** Function table for the VPXOR instruction */
+IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpxor          = { iemAImpl_vpxor_u128,   iemAImpl_vpxor_u256 };
+# endif
+
+/** Function table for the VPAND instruction, software fallback. */
+IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpand_fallback = { iemAImpl_vpand_u128_fallback,  iemAImpl_vpand_u256_fallback };
+/** Function table for the VPANDN instruction, software fallback. */
+IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpandn_fallback= { iemAImpl_vpandn_u128_fallback, iemAImpl_vpandn_u256_fallback };
+/** Function table for the VPOR instruction, software fallback. */
+IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpor_fallback  = { iemAImpl_vpor_u128_fallback,   iemAImpl_vpor_u256_fallback };
+/** Function table for the VPXOR instruction, software fallback. */
+IEM_STATIC const IEMOPMEDIAF3 g_iemAImpl_vpxor_fallback = { iemAImpl_vpxor_u128_fallback,  iemAImpl_vpxor_u256_fallback };
+
+#endif /* !TST_IEM_CHECK_MC */
+
+
+/**
+ * Common worker for instructions like ADD, AND, OR, ++ with a byte
+ * memory/register as the destination.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rm_r8, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,  pu8Dst,  0);
+        IEM_MC_ARG(uint8_t,    u8Src,   1);
+        IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+        IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         * Note! We're putting the eflags on the stack here so we can commit them
+         *       after the memory.
+         */
+        uint32_t const fAccess = pImpl->pfnLockedU8 ? IEM_ACCESS_DATA_RW : IEM_ACCESS_DATA_R; /* CMP,TEST */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,  pu8Dst,           0);
+        IEM_MC_ARG(uint8_t,    u8Src,            1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        if (!pImpl->pfnLockedU8)
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MEM_MAP(pu8Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, fAccess);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for word/dword/qword instructions like ADD, AND, OR, ++ with
+ * memory/register as the destination.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rm_rv, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *, pu16Dst, 0);
+                IEM_MC_ARG(uint16_t,   u16Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *, pu32Dst, 0);
+                IEM_MC_ARG(uint32_t,   u32Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                if ((pImpl != &g_iemAImpl_test) && (pImpl != &g_iemAImpl_cmp))
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *, pu64Dst, 0);
+                IEM_MC_ARG(uint64_t,   u64Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         * Note! We're putting the eflags on the stack here so we can commit them
+         *       after the memory.
+         */
+        uint32_t const fAccess = pImpl->pfnLockedU8 ? IEM_ACCESS_DATA_RW : IEM_ACCESS_DATA_R /* CMP,TEST */;
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *, pu16Dst,          0);
+                IEM_MC_ARG(uint16_t,   u16Src,           1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (!pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *, pu32Dst,          0);
+                IEM_MC_ARG(uint32_t,   u32Src,           1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (!pImpl->pfnLockedU32)
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *, pu64Dst,          0);
+                IEM_MC_ARG(uint64_t,   u64Src,           1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (!pImpl->pfnLockedU64)
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * Common worker for byte instructions like ADD, AND, OR, ++ with a register as
+ * the destination.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_r8_rm, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,  pu8Dst,  0);
+        IEM_MC_ARG(uint8_t,    u8Src,   1);
+        IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+        IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint8_t *,  pu8Dst,  0);
+        IEM_MC_ARG(uint8_t,    u8Src,   1);
+        IEM_MC_ARG(uint32_t *, pEFlags, 2);
+        IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U8(u8Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for word/dword/qword instructions like ADD, AND, OR, ++ with a
+ * register as the destination.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rv_rm, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *, pu16Dst, 0);
+                IEM_MC_ARG(uint16_t,   u16Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *, pu32Dst, 0);
+                IEM_MC_ARG(uint32_t,   u32Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                if (pImpl != &g_iemAImpl_cmp)   /* Not used with TEST. */
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *, pu64Dst, 0);
+                IEM_MC_ARG(uint64_t,   u64Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *, pu16Dst, 0);
+                IEM_MC_ARG(uint16_t,   u16Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t *, pu32Dst, 0);
+                IEM_MC_ARG(uint32_t,   u32Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                if (pImpl != &g_iemAImpl_cmp)
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *, pu64Dst, 0);
+                IEM_MC_ARG(uint64_t,   u64Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * Common worker for instructions like ADD, AND, OR, ++ with working on AL with
+ * a byte immediate.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_AL_Ib, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG(uint8_t *,       pu8Dst,             0);
+    IEM_MC_ARG_CONST(uint8_t,   u8Src,/*=*/ u8Imm,  1);
+    IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+
+    IEM_MC_REF_GREG_U8(pu8Dst, X86_GREG_xAX);
+    IEM_MC_REF_EFLAGS(pEFlags);
+    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * Common worker for instructions like ADD, AND, OR, ++ with working on
+ * AX/EAX/RAX with a word/dword immediate.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBinaryOperator_rAX_Iz, PCIEMOPBINSIZES, pImpl)
+{
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+            IEM_MC_ARG_CONST(uint16_t,  u16Src,/*=*/ u16Imm,    1);
+            IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+            IEM_MC_REF_GREG_U16(pu16Dst, X86_GREG_xAX);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+            IEM_MC_ARG_CONST(uint32_t,  u32Src,/*=*/ u32Imm,    1);
+            IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+            IEM_MC_REF_GREG_U32(pu32Dst, X86_GREG_xAX);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+            if ((pImpl != &g_iemAImpl_test) && (pImpl != &g_iemAImpl_cmp))
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+            IEM_MC_ARG_CONST(uint64_t,  u64Src,/*=*/ u64Imm,    1);
+            IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+            IEM_MC_REF_GREG_U64(pu64Dst, X86_GREG_xAX);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcodes 0xf1, 0xd6. */
+FNIEMOP_DEF(iemOp_Invalid)
+{
+    IEMOP_MNEMONIC(Invalid, "Invalid");
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid with RM byte . */
+FNIEMOPRM_DEF(iemOp_InvalidWithRM)
+{
+    RT_NOREF_PV(bRm);
+    IEMOP_MNEMONIC(InvalidWithRm, "InvalidWithRM");
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid with RM byte where intel decodes any additional address encoding
+ *  bytes. */
+FNIEMOPRM_DEF(iemOp_InvalidWithRMNeedDecode)
+{
+    IEMOP_MNEMONIC(InvalidWithRMNeedDecode, "InvalidWithRMNeedDecode");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+#ifndef TST_IEM_CHECK_MC
+        if (IEM_IS_MODRM_MEM_MODE(bRm))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+    }
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid with RM byte where both AMD and Intel decodes any additional
+ *  address encoding bytes. */
+FNIEMOPRM_DEF(iemOp_InvalidWithRMAllNeeded)
+{
+    IEMOP_MNEMONIC(InvalidWithRMAllNeeded, "InvalidWithRMAllNeeded");
+#ifndef TST_IEM_CHECK_MC
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        RTGCPTR      GCPtrEff;
+        VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+#endif
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid with RM byte where intel requires 8-byte immediate.
+ * Intel will also need SIB and displacement if bRm indicates memory. */
+FNIEMOPRM_DEF(iemOp_InvalidWithRMNeedImm8)
+{
+    IEMOP_MNEMONIC(InvalidWithRMNeedImm8, "InvalidWithRMNeedImm8");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+#ifndef TST_IEM_CHECK_MC
+        if (IEM_IS_MODRM_MEM_MODE(bRm))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+        uint8_t bImm8;  IEM_OPCODE_GET_NEXT_U8(&bImm8);  RT_NOREF(bRm);
+    }
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid with RM byte where intel requires 8-byte immediate.
+ * Both AMD and Intel also needs SIB and displacement according to bRm. */
+FNIEMOPRM_DEF(iemOp_InvalidWithRMAllNeedImm8)
+{
+    IEMOP_MNEMONIC(InvalidWithRMAllNeedImm8, "InvalidWithRMAllNeedImm8");
+#ifndef TST_IEM_CHECK_MC
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        RTGCPTR      GCPtrEff;
+        VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+#endif
+    uint8_t bImm8;  IEM_OPCODE_GET_NEXT_U8(&bImm8);  RT_NOREF(bRm);
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid opcode where intel requires Mod R/M sequence. */
+FNIEMOP_DEF(iemOp_InvalidNeedRM)
+{
+    IEMOP_MNEMONIC(InvalidNeedRM, "InvalidNeedRM");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        if (IEM_IS_MODRM_MEM_MODE(bRm))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+    }
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid opcode where both AMD and Intel requires Mod R/M sequence. */
+FNIEMOP_DEF(iemOp_InvalidAllNeedRM)
+{
+    IEMOP_MNEMONIC(InvalidAllNeedRM, "InvalidAllNeedRM");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        RTGCPTR      GCPtrEff;
+        VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+#endif
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid opcode where intel requires Mod R/M sequence and 8-byte
+ *  immediate. */
+FNIEMOP_DEF(iemOp_InvalidNeedRMImm8)
+{
+    IEMOP_MNEMONIC(InvalidNeedRMImm8, "InvalidNeedRMImm8");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t bRm;  IEM_OPCODE_GET_NEXT_U8(&bRm);  RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        if (IEM_IS_MODRM_MEM_MODE(bRm))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); RT_NOREF(bImm);
+    }
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid opcode where intel requires a 3rd escape byte and a Mod R/M
+ *  sequence. */
+FNIEMOP_DEF(iemOp_InvalidNeed3ByteEscRM)
+{
+    IEMOP_MNEMONIC(InvalidNeed3ByteEscRM, "InvalidNeed3ByteEscRM");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t b3rd; IEM_OPCODE_GET_NEXT_U8(&b3rd); RT_NOREF(b3rd);
+        uint8_t bRm;  IEM_OPCODE_GET_NEXT_U8(&bRm);  RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        if (IEM_IS_MODRM_MEM_MODE(bRm))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+    }
+    IEMOP_HLP_DONE_DECODING();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Invalid opcode where intel requires a 3rd escape byte, Mod R/M sequence, and
+ *  a 8-byte immediate. */
+FNIEMOP_DEF(iemOp_InvalidNeed3ByteEscRMImm8)
+{
+    IEMOP_MNEMONIC(InvalidNeed3ByteEscRMImm8, "InvalidNeed3ByteEscRMImm8");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t b3rd; IEM_OPCODE_GET_NEXT_U8(&b3rd); RT_NOREF(b3rd);
+        uint8_t bRm;  IEM_OPCODE_GET_NEXT_U8(&bRm);  RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        if (IEM_IS_MODRM_MEM_MODE(bRm))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 1, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); RT_NOREF(bImm);
+        IEMOP_HLP_DONE_DECODING();
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Repeats a_fn four times.  For decoding tables. */
+#define IEMOP_X4(a_fn) a_fn, a_fn, a_fn, a_fn
+
+/*
+ * Include the tables.
+ */
+#ifdef IEM_WITH_3DNOW
+# include "IEMAllInstructions3DNow.cpp.h"
+#endif
+#ifdef IEM_WITH_THREE_0F_38
+# include "IEMAllInstructionsThree0f38.cpp.h"
+#endif
+#ifdef IEM_WITH_THREE_0F_3A
+# include "IEMAllInstructionsThree0f3a.cpp.h"
+#endif
+#include "IEMAllInstructionsTwoByte0f.cpp.h"
+#ifdef IEM_WITH_VEX
+# include "IEMAllInstructionsVexMap1.cpp.h"
+# include "IEMAllInstructionsVexMap2.cpp.h"
+# include "IEMAllInstructionsVexMap3.cpp.h"
+#endif
+#include "IEMAllInstructionsOneByte.cpp.h"
+
+
+#ifdef _MSC_VER
+# pragma warning(pop)
+#endif
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h
new file mode 100644
index 00000000..8c181e54
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsOneByte.cpp.h
@@ -0,0 +1,11860 @@
+/* $Id: IEMAllInstructionsOneByte.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+extern const PFNIEMOP g_apfnOneByteMap[256]; /* not static since we need to forward declare it. */
+
+/* Instruction group definitions: */
+
+/** @defgroup og_gen    General
+ * @{ */
+ /** @defgroup og_gen_arith     Arithmetic
+  * @{  */
+  /** @defgroup og_gen_arith_bin    Binary numbers */
+  /** @defgroup og_gen_arith_dec    Decimal numbers */
+ /** @} */
+/** @} */
+
+/** @defgroup og_stack Stack
+ * @{ */
+ /** @defgroup og_stack_sreg    Segment registers */
+/** @} */
+
+/** @defgroup og_prefix     Prefixes */
+/** @defgroup og_escapes    Escape bytes */
+
+
+
+/** @name One byte opcodes.
+ * @{
+ */
+
+/* Instruction specification format - work in progress:  */
+
+/**
+ * @opcode      0x00
+ * @opmnemonic  add
+ * @op1         rm:Eb
+ * @op2         reg:Gb
+ * @opmaps      one
+ * @openc       ModR/M
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @ophints     harmless ignores_op_sizes
+ * @opstats     add_Eb_Gb
+ * @opgroup     og_gen_arith_bin
+ * @optest              op1=1   op2=1   -> op1=2   efl&|=nc,pe,na,nz,pl,nv
+ * @optest      efl|=cf op1=1   op2=2   -> op1=3   efl&|=nc,po,na,nz,pl,nv
+ * @optest              op1=254 op2=1   -> op1=255 efl&|=nc,po,na,nz,ng,nv
+ * @optest              op1=128 op2=128 -> op1=0   efl&|=ov,pl,zf,na,po,cf
+ */
+FNIEMOP_DEF(iemOp_add_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, ADD, add, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x01
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @optest               op1=1  op2=1  -> op1=2  efl&|=nc,pe,na,nz,pl,nv
+ * @optest      efl|=cf  op1=2  op2=2  -> op1=4  efl&|=nc,pe,na,nz,pl,nv
+ * @optest      efl&~=cf op1=-1 op2=1  -> op1=0  efl&|=cf,po,af,zf,pl,nv
+ * @optest               op1=-1 op2=-1 -> op1=-2 efl&|=cf,pe,af,nz,ng,nv
+ */
+FNIEMOP_DEF(iemOp_add_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, ADD, add, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x02
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_add_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_add_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, ADD, add, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x03
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_add_Ev_Gv
+ */
+FNIEMOP_DEF(iemOp_add_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, ADD, add, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x04
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_add_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_add_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, ADD, add, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x05
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @optest      op1=1 op2=1 -> op1=2 efl&|=nv,pl,nz,na,pe
+ * @optest      efl|=cf  op1=2  op2=2  -> op1=4  efl&|=nc,pe,na,nz,pl,nv
+ * @optest      efl&~=cf op1=-1 op2=1  -> op1=0  efl&|=cf,po,af,zf,pl,nv
+ * @optest               op1=-1 op2=-1 -> op1=-2 efl&|=cf,pe,af,nz,ng,nv
+ */
+FNIEMOP_DEF(iemOp_add_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, ADD, add, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_add);
+}
+
+
+/**
+ * @opcode      0x06
+ * @opgroup     og_stack_sreg
+ */
+FNIEMOP_DEF(iemOp_push_ES)
+{
+    IEMOP_MNEMONIC1(FIXED, PUSH, push, ES, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0);
+    IEMOP_HLP_NO_64BIT();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_ES);
+}
+
+
+/**
+ * @opcode      0x07
+ * @opgroup     og_stack_sreg
+ */
+FNIEMOP_DEF(iemOp_pop_ES)
+{
+    IEMOP_MNEMONIC1(FIXED, POP, pop, ES, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0);
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_ES, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0x08
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @optest                  op1=7 op2=12 -> op1=15   efl&|=nc,po,na,nz,pl,nv
+ * @optest      efl|=of,cf  op1=0 op2=0  -> op1=0    efl&|=nc,po,na,zf,pl,nv
+ * @optest            op1=0xee op2=0x11  -> op1=0xff efl&|=nc,po,na,nz,ng,nv
+ * @optest            op1=0xff op2=0xff  -> op1=0xff efl&|=nc,po,na,nz,ng,nv
+ */
+FNIEMOP_DEF(iemOp_or_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, OR, or, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_or);
+}
+
+
+/*
+ * @opcode      0x09
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @optest      efl|=of,cf  op1=12 op2=7 -> op1=15   efl&|=nc,po,na,nz,pl,nv
+ * @optest      efl|=of,cf  op1=0 op2=0  -> op1=0    efl&|=nc,po,na,zf,pl,nv
+ * @optest      op1=-2 op2=1  -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o16 / op1=0x5a5a             op2=0xa5a5             -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o32 / op1=0x5a5a5a5a         op2=0xa5a5a5a5         -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o64 / op1=0x5a5a5a5a5a5a5a5a op2=0xa5a5a5a5a5a5a5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ */
+FNIEMOP_DEF(iemOp_or_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, OR, or, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_or);
+}
+
+
+/**
+ * @opcode      0x0a
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @opcopytests iemOp_or_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_or_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, OR, or, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_or);
+}
+
+
+/**
+ * @opcode      0x0b
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @opcopytests iemOp_or_Ev_Gv
+ */
+FNIEMOP_DEF(iemOp_or_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, OR, or, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_or);
+}
+
+
+/**
+ * @opcode      0x0c
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @opcopytests iemOp_or_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_or_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, OR, or, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_or);
+}
+
+
+/**
+ * @opcode      0x0d
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ * @optest      efl|=of,cf  op1=12 op2=7 -> op1=15   efl&|=nc,po,na,nz,pl,nv
+ * @optest      efl|=of,cf  op1=0 op2=0  -> op1=0    efl&|=nc,po,na,zf,pl,nv
+ * @optest      op1=-2 op2=1  -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o16 / op1=0x5a5a             op2=0xa5a5     -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o32 / op1=0x5a5a5a5a         op2=0xa5a5a5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o64 / op1=0x5a5a5a5a5a5a5a5a op2=0xa5a5a5a5 -> op1=-1 efl&|=nc,po,na,nz,ng,nv
+ * @optest      o64 / op1=0x5a5a5a5aa5a5a5a5 op2=0x5a5a5a5a -> op1=0x5a5a5a5affffffff efl&|=nc,po,na,nz,pl,nv
+ */
+FNIEMOP_DEF(iemOp_or_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, OR, or, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_or);
+}
+
+
+/**
+ * @opcode      0x0e
+ * @opgroup     og_stack_sreg
+ */
+FNIEMOP_DEF(iemOp_push_CS)
+{
+    IEMOP_MNEMONIC1(FIXED, PUSH, push, CS, DISOPTYPE_HARMLESS | DISOPTYPE_POTENTIALLY_DANGEROUS | DISOPTYPE_INVALID_64, 0);
+    IEMOP_HLP_NO_64BIT();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_CS);
+}
+
+
+/**
+ * @opcode      0x0f
+ * @opmnemonic  EscTwo0f
+ * @openc       two0f
+ * @opdisenum   OP_2B_ESC
+ * @ophints     harmless
+ * @opgroup     og_escapes
+ */
+FNIEMOP_DEF(iemOp_2byteEscape)
+{
+#ifdef VBOX_STRICT
+    /* Sanity check the table the first time around. */
+    static bool s_fTested = false;
+    if (RT_LIKELY(s_fTested)) { /* likely */  }
+    else
+    {
+        s_fTested = true;
+        Assert(g_apfnTwoByteMap[0xbc * 4 + 0] == iemOp_bsf_Gv_Ev);
+        Assert(g_apfnTwoByteMap[0xbc * 4 + 1] == iemOp_bsf_Gv_Ev);
+        Assert(g_apfnTwoByteMap[0xbc * 4 + 2] == iemOp_tzcnt_Gv_Ev);
+        Assert(g_apfnTwoByteMap[0xbc * 4 + 3] == iemOp_bsf_Gv_Ev);
+    }
+#endif
+
+    if (RT_LIKELY(IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_286))
+    {
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        IEMOP_HLP_MIN_286();
+        return FNIEMOP_CALL(g_apfnTwoByteMap[(uintptr_t)b * 4 + pVCpu->iem.s.idxPrefix]);
+    }
+    /* @opdone */
+
+    /*
+     * On the 8086 this is a POP CS instruction.
+     * For the time being we don't specify this this.
+     */
+    IEMOP_MNEMONIC1(FIXED, POP, pop, CS, DISOPTYPE_HARMLESS | DISOPTYPE_POTENTIALLY_DANGEROUS | DISOPTYPE_INVALID_64, IEMOPHINT_SKIP_PYTHON);
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_ES, pVCpu->iem.s.enmEffOpSize);
+}
+
+/**
+ * @opcode      0x10
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @optest      op1=1 op2=1 efl&~=cf -> op1=2 efl&|=nc,pe,na,nz,pl,nv
+ * @optest      op1=1 op2=1 efl|=cf  -> op1=3 efl&|=nc,po,na,nz,pl,nv
+ * @optest      op1=0xff op2=0 efl|=cf -> op1=0 efl&|=cf,po,af,zf,pl,nv
+ * @optest      op1=0  op2=0 efl|=cf -> op1=1 efl&|=nc,pe,na,nz,pl,nv
+ * @optest      op1=0  op2=0 efl&~=cf -> op1=0 efl&|=nc,po,na,zf,pl,nv
+ */
+FNIEMOP_DEF(iemOp_adc_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, ADC, adc, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x11
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @optest      op1=1 op2=1 efl&~=cf -> op1=2 efl&|=nc,pe,na,nz,pl,nv
+ * @optest      op1=1 op2=1 efl|=cf  -> op1=3 efl&|=nc,po,na,nz,pl,nv
+ * @optest      op1=-1 op2=0 efl|=cf -> op1=0 efl&|=cf,po,af,zf,pl,nv
+ * @optest      op1=0  op2=0 efl|=cf -> op1=1 efl&|=nc,pe,na,nz,pl,nv
+ * @optest      op1=0  op2=0 efl&~=cf -> op1=0 efl&|=nc,po,na,zf,pl,nv
+ */
+FNIEMOP_DEF(iemOp_adc_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, ADC, adc, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x12
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_adc_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_adc_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, ADC, adc, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x13
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_adc_Ev_Gv
+ */
+FNIEMOP_DEF(iemOp_adc_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, ADC, adc, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x14
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_adc_Eb_Gb
+ */
+FNIEMOP_DEF(iemOp_adc_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, ADC, adc, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x15
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opcopytests iemOp_adc_Ev_Gv
+ */
+FNIEMOP_DEF(iemOp_adc_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, ADC, adc, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_adc);
+}
+
+
+/**
+ * @opcode      0x16
+ */
+FNIEMOP_DEF(iemOp_push_SS)
+{
+    IEMOP_MNEMONIC1(FIXED, PUSH, push, SS, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64 | DISOPTYPE_RRM_DANGEROUS, 0);
+    IEMOP_HLP_NO_64BIT();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_SS);
+}
+
+
+/**
+ * @opcode      0x17
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_pop_SS)
+{
+    IEMOP_MNEMONIC1(FIXED, POP, pop, SS, DISOPTYPE_HARMLESS | DISOPTYPE_INHIBIT_IRQS | DISOPTYPE_INVALID_64 | DISOPTYPE_RRM_DANGEROUS , 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_NO_64BIT();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_SS, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0x18
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, SBB, sbb, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x19
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, SBB, sbb, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x1a
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, SBB, sbb, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x1b
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, SBB, sbb, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x1c
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, SBB, sbb, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x1d
+ * @opgroup     og_gen_arith_bin
+ * @opfltest    cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sbb_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, SBB, sbb, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_sbb);
+}
+
+
+/**
+ * @opcode      0x1e
+ * @opgroup     og_stack_sreg
+ */
+FNIEMOP_DEF(iemOp_push_DS)
+{
+    IEMOP_MNEMONIC1(FIXED, PUSH, push, DS, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0);
+    IEMOP_HLP_NO_64BIT();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_DS);
+}
+
+
+/**
+ * @opcode      0x1f
+ * @opgroup     og_stack_sreg
+ */
+FNIEMOP_DEF(iemOp_pop_DS)
+{
+    IEMOP_MNEMONIC1(FIXED, POP, pop, DS, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64 | DISOPTYPE_RRM_DANGEROUS, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_NO_64BIT();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_DS, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0x20
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, AND, and, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x21
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, AND, and, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x22
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, AND, and, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x23
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, AND, and, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x24
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, AND, and, AL, Ib, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x25
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_and_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, AND, and, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_and);
+}
+
+
+/**
+ * @opcode      0x26
+ * @opmnemonic  SEG
+ * @op1         ES
+ * @opgroup     og_prefix
+ * @openc       prefix
+ * @opdisenum   OP_SEG
+ * @ophints     harmless
+ */
+FNIEMOP_DEF(iemOp_seg_ES)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg es");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_ES;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_ES;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x27
+ * @opfltest    af,cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   of
+ */
+FNIEMOP_DEF(iemOp_daa)
+{
+    IEMOP_MNEMONIC0(FIXED, DAA, daa, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL register use */
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF);
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_daa);
+}
+
+
+/**
+ * @opcode      0x28
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, SUB, sub, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x29
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, SUB, sub, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x2a
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, SUB, sub, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x2b
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, SUB, sub, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x2c
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, SUB, sub, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x2d
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ */
+FNIEMOP_DEF(iemOp_sub_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, SUB, sub, rAX, Iz, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_sub);
+}
+
+
+/**
+ * @opcode      0x2e
+ * @opmnemonic  SEG
+ * @op1         CS
+ * @opgroup     og_prefix
+ * @openc       prefix
+ * @opdisenum   OP_SEG
+ * @ophints     harmless
+ */
+FNIEMOP_DEF(iemOp_seg_CS)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg cs");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_CS;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_CS;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x2f
+ * @opfltest    af,cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   of
+ */
+FNIEMOP_DEF(iemOp_das)
+{
+    IEMOP_MNEMONIC0(FIXED, DAS, das, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL register use */
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF);
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_das);
+}
+
+
+/**
+ * @opcode      0x30
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_Eb_Gb)
+{
+    IEMOP_MNEMONIC2(MR, XOR, xor, Eb, Gb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x31
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_Ev_Gv)
+{
+    IEMOP_MNEMONIC2(MR, XOR, xor, Ev, Gv, DISOPTYPE_HARMLESS, IEMOPHINT_LOCK_ALLOWED);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x32
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_Gb_Eb)
+{
+    IEMOP_MNEMONIC2(RM, XOR, xor, Gb, Eb, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x33
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, XOR, xor, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x34
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_Al_Ib)
+{
+    IEMOP_MNEMONIC2(FIXED, XOR, xor, AL, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x35
+ * @opgroup     og_gen_arith_bin
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   af
+ * @opflclear   of,cf
+ */
+FNIEMOP_DEF(iemOp_xor_eAX_Iz)
+{
+    IEMOP_MNEMONIC2(FIXED, XOR, xor, rAX, Iz, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_xor);
+}
+
+
+/**
+ * @opcode      0x36
+ * @opmnemonic  SEG
+ * @op1         SS
+ * @opgroup     og_prefix
+ * @openc       prefix
+ * @opdisenum   OP_SEG
+ * @ophints     harmless
+ */
+FNIEMOP_DEF(iemOp_seg_SS)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg ss");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_SS;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_SS;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x37
+ * @opfltest    af,cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   pf,zf,sf,of
+ * @opgroup     og_gen_arith_dec
+ * @optest              efl&~=af ax=9      -> efl&|=nc,po,na,nz,pl,nv
+ * @optest              efl&~=af ax=0      -> efl&|=nc,po,na,zf,pl,nv
+ * @optest      intel / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,zf,pl,nv
+ * @optest      amd   / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,nz,pl,nv
+ * @optest              efl&~=af ax=0x00f9 -> ax=0x0009 efl&|=nc,po,na,nz,pl,nv
+ * @optest              efl|=af  ax=0      -> ax=0x0106 efl&|=cf,po,af,nz,pl,nv
+ * @optest              efl|=af  ax=0x0100 -> ax=0x0206 efl&|=cf,po,af,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0x000a -> ax=0x0100 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl|=af  ax=0x000a -> ax=0x0100 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0x010a -> ax=0x0200 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl|=af  ax=0x010a -> ax=0x0200 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0x0f0a -> ax=0x1000 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl|=af  ax=0x0f0a -> ax=0x1000 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0x7f0a -> ax=0x8000 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl|=af  ax=0x7f0a -> ax=0x8000 efl&|=cf,pe,af,nz,ng,ov
+ * @optest      intel / efl|=af  ax=0xff0a -> ax=0x0000 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl|=af  ax=0xff0a -> ax=0x0000 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0xff0a -> ax=0x0000 efl&|=cf,po,af,zf,pl,nv
+ * @optest      amd   / efl&~=af ax=0xff0a -> ax=0x0000 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x000b -> ax=0x0101 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000b -> ax=0x0101 efl&|=cf,po,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x000c -> ax=0x0102 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000c -> ax=0x0102 efl&|=cf,po,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x000d -> ax=0x0103 efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000d -> ax=0x0103 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x000e -> ax=0x0104 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000e -> ax=0x0104 efl&|=cf,po,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x000f -> ax=0x0105 efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000f -> ax=0x0105 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl&~=af ax=0x020f -> ax=0x0305 efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x020f -> ax=0x0305 efl&|=cf,pe,af,nz,pl,nv
+ */
+FNIEMOP_DEF(iemOp_aaa)
+{
+    IEMOP_MNEMONIC0(FIXED, AAA, aaa, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL/AX register use */
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF);
+
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_aaa);
+}
+
+
+/**
+ * @opcode      0x38
+ */
+FNIEMOP_DEF(iemOp_cmp_Eb_Gb)
+{
+    IEMOP_MNEMONIC(cmp_Eb_Gb, "cmp Eb,Gb");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x39
+ */
+FNIEMOP_DEF(iemOp_cmp_Ev_Gv)
+{
+    IEMOP_MNEMONIC(cmp_Ev_Gv, "cmp Ev,Gv");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x3a
+ */
+FNIEMOP_DEF(iemOp_cmp_Gb_Eb)
+{
+    IEMOP_MNEMONIC(cmp_Gb_Eb, "cmp Gb,Eb");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_r8_rm, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x3b
+ */
+FNIEMOP_DEF(iemOp_cmp_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmp_Gv_Ev, "cmp Gv,Ev");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x3c
+ */
+FNIEMOP_DEF(iemOp_cmp_Al_Ib)
+{
+    IEMOP_MNEMONIC(cmp_al_Ib, "cmp al,Ib");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x3d
+ */
+FNIEMOP_DEF(iemOp_cmp_eAX_Iz)
+{
+    IEMOP_MNEMONIC(cmp_rAX_Iz, "cmp rAX,Iz");
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_cmp);
+}
+
+
+/**
+ * @opcode      0x3e
+ */
+FNIEMOP_DEF(iemOp_seg_DS)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg ds");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_DS;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_DS;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x3f
+ * @opfltest    af,cf
+ * @opflmodify  cf,pf,af,zf,sf,of
+ * @opflundef   pf,zf,sf,of
+ * @opgroup     og_gen_arith_dec
+ * @optest            / efl&~=af ax=0x0009 -> efl&|=nc,po,na,nz,pl,nv
+ * @optest            / efl&~=af ax=0x0000 -> efl&|=nc,po,na,zf,pl,nv
+ * @optest      intel / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,zf,pl,nv
+ * @optest      amd   / efl&~=af ax=0x00f0 -> ax=0x0000 efl&|=nc,po,na,nz,pl,nv
+ * @optest            / efl&~=af ax=0x00f9 -> ax=0x0009 efl&|=nc,po,na,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0x0000 -> ax=0xfe0a efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl|=af  ax=0x0000 -> ax=0xfe0a efl&|=cf,po,af,nz,ng,nv
+ * @optest      intel / efl|=af  ax=0x0100 -> ax=0xff0a efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl|=af  ax=0x0100 -> ax=0xff0a efl&|=cf,po,af,nz,ng,nv
+ * @optest      intel / efl|=af  ax=0x000a -> ax=0xff04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl|=af  ax=0x000a -> ax=0xff04 efl&|=cf,pe,af,nz,ng,nv
+ * @optest            / efl|=af  ax=0x010a -> ax=0x0004 efl&|=cf,pe,af,nz,pl,nv
+ * @optest            / efl|=af  ax=0x020a -> ax=0x0104 efl&|=cf,pe,af,nz,pl,nv
+ * @optest            / efl|=af  ax=0x0f0a -> ax=0x0e04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest            / efl|=af  ax=0x7f0a -> ax=0x7e04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      intel / efl|=af  ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl|=af  ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0xff0a -> ax=0xfe04 efl&|=cf,pe,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0xff09 -> ax=0xff09 efl&|=nc,po,na,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0xff09 -> ax=0xff09 efl&|=nc,po,na,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x000b -> ax=0xff05 efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000b -> ax=0xff05 efl&|=cf,po,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x000c -> ax=0xff06 efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000c -> ax=0xff06 efl&|=cf,po,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x000d -> ax=0xff07 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000d -> ax=0xff07 efl&|=cf,pe,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x000e -> ax=0xff08 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000e -> ax=0xff08 efl&|=cf,pe,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x000f -> ax=0xff09 efl&|=cf,po,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x000f -> ax=0xff09 efl&|=cf,po,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0x00fa -> ax=0xff04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0x00fa -> ax=0xff04 efl&|=cf,pe,af,nz,ng,nv
+ * @optest      intel / efl&~=af ax=0xfffa -> ax=0xfe04 efl&|=cf,pe,af,nz,pl,nv
+ * @optest      amd   / efl&~=af ax=0xfffa -> ax=0xfe04 efl&|=cf,pe,af,nz,ng,nv
+ */
+FNIEMOP_DEF(iemOp_aas)
+{
+    IEMOP_MNEMONIC0(FIXED, AAS, aas, DISOPTYPE_HARMLESS | DISOPTYPE_INVALID_64, 0); /* express implicit AL/AX register use */
+    IEMOP_HLP_NO_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_OF);
+
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_aas);
+}
+
+
+/**
+ * Common 'inc/dec/not/neg register' helper.
+ */
+FNIEMOP_DEF_2(iemOpCommonUnaryGReg, PCIEMOPUNARYSIZES, pImpl, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint16_t *,  pu16Dst, 0);
+            IEM_MC_ARG(uint32_t *,  pEFlags, 1);
+            IEM_MC_REF_GREG_U16(pu16Dst, iReg);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU16, pu16Dst, pEFlags);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint32_t *,  pu32Dst, 0);
+            IEM_MC_ARG(uint32_t *,  pEFlags, 1);
+            IEM_MC_REF_GREG_U32(pu32Dst, iReg);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU32, pu32Dst, pEFlags);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint64_t *,  pu64Dst, 0);
+            IEM_MC_ARG(uint32_t *,  pEFlags, 1);
+            IEM_MC_REF_GREG_U64(pu64Dst, iReg);
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU64, pu64Dst, pEFlags);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x40
+ */
+FNIEMOP_DEF(iemOp_inc_eAX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eAX, "inc eAX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xAX);
+}
+
+
+/**
+ * @opcode      0x41
+ */
+FNIEMOP_DEF(iemOp_inc_eCX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.b");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eCX, "inc eCX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xCX);
+}
+
+
+/**
+ * @opcode      0x42
+ */
+FNIEMOP_DEF(iemOp_inc_eDX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.x");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_X;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eDX, "inc eDX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xDX);
+}
+
+
+
+/**
+ * @opcode      0x43
+ */
+FNIEMOP_DEF(iemOp_inc_eBX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.bx");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eBX, "inc eBX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xBX);
+}
+
+
+/**
+ * @opcode      0x44
+ */
+FNIEMOP_DEF(iemOp_inc_eSP)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.r");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eSP, "inc eSP");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xSP);
+}
+
+
+/**
+ * @opcode      0x45
+ */
+FNIEMOP_DEF(iemOp_inc_eBP)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rb");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eBP, "inc eBP");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xBP);
+}
+
+
+/**
+ * @opcode      0x46
+ */
+FNIEMOP_DEF(iemOp_inc_eSI)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rx");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_X;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eSI, "inc eSI");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xSI);
+}
+
+
+/**
+ * @opcode      0x47
+ */
+FNIEMOP_DEF(iemOp_inc_eDI)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rbx");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(inc_eDI, "inc eDI");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_inc, X86_GREG_xDI);
+}
+
+
+/**
+ * @opcode      0x48
+ */
+FNIEMOP_DEF(iemOp_dec_eAX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.w");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_SIZE_REX_W;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eAX, "dec eAX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xAX);
+}
+
+
+/**
+ * @opcode      0x49
+ */
+FNIEMOP_DEF(iemOp_dec_eCX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.bw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eCX, "dec eCX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xCX);
+}
+
+
+/**
+ * @opcode      0x4a
+ */
+FNIEMOP_DEF(iemOp_dec_eDX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.xw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eDX, "dec eDX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xDX);
+}
+
+
+/**
+ * @opcode      0x4b
+ */
+FNIEMOP_DEF(iemOp_dec_eBX)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.bxw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eBX, "dec eBX");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xBX);
+}
+
+
+/**
+ * @opcode      0x4c
+ */
+FNIEMOP_DEF(iemOp_dec_eSP)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eSP, "dec eSP");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xSP);
+}
+
+
+/**
+ * @opcode      0x4d
+ */
+FNIEMOP_DEF(iemOp_dec_eBP)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rbw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eBP, "dec eBP");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xBP);
+}
+
+
+/**
+ * @opcode      0x4e
+ */
+FNIEMOP_DEF(iemOp_dec_eSI)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rxw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eSI, "dec eSI");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xSI);
+}
+
+
+/**
+ * @opcode      0x4f
+ */
+FNIEMOP_DEF(iemOp_dec_eDI)
+{
+    /*
+     * This is a REX prefix in 64-bit mode.
+     */
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("rex.rbxw");
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REX | IEM_OP_PRF_REX_R | IEM_OP_PRF_REX_B | IEM_OP_PRF_REX_X | IEM_OP_PRF_SIZE_REX_W;
+        pVCpu->iem.s.uRexReg   = 1 << 3;
+        pVCpu->iem.s.uRexB     = 1 << 3;
+        pVCpu->iem.s.uRexIndex = 1 << 3;
+        iemRecalEffOpSize(pVCpu);
+
+        uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+        return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+    }
+
+    IEMOP_MNEMONIC(dec_eDI, "dec eDI");
+    return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, &g_iemAImpl_dec, X86_GREG_xDI);
+}
+
+
+/**
+ * Common 'push register' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonPushGReg, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        iReg |= pVCpu->iem.s.uRexB;
+        pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+        pVCpu->iem.s.enmEffOpSize = !(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP) ? IEMMODE_64BIT : IEMMODE_16BIT;
+    }
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint16_t, u16Value);
+            IEM_MC_FETCH_GREG_U16(u16Value, iReg);
+            IEM_MC_PUSH_U16(u16Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_FETCH_GREG_U32(u32Value, iReg);
+            IEM_MC_PUSH_U32(u32Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_FETCH_GREG_U64(u64Value, iReg);
+            IEM_MC_PUSH_U64(u64Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x50
+ */
+FNIEMOP_DEF(iemOp_push_eAX)
+{
+    IEMOP_MNEMONIC(push_rAX, "push rAX");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xAX);
+}
+
+
+/**
+ * @opcode      0x51
+ */
+FNIEMOP_DEF(iemOp_push_eCX)
+{
+    IEMOP_MNEMONIC(push_rCX, "push rCX");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xCX);
+}
+
+
+/**
+ * @opcode      0x52
+ */
+FNIEMOP_DEF(iemOp_push_eDX)
+{
+    IEMOP_MNEMONIC(push_rDX, "push rDX");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xDX);
+}
+
+
+/**
+ * @opcode      0x53
+ */
+FNIEMOP_DEF(iemOp_push_eBX)
+{
+    IEMOP_MNEMONIC(push_rBX, "push rBX");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xBX);
+}
+
+
+/**
+ * @opcode      0x54
+ */
+FNIEMOP_DEF(iemOp_push_eSP)
+{
+    IEMOP_MNEMONIC(push_rSP, "push rSP");
+    if (IEM_GET_TARGET_CPU(pVCpu) == IEMTARGETCPU_8086)
+    {
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint16_t, u16Value);
+        IEM_MC_FETCH_GREG_U16(u16Value, X86_GREG_xSP);
+        IEM_MC_SUB_LOCAL_U16(u16Value, 2);
+        IEM_MC_PUSH_U16(u16Value);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xSP);
+}
+
+
+/**
+ * @opcode      0x55
+ */
+FNIEMOP_DEF(iemOp_push_eBP)
+{
+    IEMOP_MNEMONIC(push_rBP, "push rBP");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xBP);
+}
+
+
+/**
+ * @opcode      0x56
+ */
+FNIEMOP_DEF(iemOp_push_eSI)
+{
+    IEMOP_MNEMONIC(push_rSI, "push rSI");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xSI);
+}
+
+
+/**
+ * @opcode      0x57
+ */
+FNIEMOP_DEF(iemOp_push_eDI)
+{
+    IEMOP_MNEMONIC(push_rDI, "push rDI");
+    return FNIEMOP_CALL_1(iemOpCommonPushGReg, X86_GREG_xDI);
+}
+
+
+/**
+ * Common 'pop register' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonPopGReg, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        iReg |= pVCpu->iem.s.uRexB;
+        pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+        pVCpu->iem.s.enmEffOpSize = !(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP) ? IEMMODE_64BIT : IEMMODE_16BIT;
+    }
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint16_t *, pu16Dst);
+            IEM_MC_REF_GREG_U16(pu16Dst, iReg);
+            IEM_MC_POP_U16(pu16Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t *, pu32Dst);
+            IEM_MC_REF_GREG_U32(pu32Dst, iReg);
+            IEM_MC_POP_U32(pu32Dst);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst); /** @todo testcase*/
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t *, pu64Dst);
+            IEM_MC_REF_GREG_U64(pu64Dst, iReg);
+            IEM_MC_POP_U64(pu64Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x58
+ */
+FNIEMOP_DEF(iemOp_pop_eAX)
+{
+    IEMOP_MNEMONIC(pop_rAX, "pop rAX");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xAX);
+}
+
+
+/**
+ * @opcode      0x59
+ */
+FNIEMOP_DEF(iemOp_pop_eCX)
+{
+    IEMOP_MNEMONIC(pop_rCX, "pop rCX");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xCX);
+}
+
+
+/**
+ * @opcode      0x5a
+ */
+FNIEMOP_DEF(iemOp_pop_eDX)
+{
+    IEMOP_MNEMONIC(pop_rDX, "pop rDX");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xDX);
+}
+
+
+/**
+ * @opcode      0x5b
+ */
+FNIEMOP_DEF(iemOp_pop_eBX)
+{
+    IEMOP_MNEMONIC(pop_rBX, "pop rBX");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xBX);
+}
+
+
+/**
+ * @opcode      0x5c
+ */
+FNIEMOP_DEF(iemOp_pop_eSP)
+{
+    IEMOP_MNEMONIC(pop_rSP, "pop rSP");
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+    {
+        if (pVCpu->iem.s.uRexB)
+            return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xSP);
+        pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+        pVCpu->iem.s.enmEffOpSize = !(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_OP) ? IEMMODE_64BIT : IEMMODE_16BIT;
+    }
+
+    IEMOP_HLP_DECODED_NL_1(OP_POP, IEMOPFORM_FIXED, OP_PARM_REG_ESP,
+                           DISOPTYPE_HARMLESS | DISOPTYPE_DEFAULT_64_OP_SIZE | DISOPTYPE_REXB_EXTENDS_OPREG);
+    /** @todo add testcase for this instruction. */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint16_t, u16Dst);
+            IEM_MC_POP_U16(&u16Dst); /** @todo not correct MC, fix later. */
+            IEM_MC_STORE_GREG_U16(X86_GREG_xSP, u16Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Dst);
+            IEM_MC_POP_U32(&u32Dst);
+            IEM_MC_STORE_GREG_U32(X86_GREG_xSP, u32Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Dst);
+            IEM_MC_POP_U64(&u64Dst);
+            IEM_MC_STORE_GREG_U64(X86_GREG_xSP, u64Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x5d
+ */
+FNIEMOP_DEF(iemOp_pop_eBP)
+{
+    IEMOP_MNEMONIC(pop_rBP, "pop rBP");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xBP);
+}
+
+
+/**
+ * @opcode      0x5e
+ */
+FNIEMOP_DEF(iemOp_pop_eSI)
+{
+    IEMOP_MNEMONIC(pop_rSI, "pop rSI");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xSI);
+}
+
+
+/**
+ * @opcode      0x5f
+ */
+FNIEMOP_DEF(iemOp_pop_eDI)
+{
+    IEMOP_MNEMONIC(pop_rDI, "pop rDI");
+    return FNIEMOP_CALL_1(iemOpCommonPopGReg, X86_GREG_xDI);
+}
+
+
+/**
+ * @opcode      0x60
+ */
+FNIEMOP_DEF(iemOp_pusha)
+{
+    IEMOP_MNEMONIC(pusha, "pusha");
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_NO_64BIT();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+        return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_pusha_16);
+    Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT);
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_pusha_32);
+}
+
+
+/**
+ * @opcode      0x61
+ */
+FNIEMOP_DEF(iemOp_popa__mvex)
+{
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+    {
+        IEMOP_MNEMONIC(popa, "popa");
+        IEMOP_HLP_MIN_186();
+        IEMOP_HLP_NO_64BIT();
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+            return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_popa_16);
+        Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_32BIT);
+        return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_popa_32);
+    }
+    IEMOP_MNEMONIC(mvex, "mvex");
+    Log(("mvex prefix is not supported!\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x62
+ * @opmnemonic  bound
+ * @op1         Gv_RO
+ * @op2         Ma
+ * @opmincpu    80186
+ * @ophints     harmless invalid_64
+ * @optest      op1=0 op2=0 ->
+ * @optest      op1=1 op2=0 -> value.xcpt=5
+ * @optest      o16 / op1=0xffff op2=0x0000fffe ->
+ * @optest      o16 / op1=0xfffe op2=0x0000fffe ->
+ * @optest      o16 / op1=0x7fff op2=0x0000fffe -> value.xcpt=5
+ * @optest      o16 / op1=0x7fff op2=0x7ffffffe ->
+ * @optest      o16 / op1=0x7fff op2=0xfffe8000 -> value.xcpt=5
+ * @optest      o16 / op1=0x8000 op2=0xfffe8000 ->
+ * @optest      o16 / op1=0xffff op2=0xfffe8000 -> value.xcpt=5
+ * @optest      o16 / op1=0xfffe op2=0xfffe8000 ->
+ * @optest      o16 / op1=0xfffe op2=0x8000fffe -> value.xcpt=5
+ * @optest      o16 / op1=0x8000 op2=0x8000fffe -> value.xcpt=5
+ * @optest      o16 / op1=0x0000 op2=0x8000fffe -> value.xcpt=5
+ * @optest      o16 / op1=0x0001 op2=0x8000fffe -> value.xcpt=5
+ * @optest      o16 / op1=0xffff op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0000 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0001 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0002 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0003 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0004 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x000e op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x000f op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0010 op2=0x0001000f -> value.xcpt=5
+ * @optest      o16 / op1=0x0011 op2=0x0001000f -> value.xcpt=5
+ * @optest      o32 / op1=0xffffffff op2=0x00000000fffffffe ->
+ * @optest      o32 / op1=0xfffffffe op2=0x00000000fffffffe ->
+ * @optest      o32 / op1=0x7fffffff op2=0x00000000fffffffe -> value.xcpt=5
+ * @optest      o32 / op1=0x7fffffff op2=0x7ffffffffffffffe ->
+ * @optest      o32 / op1=0x7fffffff op2=0xfffffffe80000000 -> value.xcpt=5
+ * @optest      o32 / op1=0x80000000 op2=0xfffffffe80000000 ->
+ * @optest      o32 / op1=0xffffffff op2=0xfffffffe80000000 -> value.xcpt=5
+ * @optest      o32 / op1=0xfffffffe op2=0xfffffffe80000000 ->
+ * @optest      o32 / op1=0xfffffffe op2=0x80000000fffffffe -> value.xcpt=5
+ * @optest      o32 / op1=0x80000000 op2=0x80000000fffffffe -> value.xcpt=5
+ * @optest      o32 / op1=0x00000000 op2=0x80000000fffffffe -> value.xcpt=5
+ * @optest      o32 / op1=0x00000002 op2=0x80000000fffffffe -> value.xcpt=5
+ * @optest      o32 / op1=0x00000001 op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x00000002 op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x00000003 op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x00000004 op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x00000005 op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x0000000e op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x0000000f op2=0x0000000100000003 -> value.xcpt=5
+ * @optest      o32 / op1=0x00000010 op2=0x0000000100000003 -> value.xcpt=5
+ */
+FNIEMOP_DEF(iemOp_bound_Gv_Ma__evex)
+{
+    /* The BOUND instruction is invalid 64-bit mode. In legacy and
+       compatability mode it is invalid with MOD=3.
+
+       In 32-bit mode, the EVEX prefix works by having the top two bits (MOD)
+       both be set.  In the Intel EVEX documentation (sdm vol 2) these are simply
+       given as R and X without an exact description, so we assume it builds on
+       the VEX one and means they are inverted wrt REX.R and REX.X.  Thus, just
+       like with the 3-byte VEX, 32-bit code is restrict wrt addressable registers. */
+    uint8_t bRm;
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+    {
+        IEMOP_MNEMONIC2(RM_MEM, BOUND, bound, Gv_RO, Ma, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+        IEMOP_HLP_MIN_186();
+        IEM_OPCODE_GET_NEXT_U8(&bRm);
+        if (IEM_IS_MODRM_MEM_MODE(bRm))
+        {
+            /** @todo testcase: check that there are two memory accesses involved.  Check
+             *        whether they're both read before the \#BR triggers. */
+            if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+            {
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t,    u16Index,       0); /* Note! All operands are actually signed. Lazy unsigned bird. */
+                IEM_MC_ARG(uint16_t,    u16LowerBounds, 1);
+                IEM_MC_ARG(uint16_t,    u16UpperBounds, 2);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_FETCH_GREG_U16(u16Index, IEM_GET_MODRM_REG_8(bRm));
+                IEM_MC_FETCH_MEM_U16(u16LowerBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_FETCH_MEM_U16_DISP(u16UpperBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 2);
+
+                IEM_MC_CALL_CIMPL_3(iemCImpl_bound_16, u16Index, u16LowerBounds, u16UpperBounds); /* returns */
+                IEM_MC_END();
+            }
+            else /* 32-bit operands */
+            {
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t,    u32Index,       0); /* Note! All operands are actually signed. Lazy unsigned bird. */
+                IEM_MC_ARG(uint32_t,    u32LowerBounds, 1);
+                IEM_MC_ARG(uint32_t,    u32UpperBounds, 2);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_FETCH_GREG_U32(u32Index, IEM_GET_MODRM_REG_8(bRm));
+                IEM_MC_FETCH_MEM_U32(u32LowerBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_FETCH_MEM_U32_DISP(u32UpperBounds, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 4);
+
+                IEM_MC_CALL_CIMPL_3(iemCImpl_bound_32, u32Index, u32LowerBounds, u32UpperBounds); /* returns */
+                IEM_MC_END();
+            }
+        }
+
+        /*
+         * @opdone
+         */
+        if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx512Foundation)
+        {
+            /* Note that there is no need for the CPU to fetch further bytes
+               here because MODRM.MOD == 3. */
+            Log(("evex not supported by the guest CPU!\n"));
+            return IEMOP_RAISE_INVALID_OPCODE();
+        }
+    }
+    else
+    {
+        /** @todo check how this is decoded in 64-bit mode w/o EVEX. Intel probably
+         *        does modr/m read, whereas AMD probably doesn't... */
+        if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx512Foundation)
+        {
+            Log(("evex not supported by the guest CPU!\n"));
+            return FNIEMOP_CALL(iemOp_InvalidAllNeedRM);
+        }
+        IEM_OPCODE_GET_NEXT_U8(&bRm);
+    }
+
+    IEMOP_MNEMONIC(evex, "evex");
+    uint8_t bP2; IEM_OPCODE_GET_NEXT_U8(&bP2);
+    uint8_t bP3; IEM_OPCODE_GET_NEXT_U8(&bP3);
+    Log(("evex prefix is not implemented!\n"));
+    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+}
+
+
+/** Opcode 0x63 - non-64-bit modes. */
+FNIEMOP_DEF(iemOp_arpl_Ew_Gw)
+{
+    IEMOP_MNEMONIC(arpl_Ew_Gw, "arpl Ew,Gw");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* Register */
+        IEMOP_HLP_DECODED_NL_2(OP_ARPL, IEMOPFORM_MR_REG, OP_PARM_Ew, OP_PARM_Gw, DISOPTYPE_HARMLESS);
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint16_t *,      pu16Dst,    0);
+        IEM_MC_ARG(uint16_t,        u16Src,     1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+
+        IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_REF_GREG_U16(pu16Dst,  IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_arpl, pu16Dst, u16Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* Memory */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint16_t *, pu16Dst,          0);
+        IEM_MC_ARG(uint16_t,   u16Src,           1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DECODED_NL_2(OP_ARPL, IEMOPFORM_MR_REG, OP_PARM_Ew, OP_PARM_Gw, DISOPTYPE_HARMLESS);
+        IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_arpl, pu16Dst, u16Src, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode 0x63
+ *
+ * @note This is a weird one. It works like a regular move instruction if
+ *       REX.W isn't set, at least according to AMD docs (rev 3.15, 2009-11).
+ * @todo This definitely needs a testcase to verify the odd cases.  */
+FNIEMOP_DEF(iemOp_movsxd_Gv_Ev)
+{
+    Assert(pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT); /* Caller branched already . */
+
+    IEMOP_MNEMONIC(movsxd_Gv_Ev, "movsxd Gv,Ev");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /*
+             * Register to register.
+             */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_FETCH_GREG_U32_SX_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /*
+             * We're loading a register from memory.
+             */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U32_SX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+        AssertFailedReturn(VERR_IEM_INSTR_NOT_IMPLEMENTED);
+}
+
+
+/**
+ * @opcode      0x64
+ * @opmnemonic  segfs
+ * @opmincpu    80386
+ * @opgroup     og_prefixes
+ */
+FNIEMOP_DEF(iemOp_seg_FS)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg fs");
+    IEMOP_HLP_MIN_386();
+
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_FS;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_FS;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x65
+ * @opmnemonic  seggs
+ * @opmincpu    80386
+ * @opgroup     og_prefixes
+ */
+FNIEMOP_DEF(iemOp_seg_GS)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("seg gs");
+    IEMOP_HLP_MIN_386();
+
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SEG_GS;
+    pVCpu->iem.s.iEffSeg    = X86_SREG_GS;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x66
+ * @opmnemonic  opsize
+ * @openc       prefix
+ * @opmincpu    80386
+ * @ophints     harmless
+ * @opgroup     og_prefixes
+ */
+FNIEMOP_DEF(iemOp_op_size)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("op size");
+    IEMOP_HLP_MIN_386();
+
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_OP;
+    iemRecalEffOpSize(pVCpu);
+
+    /* For the 4 entry opcode tables, the operand prefix doesn't not count
+       when REPZ or REPNZ are present. */
+    if (pVCpu->iem.s.idxPrefix == 0)
+        pVCpu->iem.s.idxPrefix = 1;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x67
+ * @opmnemonic  addrsize
+ * @openc       prefix
+ * @opmincpu    80386
+ * @ophints     harmless
+ * @opgroup     og_prefixes
+ */
+FNIEMOP_DEF(iemOp_addr_size)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("addr size");
+    IEMOP_HLP_MIN_386();
+
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_ADDR;
+    switch (pVCpu->iem.s.enmDefAddrMode)
+    {
+        case IEMMODE_16BIT: pVCpu->iem.s.enmEffAddrMode = IEMMODE_32BIT; break;
+        case IEMMODE_32BIT: pVCpu->iem.s.enmEffAddrMode = IEMMODE_16BIT; break;
+        case IEMMODE_64BIT: pVCpu->iem.s.enmEffAddrMode = IEMMODE_32BIT; break;
+        default: AssertFailed();
+    }
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0x68
+ */
+FNIEMOP_DEF(iemOp_push_Iz)
+{
+    IEMOP_MNEMONIC(push_Iz, "push Iz");
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_PUSH_U16(u16Imm);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_PUSH_U32(u32Imm);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_PUSH_U64(u64Imm);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x69
+ */
+FNIEMOP_DEF(iemOp_imul_Gv_Ev_Iz)
+{
+    IEMOP_MNEMONIC(imul_Gv_Ev_Iz, "imul Gv,Ev,Iz"); /* Gv = Ev * Iz; */
+    IEMOP_HLP_MIN_186();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            if (IEM_IS_MODRM_REG_MODE(bRm))
+            {
+                /* register operand */
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,            0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src,/*=*/ u16Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint16_t,      u16Tmp);
+
+                IEM_MC_FETCH_GREG_U16(u16Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_LOCAL(pu16Dst, u16Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u16_eflags),
+                                         pu16Dst, u16Src, pEFlags);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,            0);
+                IEM_MC_ARG(uint16_t,        u16Src,             1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint16_t,      u16Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEM_MC_ASSIGN(u16Src, u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu16Dst, u16Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u16_eflags),
+                                         pu16Dst, u16Src, pEFlags);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            if (IEM_IS_MODRM_REG_MODE(bRm))
+            {
+                /* register operand */
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,            0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src,/*=*/ u32Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint32_t,      u32Tmp);
+
+                IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_LOCAL(pu32Dst, u32Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u32_eflags),
+                                         pu32Dst, u32Src, pEFlags);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,            0);
+                IEM_MC_ARG(uint32_t,        u32Src,             1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint32_t,      u32Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEM_MC_ASSIGN(u32Src, u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu32Dst, u32Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u32_eflags),
+                                         pu32Dst, u32Src, pEFlags);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            if (IEM_IS_MODRM_REG_MODE(bRm))
+            {
+                /* register operand */
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,            0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src,/*=*/ u64Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint64_t,      u64Tmp);
+
+                IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_LOCAL(pu64Dst, u64Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u64_eflags),
+                                         pu64Dst, u64Src, pEFlags);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,            0);
+                IEM_MC_ARG(uint64_t,        u64Src,             1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+                IEM_MC_LOCAL(uint64_t,      u64Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEM_MC_ASSIGN(u64Src, u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu64Dst, u64Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u64_eflags),
+                                         pu64Dst, u64Src, pEFlags);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x6a
+ */
+FNIEMOP_DEF(iemOp_push_Ib)
+{
+    IEMOP_MNEMONIC(push_Ib, "push Ib");
+    IEMOP_HLP_MIN_186();
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    IEM_MC_BEGIN(0,0);
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_PUSH_U16(i8Imm);
+            break;
+        case IEMMODE_32BIT:
+            IEM_MC_PUSH_U32(i8Imm);
+            break;
+        case IEMMODE_64BIT:
+            IEM_MC_PUSH_U64(i8Imm);
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x6b
+ */
+FNIEMOP_DEF(iemOp_imul_Gv_Ev_Ib)
+{
+    IEMOP_MNEMONIC(imul_Gv_Ev_Ib, "imul Gv,Ev,Ib"); /* Gv = Ev * Iz; */
+    IEMOP_HLP_MIN_186();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            if (IEM_IS_MODRM_REG_MODE(bRm))
+            {
+                /* register operand */
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                    0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src,/*=*/ (int8_t)u8Imm, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint16_t,      u16Tmp);
+
+                IEM_MC_FETCH_GREG_U16(u16Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_LOCAL(pu16Dst, u16Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u16_eflags),
+                                         pu16Dst, u16Src, pEFlags);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                    0);
+                IEM_MC_ARG(uint16_t,        u16Src,                     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint16_t,      u16Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_S8_SX_U16(&u16Imm);
+                IEM_MC_ASSIGN(u16Src, u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu16Dst, u16Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u16_eflags),
+                                         pu16Dst, u16Src, pEFlags);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            if (IEM_IS_MODRM_REG_MODE(bRm))
+            {
+                /* register operand */
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                    0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src,/*=*/ (int8_t)u8Imm, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint32_t,      u32Tmp);
+
+                IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_LOCAL(pu32Dst, u32Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u32_eflags),
+                                         pu32Dst, u32Src, pEFlags);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                    0);
+                IEM_MC_ARG(uint32_t,        u32Src,                     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint32_t,      u32Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_S8_SX_U32(&u32Imm);
+                IEM_MC_ASSIGN(u32Src, u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu32Dst, u32Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u32_eflags),
+                                         pu32Dst, u32Src, pEFlags);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            if (IEM_IS_MODRM_REG_MODE(bRm))
+            {
+                /* register operand */
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                    0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src,/*=*/ (int8_t)u8Imm, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint64_t,      u64Tmp);
+
+                IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_LOCAL(pu64Dst, u64Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u64_eflags),
+                                         pu64Dst, u64Src, pEFlags);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory operand */
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                    0);
+                IEM_MC_ARG(uint64_t,        u64Src,                     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+                IEM_MC_LOCAL(uint64_t,      u64Tmp);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S8_SX_U64(&u64Imm);
+                IEM_MC_ASSIGN(u64Src, u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_LOCAL(pu64Dst, u64Tmp);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_u64_eflags),
+                                         pu64Dst, u64Src, pEFlags);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    AssertFailedReturn(VERR_IEM_IPE_8);
+}
+
+
+/**
+ * @opcode      0x6c
+ */
+FNIEMOP_DEF(iemOp_insb_Yb_DX)
+{
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_insb_Yb_DX, "rep ins Yb,DX");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op8_addr16, false);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op8_addr32, false);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op8_addr64, false);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        IEMOP_MNEMONIC(ins_Yb_DX, "ins Yb,DX");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op8_addr16, false);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op8_addr32, false);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op8_addr64, false);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x6d
+ */
+FNIEMOP_DEF(iemOp_inswd_Yv_DX)
+{
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ))
+    {
+        IEMOP_MNEMONIC(rep_ins_Yv_DX, "rep ins Yv,DX");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op16_addr16, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op16_addr32, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op16_addr64, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_64BIT:
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op32_addr16, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op32_addr32, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_ins_op32_addr64, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        IEMOP_MNEMONIC(ins_Yv_DX, "ins Yv,DX");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op16_addr16, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op16_addr32, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op16_addr64, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_64BIT:
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op32_addr16, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op32_addr32, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_ins_op32_addr64, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x6e
+ */
+FNIEMOP_DEF(iemOp_outsb_Yb_DX)
+{
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_outsb_DX_Yb, "rep outs DX,Yb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op8_addr16, pVCpu->iem.s.iEffSeg, false);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op8_addr32, pVCpu->iem.s.iEffSeg, false);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op8_addr64, pVCpu->iem.s.iEffSeg, false);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        IEMOP_MNEMONIC(outs_DX_Yb, "outs DX,Yb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op8_addr16, pVCpu->iem.s.iEffSeg, false);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op8_addr32, pVCpu->iem.s.iEffSeg, false);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op8_addr64, pVCpu->iem.s.iEffSeg, false);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x6f
+ */
+FNIEMOP_DEF(iemOp_outswd_Yv_DX)
+{
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ))
+    {
+        IEMOP_MNEMONIC(rep_outs_DX_Yv, "rep outs DX,Yv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op16_addr16, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op16_addr32, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op16_addr64, pVCpu->iem.s.iEffSeg, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_64BIT:
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op32_addr16, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op32_addr32, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_rep_outs_op32_addr64, pVCpu->iem.s.iEffSeg, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        IEMOP_MNEMONIC(outs_DX_Yv, "outs DX,Yv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op16_addr16, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op16_addr32, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op16_addr64, pVCpu->iem.s.iEffSeg, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_64BIT:
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op32_addr16, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op32_addr32, pVCpu->iem.s.iEffSeg, false);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_outs_op32_addr64, pVCpu->iem.s.iEffSeg, false);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x70
+ */
+FNIEMOP_DEF(iemOp_jo_Jb)
+{
+    IEMOP_MNEMONIC(jo_Jb, "jo  Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x71
+ */
+FNIEMOP_DEF(iemOp_jno_Jb)
+{
+    IEMOP_MNEMONIC(jno_Jb, "jno Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+/**
+ * @opcode      0x72
+ */
+FNIEMOP_DEF(iemOp_jc_Jb)
+{
+    IEMOP_MNEMONIC(jc_Jb, "jc/jnae Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x73
+ */
+FNIEMOP_DEF(iemOp_jnc_Jb)
+{
+    IEMOP_MNEMONIC(jnc_Jb, "jnc/jnb Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x74
+ */
+FNIEMOP_DEF(iemOp_je_Jb)
+{
+    IEMOP_MNEMONIC(je_Jb, "je/jz   Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x75
+ */
+FNIEMOP_DEF(iemOp_jne_Jb)
+{
+    IEMOP_MNEMONIC(jne_Jb, "jne/jnz Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x76
+ */
+FNIEMOP_DEF(iemOp_jbe_Jb)
+{
+    IEMOP_MNEMONIC(jbe_Jb, "jbe/jna Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x77
+ */
+FNIEMOP_DEF(iemOp_jnbe_Jb)
+{
+    IEMOP_MNEMONIC(ja_Jb, "ja/jnbe Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x78
+ */
+FNIEMOP_DEF(iemOp_js_Jb)
+{
+    IEMOP_MNEMONIC(js_Jb, "js  Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x79
+ */
+FNIEMOP_DEF(iemOp_jns_Jb)
+{
+    IEMOP_MNEMONIC(jns_Jb, "jns Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x7a
+ */
+FNIEMOP_DEF(iemOp_jp_Jb)
+{
+    IEMOP_MNEMONIC(jp_Jb, "jp  Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x7b
+ */
+FNIEMOP_DEF(iemOp_jnp_Jb)
+{
+    IEMOP_MNEMONIC(jnp_Jb, "jnp Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x7c
+ */
+FNIEMOP_DEF(iemOp_jl_Jb)
+{
+    IEMOP_MNEMONIC(jl_Jb, "jl/jnge Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x7d
+ */
+FNIEMOP_DEF(iemOp_jnl_Jb)
+{
+    IEMOP_MNEMONIC(jge_Jb, "jnl/jge Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x7e
+ */
+FNIEMOP_DEF(iemOp_jle_Jb)
+{
+    IEMOP_MNEMONIC(jle_Jb, "jle/jng Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ELSE() {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x7f
+ */
+FNIEMOP_DEF(iemOp_jnle_Jb)
+{
+    IEMOP_MNEMONIC(jg_Jb, "jnle/jg Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+    } IEM_MC_ELSE() {
+        IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    } IEM_MC_ENDIF();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x80
+ */
+FNIEMOP_DEF(iemOp_Grp1_Eb_Ib_80)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0: IEMOP_MNEMONIC(add_Eb_Ib, "add Eb,Ib"); break;
+        case 1: IEMOP_MNEMONIC(or_Eb_Ib,  "or  Eb,Ib"); break;
+        case 2: IEMOP_MNEMONIC(adc_Eb_Ib, "adc Eb,Ib"); break;
+        case 3: IEMOP_MNEMONIC(sbb_Eb_Ib, "sbb Eb,Ib"); break;
+        case 4: IEMOP_MNEMONIC(and_Eb_Ib, "and Eb,Ib"); break;
+        case 5: IEMOP_MNEMONIC(sub_Eb_Ib, "sub Eb,Ib"); break;
+        case 6: IEMOP_MNEMONIC(xor_Eb_Ib, "xor Eb,Ib"); break;
+        case 7: IEMOP_MNEMONIC(cmp_Eb_Ib, "cmp Eb,Ib"); break;
+    }
+    PCIEMOPBINSIZES pImpl = g_apIemImplGrp1[IEM_GET_MODRM_REG_8(bRm)];
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,                 0);
+        IEM_MC_ARG_CONST(uint8_t,   u8Src, /*=*/ u8Imm,     1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        uint32_t fAccess;
+        if (pImpl->pfnLockedU8)
+            fAccess = IEM_ACCESS_DATA_RW;
+        else /* CMP */
+            fAccess = IEM_ACCESS_DATA_R;
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,                 0);
+        IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEM_MC_ARG_CONST(uint8_t,   u8Src, /*=*/ u8Imm,     1);
+        if (pImpl->pfnLockedU8)
+            IEMOP_HLP_DONE_DECODING();
+        else
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_MEM_MAP(pu8Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, u8Src, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, fAccess);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x81
+ */
+FNIEMOP_DEF(iemOp_Grp1_Ev_Iz)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0: IEMOP_MNEMONIC(add_Ev_Iz, "add Ev,Iz"); break;
+        case 1: IEMOP_MNEMONIC(or_Ev_Iz,  "or  Ev,Iz"); break;
+        case 2: IEMOP_MNEMONIC(adc_Ev_Iz, "adc Ev,Iz"); break;
+        case 3: IEMOP_MNEMONIC(sbb_Ev_Iz, "sbb Ev,Iz"); break;
+        case 4: IEMOP_MNEMONIC(and_Ev_Iz, "and Ev,Iz"); break;
+        case 5: IEMOP_MNEMONIC(sub_Ev_Iz, "sub Ev,Iz"); break;
+        case 6: IEMOP_MNEMONIC(xor_Ev_Iz, "xor Ev,Iz"); break;
+        case 7: IEMOP_MNEMONIC(cmp_Ev_Iz, "cmp Ev,Iz"); break;
+    }
+    PCIEMOPBINSIZES pImpl = g_apIemImplGrp1[IEM_GET_MODRM_REG_8(bRm)];
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            if (IEM_IS_MODRM_REG_MODE(bRm))
+            {
+                /* register target */
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src, /*=*/ u16Imm,   1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory target */
+                uint32_t fAccess;
+                if (pImpl->pfnLockedU16)
+                    fAccess = IEM_ACCESS_DATA_RW;
+                else /* CMP, TEST */
+                    fAccess = IEM_ACCESS_DATA_R;
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,        u16Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEM_MC_ASSIGN(u16Src, u16Imm);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            if (IEM_IS_MODRM_REG_MODE(bRm))
+            {
+                /* register target */
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src, /*=*/ u32Imm,   1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                if (pImpl != &g_iemAImpl_cmp)   /* TEST won't get here, no need to check for it. */
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory target */
+                uint32_t fAccess;
+                if (pImpl->pfnLockedU32)
+                    fAccess = IEM_ACCESS_DATA_RW;
+                else /* CMP, TEST */
+                    fAccess = IEM_ACCESS_DATA_R;
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,        u32Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEM_MC_ASSIGN(u32Src, u32Imm);
+                if (pImpl->pfnLockedU32)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            break;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            if (IEM_IS_MODRM_REG_MODE(bRm))
+            {
+                /* register target */
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src, /*=*/ u64Imm,   1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            else
+            {
+                /* memory target */
+                uint32_t fAccess;
+                if (pImpl->pfnLockedU64)
+                    fAccess = IEM_ACCESS_DATA_RW;
+                else /* CMP */
+                    fAccess = IEM_ACCESS_DATA_R;
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,        u64Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                if (pImpl->pfnLockedU64)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_ASSIGN(u64Src, u64Imm);
+                IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+            }
+            break;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x82
+ * @opmnemonic  grp1_82
+ * @opgroup     og_groups
+ */
+FNIEMOP_DEF(iemOp_Grp1_Eb_Ib_82)
+{
+    IEMOP_HLP_NO_64BIT(); /** @todo do we need to decode the whole instruction or is this ok? */
+    return FNIEMOP_CALL(iemOp_Grp1_Eb_Ib_80);
+}
+
+
+/**
+ * @opcode      0x83
+ */
+FNIEMOP_DEF(iemOp_Grp1_Ev_Ib)
+{
+    uint8_t bRm;   IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0: IEMOP_MNEMONIC(add_Ev_Ib, "add Ev,Ib"); break;
+        case 1: IEMOP_MNEMONIC(or_Ev_Ib,  "or  Ev,Ib"); break;
+        case 2: IEMOP_MNEMONIC(adc_Ev_Ib, "adc Ev,Ib"); break;
+        case 3: IEMOP_MNEMONIC(sbb_Ev_Ib, "sbb Ev,Ib"); break;
+        case 4: IEMOP_MNEMONIC(and_Ev_Ib, "and Ev,Ib"); break;
+        case 5: IEMOP_MNEMONIC(sub_Ev_Ib, "sub Ev,Ib"); break;
+        case 6: IEMOP_MNEMONIC(xor_Ev_Ib, "xor Ev,Ib"); break;
+        case 7: IEMOP_MNEMONIC(cmp_Ev_Ib, "cmp Ev,Ib"); break;
+    }
+    /* Note! Seems the OR, AND, and XOR instructions are present on CPUs prior
+             to the 386 even if absent in the intel reference manuals and some
+             3rd party opcode listings. */
+    PCIEMOPBINSIZES pImpl = g_apIemImplGrp1[IEM_GET_MODRM_REG_8(bRm)];
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register target
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                    0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src, /*=*/ (int8_t)u8Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                    0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src, /*=*/ (int8_t)u8Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                if (pImpl != &g_iemAImpl_cmp)   /* TEST won't get here, no need to check for it. */
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                    0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src, /*=*/ (int8_t)u8Imm,1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * Memory target.
+         */
+        uint32_t fAccess;
+        if (pImpl->pfnLockedU16)
+            fAccess = IEM_ACCESS_DATA_RW;
+        else /* CMP */
+            fAccess = IEM_ACCESS_DATA_R;
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                    0);
+                IEM_MC_ARG(uint16_t,        u16Src,                     1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,            2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEM_MC_ASSIGN(u16Src, (int8_t)u8Imm);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                    0);
+                IEM_MC_ARG(uint32_t,        u32Src,                     1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,            2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEM_MC_ASSIGN(u32Src, (int8_t)u8Imm);
+                if (pImpl->pfnLockedU32)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                    0);
+                IEM_MC_ARG(uint64_t,        u64Src,                     1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,            2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+                IEM_MC_ASSIGN(u64Src, (int8_t)u8Imm);
+                if (pImpl->pfnLockedU64)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x84
+ */
+FNIEMOP_DEF(iemOp_test_Eb_Gb)
+{
+    IEMOP_MNEMONIC(test_Eb_Gb, "test Eb,Gb");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_r8, &g_iemAImpl_test);
+}
+
+
+/**
+ * @opcode      0x85
+ */
+FNIEMOP_DEF(iemOp_test_Ev_Gv)
+{
+    IEMOP_MNEMONIC(test_Ev_Gv, "test Ev,Gv");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rm_rv, &g_iemAImpl_test);
+}
+
+
+/**
+ * @opcode      0x86
+ */
+FNIEMOP_DEF(iemOp_xchg_Eb_Gb)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_MNEMONIC(xchg_Eb_Gb, "xchg Eb,Gb");
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint8_t, uTmp1);
+        IEM_MC_LOCAL(uint8_t, uTmp2);
+
+        IEM_MC_FETCH_GREG_U8(uTmp1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_GREG_U8(uTmp2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_RM(pVCpu, bRm),                              uTmp1);
+        IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), uTmp2);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+/** @todo the register must be committed separately! */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint8_t *,  pu8Mem,           0);
+        IEM_MC_ARG(uint8_t *,  pu8Reg,           1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEM_MC_MEM_MAP(pu8Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_REF_GREG_U8(pu8Reg, IEM_GET_MODRM_REG(pVCpu, bRm));
+        if (!pVCpu->iem.s.fDisregardLock)
+            IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u8_locked, pu8Mem, pu8Reg);
+        else
+            IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u8_unlocked, pu8Mem, pu8Reg);
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Mem, IEM_ACCESS_DATA_RW);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x87
+ */
+FNIEMOP_DEF(iemOp_xchg_Ev_Gv)
+{
+    IEMOP_MNEMONIC(xchg_Ev_Gv, "xchg Ev,Gv");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, uTmp1);
+                IEM_MC_LOCAL(uint16_t, uTmp2);
+
+                IEM_MC_FETCH_GREG_U16(uTmp1, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U16(uTmp2, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_RM(pVCpu, bRm),                              uTmp1);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), uTmp2);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, uTmp1);
+                IEM_MC_LOCAL(uint32_t, uTmp2);
+
+                IEM_MC_FETCH_GREG_U32(uTmp1, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U32(uTmp2, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm),                              uTmp1);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), uTmp2);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, uTmp1);
+                IEM_MC_LOCAL(uint64_t, uTmp2);
+
+                IEM_MC_FETCH_GREG_U64(uTmp1, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U64(uTmp2, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm),                              uTmp1);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uTmp2);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+/** @todo the register must be committed separately! */
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(2, 2);
+                IEM_MC_ARG(uint16_t *,  pu16Mem, 0);
+                IEM_MC_ARG(uint16_t *,  pu16Reg, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu16Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_REF_GREG_U16(pu16Reg, IEM_GET_MODRM_REG(pVCpu, bRm));
+                if (!pVCpu->iem.s.fDisregardLock)
+                    IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u16_locked, pu16Mem, pu16Reg);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u16_unlocked, pu16Mem, pu16Reg);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Mem, IEM_ACCESS_DATA_RW);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(2, 2);
+                IEM_MC_ARG(uint32_t *,  pu32Mem, 0);
+                IEM_MC_ARG(uint32_t *,  pu32Reg, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu32Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_REF_GREG_U32(pu32Reg, IEM_GET_MODRM_REG(pVCpu, bRm));
+                if (!pVCpu->iem.s.fDisregardLock)
+                    IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u32_locked, pu32Mem, pu32Reg);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u32_unlocked, pu32Mem, pu32Reg);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Mem, IEM_ACCESS_DATA_RW);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Reg);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(2, 2);
+                IEM_MC_ARG(uint64_t *,  pu64Mem, 0);
+                IEM_MC_ARG(uint64_t *,  pu64Reg, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu64Mem, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_REF_GREG_U64(pu64Reg, IEM_GET_MODRM_REG(pVCpu, bRm));
+                if (!pVCpu->iem.s.fDisregardLock)
+                    IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u64_locked, pu64Mem, pu64Reg);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_xchg_u64_unlocked, pu64Mem, pu64Reg);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Mem, IEM_ACCESS_DATA_RW);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x88
+ */
+FNIEMOP_DEF(iemOp_mov_Eb_Gb)
+{
+    IEMOP_MNEMONIC(mov_Eb_Gb, "mov Eb,Gb");
+
+    uint8_t bRm;
+    IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint8_t, u8Value);
+        IEM_MC_FETCH_GREG_U8(u8Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_RM(pVCpu, bRm), u8Value);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're writing a register to memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint8_t, u8Value);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_GREG_U8(u8Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u8Value);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x89
+ */
+FNIEMOP_DEF(iemOp_mov_Ev_Gv)
+{
+    IEMOP_MNEMONIC(mov_Ev_Gv, "mov Ev,Gv");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_RM(pVCpu, bRm), u16Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're writing a register to memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x8a
+ */
+FNIEMOP_DEF(iemOp_mov_Gb_Eb)
+{
+    IEMOP_MNEMONIC(mov_Gb_Eb, "mov Gb,Eb");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint8_t, u8Value);
+        IEM_MC_FETCH_GREG_U8(u8Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), u8Value);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint8_t, u8Value);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U8(u8Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), u8Value);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x8b
+ */
+FNIEMOP_DEF(iemOp_mov_Gv_Ev)
+{
+    IEMOP_MNEMONIC(mov_Gv_Ev, "mov Gv,Ev");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * opcode      0x63
+ * @todo Table fixme
+ */
+FNIEMOP_DEF(iemOp_arpl_Ew_Gw_movsx_Gv_Ev)
+{
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT)
+        return FNIEMOP_CALL(iemOp_arpl_Ew_Gw);
+    if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT)
+        return FNIEMOP_CALL(iemOp_mov_Gv_Ev);
+    return FNIEMOP_CALL(iemOp_movsxd_Gv_Ev);
+}
+
+
+/**
+ * @opcode      0x8c
+ */
+FNIEMOP_DEF(iemOp_mov_Ev_Sw)
+{
+    IEMOP_MNEMONIC(mov_Ev_Sw, "mov Ev,Sw");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * Check that the destination register exists. The REX.R prefix is ignored.
+     */
+    uint8_t const iSegReg = IEM_GET_MODRM_REG_8(bRm);
+    if (   iSegReg > X86_SREG_GS)
+        return IEMOP_RAISE_INVALID_OPCODE(); /** @todo should probably not be raised until we've fetched all the opcode bytes? */
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     * In that case, the operand size is respected and the upper bits are
+     * cleared (starting with some pentium).
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_FETCH_SREG_U16(u16Value, iSegReg);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_RM(pVCpu, bRm), u16Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_FETCH_SREG_ZX_U32(u32Value, iSegReg);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_FETCH_SREG_ZX_U64(u64Value, iSegReg);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're saving the register to memory.  The access is word sized
+         * regardless of operand size prefixes.
+         */
+#if 0 /* not necessary */
+        pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_16BIT;
+#endif
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint16_t,  u16Value);
+        IEM_MC_LOCAL(RTGCPTR,   GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_SREG_U16(u16Value, iSegReg);
+        IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Value);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+
+
+/**
+ * @opcode      0x8d
+ */
+FNIEMOP_DEF(iemOp_lea_Gv_M)
+{
+    IEMOP_MNEMONIC(lea_Gv_M, "lea Gv,M");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        return IEMOP_RAISE_INVALID_OPCODE(); /* no register form */
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc);
+            IEM_MC_LOCAL(uint16_t, u16Cast);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_ASSIGN_TO_SMALLER(u16Cast, GCPtrEffSrc);
+            IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Cast);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Cast);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_ASSIGN_TO_SMALLER(u32Cast, GCPtrEffSrc);
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Cast);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), GCPtrEffSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x8e
+ */
+FNIEMOP_DEF(iemOp_mov_Sw_Ev)
+{
+    IEMOP_MNEMONIC(mov_Sw_Ev, "mov Sw,Ev");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * The practical operand size is 16-bit.
+     */
+#if 0 /* not necessary */
+    pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_16BIT;
+#endif
+
+    /*
+     * Check that the destination register exists and can be used with this
+     * instruction.  The REX.R prefix is ignored.
+     */
+    uint8_t const iSegReg = IEM_GET_MODRM_REG_8(bRm);
+    if (   iSegReg == X86_SREG_CS
+        || iSegReg > X86_SREG_GS)
+        return IEMOP_RAISE_INVALID_OPCODE(); /** @todo should probably not be raised until we've fetched all the opcode bytes? */
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG_CONST(uint8_t, iSRegArg, iSegReg, 0);
+        IEM_MC_ARG(uint16_t,      u16Value,          1);
+        IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_CIMPL_2(iemCImpl_load_SReg, iSRegArg, u16Value);
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're loading the register from memory.  The access is word sized
+         * regardless of operand size prefixes.
+         */
+        IEM_MC_BEGIN(2, 1);
+        IEM_MC_ARG_CONST(uint8_t, iSRegArg, iSegReg, 0);
+        IEM_MC_ARG(uint16_t,      u16Value,          1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_CALL_CIMPL_2(iemCImpl_load_SReg, iSRegArg, u16Value);
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x8f /0. */
+FNIEMOP_DEF_1(iemOp_pop_Ev, uint8_t, bRm)
+{
+    /* This bugger is rather annoying as it requires rSP to be updated before
+       doing the effective address calculations.  Will eventually require a
+       split between the R/M+SIB decoding and the effective address
+       calculation - which is something that is required for any attempt at
+       reusing this code for a recompiler.  It may also be good to have if we
+       need to delay #UD exception caused by invalid lock prefixes.
+
+       For now, we'll do a mostly safe interpreter-only implementation here. */
+    /** @todo What's the deal with the 'reg' field and pop Ev?  Ignorning it for
+     *        now until tests show it's checked.. */
+    IEMOP_MNEMONIC(pop_Ev, "pop Ev");
+
+    /* Register access is relatively easy and can share code. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        return FNIEMOP_CALL_1(iemOpCommonPopGReg, IEM_GET_MODRM_RM(pVCpu, bRm));
+
+    /*
+     * Memory target.
+     *
+     * Intel says that RSP is incremented before it's used in any effective
+     * address calcuations.  This means some serious extra annoyance here since
+     * we decode and calculate the effective address in one step and like to
+     * delay committing registers till everything is done.
+     *
+     * So, we'll decode and calculate the effective address twice.  This will
+     * require some recoding if turned into a recompiler.
+     */
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE(); /* The common code does this differently. */
+
+#ifndef TST_IEM_CHECK_MC
+    /* Calc effective address with modified ESP. */
+/** @todo testcase */
+    RTGCPTR         GCPtrEff;
+    VBOXSTRICTRC    rcStrict;
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT: rcStrict = iemOpHlpCalcRmEffAddrEx(pVCpu, bRm, 0, &GCPtrEff, 2); break;
+        case IEMMODE_32BIT: rcStrict = iemOpHlpCalcRmEffAddrEx(pVCpu, bRm, 0, &GCPtrEff, 4); break;
+        case IEMMODE_64BIT: rcStrict = iemOpHlpCalcRmEffAddrEx(pVCpu, bRm, 0, &GCPtrEff, 8); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /* Perform the operation - this should be CImpl. */
+    RTUINT64U TmpRsp;
+    TmpRsp.u = pVCpu->cpum.GstCtx.rsp;
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t u16Value;
+            rcStrict = iemMemStackPopU16Ex(pVCpu, &u16Value, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemMemStoreDataU16(pVCpu, pVCpu->iem.s.iEffSeg, GCPtrEff, u16Value);
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Value;
+            rcStrict = iemMemStackPopU32Ex(pVCpu, &u32Value, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemMemStoreDataU32(pVCpu, pVCpu->iem.s.iEffSeg, GCPtrEff, u32Value);
+            break;
+        }
+
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Value;
+            rcStrict = iemMemStackPopU64Ex(pVCpu, &u64Value, &TmpRsp);
+            if (rcStrict == VINF_SUCCESS)
+                rcStrict = iemMemStoreDataU64(pVCpu, pVCpu->iem.s.iEffSeg, GCPtrEff, u64Value);
+            break;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    if (rcStrict == VINF_SUCCESS)
+    {
+        pVCpu->cpum.GstCtx.rsp = TmpRsp.u;
+        return iemRegUpdateRipAndFinishClearingRF(pVCpu);
+    }
+    return rcStrict;
+
+#else
+    return VERR_IEM_IPE_2;
+#endif
+}
+
+
+/**
+ * @opcode      0x8f
+ */
+FNIEMOP_DEF(iemOp_Grp1A__xop)
+{
+    /*
+     * AMD has defined /1 thru /7 as XOP prefix.  The prefix is similar to the
+     * three byte VEX prefix, except that the mmmmm field cannot have the values
+     * 0 thru 7, because it would then be confused with pop Ev (modrm.reg == 0).
+     */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_REG_MASK) == (0 << X86_MODRM_REG_SHIFT)) /* /0 */
+        return FNIEMOP_CALL_1(iemOp_pop_Ev, bRm);
+
+    IEMOP_MNEMONIC(xop, "xop");
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXop)
+    {
+        /** @todo Test when exctly the XOP conformance checks kick in during
+         * instruction decoding and fetching (using \#PF). */
+        uint8_t bXop2;   IEM_OPCODE_GET_NEXT_U8(&bXop2);
+        uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode);
+        if (   (  pVCpu->iem.s.fPrefixes
+                & (IEM_OP_PRF_SIZE_OP | IEM_OP_PRF_REPZ | IEM_OP_PRF_REPNZ | IEM_OP_PRF_LOCK | IEM_OP_PRF_REX))
+            == 0)
+        {
+            pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_XOP;
+            if ((bXop2 & 0x80 /* XOP.W */) && pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+                pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_W;
+            pVCpu->iem.s.uRexReg    = (~bRm >> (7 - 3)) & 0x8;
+            pVCpu->iem.s.uRexIndex  = (~bRm >> (6 - 3)) & 0x8;
+            pVCpu->iem.s.uRexB      = (~bRm >> (5 - 3)) & 0x8;
+            pVCpu->iem.s.uVex3rdReg = (~bXop2 >> 3) & 0xf;
+            pVCpu->iem.s.uVexLength = (bXop2 >> 2) & 1;
+            pVCpu->iem.s.idxPrefix  = bXop2 & 0x3;
+
+            /** @todo XOP: Just use new tables and decoders. */
+            switch (bRm & 0x1f)
+            {
+                case 8: /* xop opcode map 8. */
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+
+                case 9: /* xop opcode map 9. */
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+
+                case 10: /* xop opcode map 10. */
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+
+                default:
+                    Log(("XOP: Invalid vvvv value: %#x!\n", bRm & 0x1f));
+                    return IEMOP_RAISE_INVALID_OPCODE();
+            }
+        }
+        else
+            Log(("XOP: Invalid prefix mix!\n"));
+    }
+    else
+        Log(("XOP: XOP support disabled!\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * Common 'xchg reg,rAX' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonXchgGRegRax, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    iReg |= pVCpu->iem.s.uRexB;
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint16_t, u16Tmp1);
+            IEM_MC_LOCAL(uint16_t, u16Tmp2);
+            IEM_MC_FETCH_GREG_U16(u16Tmp1, iReg);
+            IEM_MC_FETCH_GREG_U16(u16Tmp2, X86_GREG_xAX);
+            IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Tmp1);
+            IEM_MC_STORE_GREG_U16(iReg,         u16Tmp2);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint32_t, u32Tmp1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp2);
+            IEM_MC_FETCH_GREG_U32(u32Tmp1, iReg);
+            IEM_MC_FETCH_GREG_U32(u32Tmp2, X86_GREG_xAX);
+            IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u32Tmp1);
+            IEM_MC_STORE_GREG_U32(iReg,         u32Tmp2);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t, u64Tmp1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp2);
+            IEM_MC_FETCH_GREG_U64(u64Tmp1, iReg);
+            IEM_MC_FETCH_GREG_U64(u64Tmp2, X86_GREG_xAX);
+            IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u64Tmp1);
+            IEM_MC_STORE_GREG_U64(iReg,         u64Tmp2);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x90
+ */
+FNIEMOP_DEF(iemOp_nop)
+{
+    /* R8/R8D and RAX/EAX can be exchanged. */
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX_B)
+    {
+        IEMOP_MNEMONIC(xchg_r8_rAX, "xchg r8,rAX");
+        return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xAX);
+    }
+
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)
+    {
+        IEMOP_MNEMONIC(pause, "pause");
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fVmx)
+            return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmx_pause);
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSvm)
+            return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_svm_pause);
+#endif
+    }
+    else
+        IEMOP_MNEMONIC(nop, "nop");
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x91
+ */
+FNIEMOP_DEF(iemOp_xchg_eCX_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rCX_rAX, "xchg rCX,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xCX);
+}
+
+
+/**
+ * @opcode      0x92
+ */
+FNIEMOP_DEF(iemOp_xchg_eDX_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rDX_rAX, "xchg rDX,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xDX);
+}
+
+
+/**
+ * @opcode      0x93
+ */
+FNIEMOP_DEF(iemOp_xchg_eBX_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rBX_rAX, "xchg rBX,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xBX);
+}
+
+
+/**
+ * @opcode      0x94
+ */
+FNIEMOP_DEF(iemOp_xchg_eSP_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rSX_rAX, "xchg rSX,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xSP);
+}
+
+
+/**
+ * @opcode      0x95
+ */
+FNIEMOP_DEF(iemOp_xchg_eBP_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rBP_rAX, "xchg rBP,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xBP);
+}
+
+
+/**
+ * @opcode      0x96
+ */
+FNIEMOP_DEF(iemOp_xchg_eSI_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rSI_rAX, "xchg rSI,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xSI);
+}
+
+
+/**
+ * @opcode      0x97
+ */
+FNIEMOP_DEF(iemOp_xchg_eDI_eAX)
+{
+    IEMOP_MNEMONIC(xchg_rDI_rAX, "xchg rDI,rAX");
+    return FNIEMOP_CALL_1(iemOpCommonXchgGRegRax, X86_GREG_xDI);
+}
+
+
+/**
+ * @opcode      0x98
+ */
+FNIEMOP_DEF(iemOp_cbw)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEMOP_MNEMONIC(cbw, "cbw");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 7) {
+                IEM_MC_OR_GREG_U16(X86_GREG_xAX, UINT16_C(0xff00));
+            } IEM_MC_ELSE() {
+                IEM_MC_AND_GREG_U16(X86_GREG_xAX, UINT16_C(0x00ff));
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEMOP_MNEMONIC(cwde, "cwde");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 15) {
+                IEM_MC_OR_GREG_U32(X86_GREG_xAX, UINT32_C(0xffff0000));
+            } IEM_MC_ELSE() {
+                IEM_MC_AND_GREG_U32(X86_GREG_xAX, UINT32_C(0x0000ffff));
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEMOP_MNEMONIC(cdqe, "cdqe");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 31) {
+                IEM_MC_OR_GREG_U64(X86_GREG_xAX, UINT64_C(0xffffffff00000000));
+            } IEM_MC_ELSE() {
+                IEM_MC_AND_GREG_U64(X86_GREG_xAX, UINT64_C(0x00000000ffffffff));
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x99
+ */
+FNIEMOP_DEF(iemOp_cwd)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEMOP_MNEMONIC(cwd, "cwd");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 15) {
+                IEM_MC_STORE_GREG_U16_CONST(X86_GREG_xDX, UINT16_C(0xffff));
+            } IEM_MC_ELSE() {
+                IEM_MC_STORE_GREG_U16_CONST(X86_GREG_xDX, 0);
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEMOP_MNEMONIC(cdq, "cdq");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 31) {
+                IEM_MC_STORE_GREG_U32_CONST(X86_GREG_xDX, UINT32_C(0xffffffff));
+            } IEM_MC_ELSE() {
+                IEM_MC_STORE_GREG_U32_CONST(X86_GREG_xDX, 0);
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEMOP_MNEMONIC(cqo, "cqo");
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_IF_GREG_BIT_SET(X86_GREG_xAX, 63) {
+                IEM_MC_STORE_GREG_U64_CONST(X86_GREG_xDX, UINT64_C(0xffffffffffffffff));
+            } IEM_MC_ELSE() {
+                IEM_MC_STORE_GREG_U64_CONST(X86_GREG_xDX, 0);
+            } IEM_MC_ENDIF();
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0x9a
+ */
+FNIEMOP_DEF(iemOp_call_Ap)
+{
+    IEMOP_MNEMONIC(call_Ap, "call Ap");
+    IEMOP_HLP_NO_64BIT();
+
+    /* Decode the far pointer address and pass it on to the far call C implementation. */
+    uint32_t offSeg;
+    if (pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT)
+        IEM_OPCODE_GET_NEXT_U32(&offSeg);
+    else
+        IEM_OPCODE_GET_NEXT_U16_ZX_U32(&offSeg);
+    uint16_t uSel;  IEM_OPCODE_GET_NEXT_U16(&uSel);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_callf, uSel, offSeg, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/** Opcode 0x9b. (aka fwait) */
+FNIEMOP_DEF(iemOp_wait)
+{
+    IEMOP_MNEMONIC(wait, "wait");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_MAYBE_RAISE_WAIT_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x9c
+ */
+FNIEMOP_DEF(iemOp_pushf_Fv)
+{
+    IEMOP_MNEMONIC(pushf_Fv, "pushf Fv");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_pushf, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0x9d
+ */
+FNIEMOP_DEF(iemOp_popf_Fv)
+{
+    IEMOP_MNEMONIC(popf_Fv, "popf Fv");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_popf, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0x9e
+ */
+FNIEMOP_DEF(iemOp_sahf)
+{
+    IEMOP_MNEMONIC(sahf, "sahf");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (   pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+        && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLahfSahf)
+        return IEMOP_RAISE_INVALID_OPCODE();
+    IEM_MC_BEGIN(0, 2);
+    IEM_MC_LOCAL(uint32_t, u32Flags);
+    IEM_MC_LOCAL(uint32_t, EFlags);
+    IEM_MC_FETCH_EFLAGS(EFlags);
+    IEM_MC_FETCH_GREG_U8_ZX_U32(u32Flags, X86_GREG_xSP/*=AH*/);
+    IEM_MC_AND_LOCAL_U32(u32Flags, X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF);
+    IEM_MC_AND_LOCAL_U32(EFlags, UINT32_C(0xffffff00));
+    IEM_MC_OR_LOCAL_U32(u32Flags, X86_EFL_1);
+    IEM_MC_OR_2LOCS_U32(EFlags, u32Flags);
+    IEM_MC_COMMIT_EFLAGS(EFlags);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0x9f
+ */
+FNIEMOP_DEF(iemOp_lahf)
+{
+    IEMOP_MNEMONIC(lahf, "lahf");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (   pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+        && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLahfSahf)
+        return IEMOP_RAISE_INVALID_OPCODE();
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(uint8_t, u8Flags);
+    IEM_MC_FETCH_EFLAGS_U8(u8Flags);
+    IEM_MC_STORE_GREG_U8(X86_GREG_xSP/*=AH*/, u8Flags);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * Macro used by iemOp_mov_AL_Ob, iemOp_mov_rAX_Ov, iemOp_mov_Ob_AL and
+ * iemOp_mov_Ov_rAX to fetch the moffsXX bit of the opcode and fend off lock
+ * prefixes.  Will return on failures.
+ * @param   a_GCPtrMemOff   The variable to store the offset in.
+ */
+#define IEMOP_FETCH_MOFFS_XX(a_GCPtrMemOff) \
+    do \
+    { \
+        switch (pVCpu->iem.s.enmEffAddrMode) \
+        { \
+            case IEMMODE_16BIT: \
+                IEM_OPCODE_GET_NEXT_U16_ZX_U64(&(a_GCPtrMemOff)); \
+                break; \
+            case IEMMODE_32BIT: \
+                IEM_OPCODE_GET_NEXT_U32_ZX_U64(&(a_GCPtrMemOff)); \
+                break; \
+            case IEMMODE_64BIT: \
+                IEM_OPCODE_GET_NEXT_U64(&(a_GCPtrMemOff)); \
+                break; \
+            IEM_NOT_REACHED_DEFAULT_CASE_RET(); \
+        } \
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \
+    } while (0)
+
+/**
+ * @opcode      0xa0
+ */
+FNIEMOP_DEF(iemOp_mov_AL_Ob)
+{
+    /*
+     * Get the offset and fend off lock prefixes.
+     */
+    IEMOP_MNEMONIC(mov_AL_Ob, "mov AL,Ob");
+    RTGCPTR GCPtrMemOff;
+    IEMOP_FETCH_MOFFS_XX(GCPtrMemOff);
+
+    /*
+     * Fetch AL.
+     */
+    IEM_MC_BEGIN(0,1);
+    IEM_MC_LOCAL(uint8_t, u8Tmp);
+    IEM_MC_FETCH_MEM_U8(u8Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff);
+    IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0xa1
+ */
+FNIEMOP_DEF(iemOp_mov_rAX_Ov)
+{
+    /*
+     * Get the offset and fend off lock prefixes.
+     */
+    IEMOP_MNEMONIC(mov_rAX_Ov, "mov rAX,Ov");
+    RTGCPTR GCPtrMemOff;
+    IEMOP_FETCH_MOFFS_XX(GCPtrMemOff);
+
+    /*
+     * Fetch rAX.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint16_t, u16Tmp);
+            IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff);
+            IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Tmp);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+            IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff);
+            IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u32Tmp);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+            IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrMemOff);
+            IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u64Tmp);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xa2
+ */
+FNIEMOP_DEF(iemOp_mov_Ob_AL)
+{
+    /*
+     * Get the offset and fend off lock prefixes.
+     */
+    IEMOP_MNEMONIC(mov_Ob_AL, "mov Ob,AL");
+    RTGCPTR GCPtrMemOff;
+    IEMOP_FETCH_MOFFS_XX(GCPtrMemOff);
+
+    /*
+     * Store AL.
+     */
+    IEM_MC_BEGIN(0,1);
+    IEM_MC_LOCAL(uint8_t, u8Tmp);
+    IEM_MC_FETCH_GREG_U8(u8Tmp, X86_GREG_xAX);
+    IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u8Tmp);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0xa3
+ */
+FNIEMOP_DEF(iemOp_mov_Ov_rAX)
+{
+    /*
+     * Get the offset and fend off lock prefixes.
+     */
+    IEMOP_MNEMONIC(mov_Ov_rAX, "mov Ov,rAX");
+    RTGCPTR GCPtrMemOff;
+    IEMOP_FETCH_MOFFS_XX(GCPtrMemOff);
+
+    /*
+     * Store rAX.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint16_t, u16Tmp);
+            IEM_MC_FETCH_GREG_U16(u16Tmp, X86_GREG_xAX);
+            IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u16Tmp);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+            IEM_MC_FETCH_GREG_U32(u32Tmp, X86_GREG_xAX);
+            IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u32Tmp);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+            IEM_MC_FETCH_GREG_U64(u64Tmp, X86_GREG_xAX);
+            IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrMemOff, u64Tmp);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+/** Macro used by iemOp_movsb_Xb_Yb and iemOp_movswd_Xv_Yv */
+#define IEM_MOVS_CASE(ValBits, AddrBits) \
+        IEM_MC_BEGIN(0, 2); \
+        IEM_MC_LOCAL(uint##ValBits##_t, uValue); \
+        IEM_MC_LOCAL(RTGCPTR,           uAddr); \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xSI); \
+        IEM_MC_FETCH_MEM_U##ValBits(uValue, pVCpu->iem.s.iEffSeg, uAddr); \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xDI); \
+        IEM_MC_STORE_MEM_U##ValBits(X86_SREG_ES, uAddr, uValue); \
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ELSE() { \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ENDIF(); \
+        IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+        IEM_MC_END()
+
+/**
+ * @opcode      0xa4
+ */
+FNIEMOP_DEF(iemOp_movsb_Xb_Yb)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_movsb_Xb_Yb, "rep movsb Xb,Yb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op8_addr16, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op8_addr32, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op8_addr64, pVCpu->iem.s.iEffSeg);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(movsb_Xb_Yb, "movsb Xb,Yb");
+
+    /*
+     * Sharing case implementation with movs[wdq] below.
+     */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT: IEM_MOVS_CASE(8, 16); break;
+        case IEMMODE_32BIT: IEM_MOVS_CASE(8, 32); break;
+        case IEMMODE_64BIT: IEM_MOVS_CASE(8, 64); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xa5
+ */
+FNIEMOP_DEF(iemOp_movswd_Xv_Yv)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_movs_Xv_Yv, "rep movs Xv,Yv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op16_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op16_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op16_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op32_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op32_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op32_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_6);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op64_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_rep_movs_op64_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(movs_Xv_Yv, "movs Xv,Yv");
+
+    /*
+     * Annoying double switch here.
+     * Using ugly macro for implementing the cases, sharing it with movsb.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_MOVS_CASE(16, 16); break;
+                case IEMMODE_32BIT: IEM_MOVS_CASE(16, 32); break;
+                case IEMMODE_64BIT: IEM_MOVS_CASE(16, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_32BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_MOVS_CASE(32, 16); break;
+                case IEMMODE_32BIT: IEM_MOVS_CASE(32, 32); break;
+                case IEMMODE_64BIT: IEM_MOVS_CASE(32, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break;
+                case IEMMODE_32BIT: IEM_MOVS_CASE(64, 32); break;
+                case IEMMODE_64BIT: IEM_MOVS_CASE(64, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+#undef IEM_MOVS_CASE
+
+/** Macro used by iemOp_cmpsb_Xb_Yb and iemOp_cmpswd_Xv_Yv */
+#define IEM_CMPS_CASE(ValBits, AddrBits) \
+        IEM_MC_BEGIN(3, 3); \
+        IEM_MC_ARG(uint##ValBits##_t *, puValue1, 0); \
+        IEM_MC_ARG(uint##ValBits##_t,   uValue2,  1); \
+        IEM_MC_ARG(uint32_t *,          pEFlags,  2); \
+        IEM_MC_LOCAL(uint##ValBits##_t, uValue1); \
+        IEM_MC_LOCAL(RTGCPTR,           uAddr); \
+        \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xSI); \
+        IEM_MC_FETCH_MEM_U##ValBits(uValue1, pVCpu->iem.s.iEffSeg, uAddr); \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xDI); \
+        IEM_MC_FETCH_MEM_U##ValBits(uValue2, X86_SREG_ES, uAddr); \
+        IEM_MC_REF_LOCAL(puValue1, uValue1); \
+        IEM_MC_REF_EFLAGS(pEFlags); \
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cmp_u##ValBits, puValue1, uValue2, pEFlags); \
+        \
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ELSE() { \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ENDIF(); \
+        IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+        IEM_MC_END()
+
+/**
+ * @opcode      0xa6
+ */
+FNIEMOP_DEF(iemOp_cmpsb_Xb_Yb)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ)
+    {
+        IEMOP_MNEMONIC(repz_cmps_Xb_Yb, "repz cmps Xb,Yb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op8_addr16, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op8_addr32, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op8_addr64, pVCpu->iem.s.iEffSeg);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ)
+    {
+        IEMOP_MNEMONIC(repnz_cmps_Xb_Yb, "repnz cmps Xb,Yb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op8_addr16, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op8_addr32, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op8_addr64, pVCpu->iem.s.iEffSeg);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(cmps_Xb_Yb, "cmps Xb,Yb");
+
+    /*
+     * Sharing case implementation with cmps[wdq] below.
+     */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT: IEM_CMPS_CASE(8, 16); break;
+        case IEMMODE_32BIT: IEM_CMPS_CASE(8, 32); break;
+        case IEMMODE_64BIT: IEM_CMPS_CASE(8, 64); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xa7
+ */
+FNIEMOP_DEF(iemOp_cmpswd_Xv_Yv)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ)
+    {
+        IEMOP_MNEMONIC(repe_cmps_Xv_Yv, "repe cmps Xv,Yv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op16_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op16_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op16_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op32_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op32_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op32_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_4);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op64_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repe_cmps_op64_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ)
+    {
+        IEMOP_MNEMONIC(repne_cmps_Xv_Yv, "repne cmps Xv,Yv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op16_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op16_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op16_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op32_addr16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op32_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op32_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_2);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op64_addr32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_repne_cmps_op64_addr64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+
+    IEMOP_MNEMONIC(cmps_Xv_Yv, "cmps Xv,Yv");
+
+    /*
+     * Annoying double switch here.
+     * Using ugly macro for implementing the cases, sharing it with cmpsb.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_CMPS_CASE(16, 16); break;
+                case IEMMODE_32BIT: IEM_CMPS_CASE(16, 32); break;
+                case IEMMODE_64BIT: IEM_CMPS_CASE(16, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_32BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_CMPS_CASE(32, 16); break;
+                case IEMMODE_32BIT: IEM_CMPS_CASE(32, 32); break;
+                case IEMMODE_64BIT: IEM_CMPS_CASE(32, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break;
+                case IEMMODE_32BIT: IEM_CMPS_CASE(64, 32); break;
+                case IEMMODE_64BIT: IEM_CMPS_CASE(64, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+#undef IEM_CMPS_CASE
+
+/**
+ * @opcode      0xa8
+ */
+FNIEMOP_DEF(iemOp_test_AL_Ib)
+{
+    IEMOP_MNEMONIC(test_al_Ib, "test al,Ib");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_AL_Ib, &g_iemAImpl_test);
+}
+
+
+/**
+ * @opcode      0xa9
+ */
+FNIEMOP_DEF(iemOp_test_eAX_Iz)
+{
+    IEMOP_MNEMONIC(test_rAX_Iz, "test rAX,Iz");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rAX_Iz, &g_iemAImpl_test);
+}
+
+
+/** Macro used by iemOp_stosb_Yb_AL and iemOp_stoswd_Yv_eAX */
+#define IEM_STOS_CASE(ValBits, AddrBits) \
+        IEM_MC_BEGIN(0, 2); \
+        IEM_MC_LOCAL(uint##ValBits##_t, uValue); \
+        IEM_MC_LOCAL(RTGCPTR, uAddr); \
+        IEM_MC_FETCH_GREG_U##ValBits(uValue, X86_GREG_xAX); \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr,  X86_GREG_xDI); \
+        IEM_MC_STORE_MEM_U##ValBits(X86_SREG_ES, uAddr, uValue); \
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+        } IEM_MC_ELSE() { \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+        } IEM_MC_ENDIF(); \
+        IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+        IEM_MC_END()
+
+/**
+ * @opcode      0xaa
+ */
+FNIEMOP_DEF(iemOp_stosb_Yb_AL)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_stos_Yb_al, "rep stos Yb,al");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_al_m16);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_al_m32);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_al_m64);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(stos_Yb_al, "stos Yb,al");
+
+    /*
+     * Sharing case implementation with stos[wdq] below.
+     */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT: IEM_STOS_CASE(8, 16); break;
+        case IEMMODE_32BIT: IEM_STOS_CASE(8, 32); break;
+        case IEMMODE_64BIT: IEM_STOS_CASE(8, 64); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xab
+ */
+FNIEMOP_DEF(iemOp_stoswd_Yv_eAX)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_stos_Yv_rAX, "rep stos Yv,rAX");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_ax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_ax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_ax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_eax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_eax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_eax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_9);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_rax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stos_rax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(stos_Yv_rAX, "stos Yv,rAX");
+
+    /*
+     * Annoying double switch here.
+     * Using ugly macro for implementing the cases, sharing it with stosb.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_STOS_CASE(16, 16); break;
+                case IEMMODE_32BIT: IEM_STOS_CASE(16, 32); break;
+                case IEMMODE_64BIT: IEM_STOS_CASE(16, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_32BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_STOS_CASE(32, 16); break;
+                case IEMMODE_32BIT: IEM_STOS_CASE(32, 32); break;
+                case IEMMODE_64BIT: IEM_STOS_CASE(32, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break;
+                case IEMMODE_32BIT: IEM_STOS_CASE(64, 32); break;
+                case IEMMODE_64BIT: IEM_STOS_CASE(64, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+#undef IEM_STOS_CASE
+
+/** Macro used by iemOp_lodsb_AL_Xb and iemOp_lodswd_eAX_Xv */
+#define IEM_LODS_CASE(ValBits, AddrBits) \
+        IEM_MC_BEGIN(0, 2); \
+        IEM_MC_LOCAL(uint##ValBits##_t, uValue); \
+        IEM_MC_LOCAL(RTGCPTR, uAddr); \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xSI); \
+        IEM_MC_FETCH_MEM_U##ValBits(uValue, pVCpu->iem.s.iEffSeg, uAddr); \
+        IEM_MC_STORE_GREG_U##ValBits(X86_GREG_xAX, uValue); \
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ELSE() { \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xSI, ValBits / 8); \
+        } IEM_MC_ENDIF(); \
+        IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+        IEM_MC_END()
+
+/**
+ * @opcode      0xac
+ */
+FNIEMOP_DEF(iemOp_lodsb_AL_Xb)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_lodsb_AL_Xb, "rep lodsb AL,Xb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_al_m16, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_al_m32, pVCpu->iem.s.iEffSeg);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_al_m64, pVCpu->iem.s.iEffSeg);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(lodsb_AL_Xb, "lodsb AL,Xb");
+
+    /*
+     * Sharing case implementation with stos[wdq] below.
+     */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT: IEM_LODS_CASE(8, 16); break;
+        case IEMMODE_32BIT: IEM_LODS_CASE(8, 32); break;
+        case IEMMODE_64BIT: IEM_LODS_CASE(8, 64); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xad
+ */
+FNIEMOP_DEF(iemOp_lodswd_eAX_Xv)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & (IEM_OP_PRF_REPNZ | IEM_OP_PRF_REPZ))
+    {
+        IEMOP_MNEMONIC(rep_lods_rAX_Xv, "rep lods rAX,Xv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_ax_m16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_ax_m32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_ax_m64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_eax_m16, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_eax_m32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_eax_m64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_7);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_rax_m32, pVCpu->iem.s.iEffSeg);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_lods_rax_m64, pVCpu->iem.s.iEffSeg);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(lods_rAX_Xv, "lods rAX,Xv");
+
+    /*
+     * Annoying double switch here.
+     * Using ugly macro for implementing the cases, sharing it with lodsb.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_LODS_CASE(16, 16); break;
+                case IEMMODE_32BIT: IEM_LODS_CASE(16, 32); break;
+                case IEMMODE_64BIT: IEM_LODS_CASE(16, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_32BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_LODS_CASE(32, 16); break;
+                case IEMMODE_32BIT: IEM_LODS_CASE(32, 32); break;
+                case IEMMODE_64BIT: IEM_LODS_CASE(32, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break;
+                case IEMMODE_32BIT: IEM_LODS_CASE(64, 32); break;
+                case IEMMODE_64BIT: IEM_LODS_CASE(64, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+#undef IEM_LODS_CASE
+
+/** Macro used by iemOp_scasb_AL_Xb and iemOp_scaswd_eAX_Xv */
+#define IEM_SCAS_CASE(ValBits, AddrBits) \
+        IEM_MC_BEGIN(3, 2); \
+        IEM_MC_ARG(uint##ValBits##_t *, puRax,   0); \
+        IEM_MC_ARG(uint##ValBits##_t,   uValue,  1); \
+        IEM_MC_ARG(uint32_t *,          pEFlags, 2); \
+        IEM_MC_LOCAL(RTGCPTR,           uAddr); \
+        \
+        IEM_MC_FETCH_GREG_U##AddrBits##_ZX_U64(uAddr, X86_GREG_xDI); \
+        IEM_MC_FETCH_MEM_U##ValBits(uValue, X86_SREG_ES, uAddr); \
+        IEM_MC_REF_GREG_U##ValBits(puRax, X86_GREG_xAX); \
+        IEM_MC_REF_EFLAGS(pEFlags); \
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cmp_u##ValBits, puRax, uValue, pEFlags); \
+        \
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_DF) { \
+            IEM_MC_SUB_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+        } IEM_MC_ELSE() { \
+            IEM_MC_ADD_GREG_U##AddrBits(X86_GREG_xDI, ValBits / 8); \
+        } IEM_MC_ENDIF(); \
+        IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+        IEM_MC_END()
+
+/**
+ * @opcode      0xae
+ */
+FNIEMOP_DEF(iemOp_scasb_AL_Xb)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ)
+    {
+        IEMOP_MNEMONIC(repe_scasb_AL_Xb, "repe scasb AL,Xb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_al_m16);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_al_m32);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_al_m64);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ)
+    {
+        IEMOP_MNEMONIC(repone_scasb_AL_Xb, "repne scasb AL,Xb");
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_al_m16);
+            case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_al_m32);
+            case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_al_m64);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(scasb_AL_Xb, "scasb AL,Xb");
+
+    /*
+     * Sharing case implementation with stos[wdq] below.
+     */
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT: IEM_SCAS_CASE(8, 16); break;
+        case IEMMODE_32BIT: IEM_SCAS_CASE(8, 32); break;
+        case IEMMODE_64BIT: IEM_SCAS_CASE(8, 64); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xaf
+ */
+FNIEMOP_DEF(iemOp_scaswd_eAX_Xv)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /*
+     * Use the C implementation if a repeat prefix is encountered.
+     */
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPZ)
+    {
+        IEMOP_MNEMONIC(repe_scas_rAX_Xv, "repe scas rAX,Xv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_ax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_ax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_ax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_eax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_eax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_eax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_6); /** @todo It's this wrong, we can do 16-bit addressing in 64-bit mode, but not 32-bit. right? */
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_rax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repe_scas_rax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REPNZ)
+    {
+        IEMOP_MNEMONIC(repne_scas_rAX_Xv, "repne scas rAX,Xv");
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_ax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_ax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_ax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case IEMMODE_32BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_eax_m16);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_eax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_eax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            case IEMMODE_64BIT:
+                switch (pVCpu->iem.s.enmEffAddrMode)
+                {
+                    case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_5);
+                    case IEMMODE_32BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_rax_m32);
+                    case IEMMODE_64BIT: return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_repne_scas_rax_m64);
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    IEMOP_MNEMONIC(scas_rAX_Xv, "scas rAX,Xv");
+
+    /*
+     * Annoying double switch here.
+     * Using ugly macro for implementing the cases, sharing it with scasb.
+     */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_SCAS_CASE(16, 16); break;
+                case IEMMODE_32BIT: IEM_SCAS_CASE(16, 32); break;
+                case IEMMODE_64BIT: IEM_SCAS_CASE(16, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_32BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: IEM_SCAS_CASE(32, 16); break;
+                case IEMMODE_32BIT: IEM_SCAS_CASE(32, 32); break;
+                case IEMMODE_64BIT: IEM_SCAS_CASE(32, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+
+        case IEMMODE_64BIT:
+            switch (pVCpu->iem.s.enmEffAddrMode)
+            {
+                case IEMMODE_16BIT: AssertFailedReturn(VERR_IEM_IPE_1); /* cannot be encoded */ break;
+                case IEMMODE_32BIT: IEM_SCAS_CASE(64, 32); break;
+                case IEMMODE_64BIT: IEM_SCAS_CASE(64, 64); break;
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+            break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+#undef IEM_SCAS_CASE
+
+/**
+ * Common 'mov r8, imm8' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonMov_r8_Ib, uint8_t, iReg)
+{
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL_CONST(uint8_t, u8Value,/*=*/ u8Imm);
+    IEM_MC_STORE_GREG_U8(iReg, u8Value);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0xb0
+ */
+FNIEMOP_DEF(iemOp_mov_AL_Ib)
+{
+    IEMOP_MNEMONIC(mov_AL_Ib, "mov AL,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xAX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb1
+ */
+FNIEMOP_DEF(iemOp_CL_Ib)
+{
+    IEMOP_MNEMONIC(mov_CL_Ib, "mov CL,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xCX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb2
+ */
+FNIEMOP_DEF(iemOp_DL_Ib)
+{
+    IEMOP_MNEMONIC(mov_DL_Ib, "mov DL,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xDX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb3
+ */
+FNIEMOP_DEF(iemOp_BL_Ib)
+{
+    IEMOP_MNEMONIC(mov_BL_Ib, "mov BL,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xBX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb4
+ */
+FNIEMOP_DEF(iemOp_mov_AH_Ib)
+{
+    IEMOP_MNEMONIC(mov_AH_Ib, "mov AH,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xSP | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb5
+ */
+FNIEMOP_DEF(iemOp_CH_Ib)
+{
+    IEMOP_MNEMONIC(mov_CH_Ib, "mov CH,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xBP | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb6
+ */
+FNIEMOP_DEF(iemOp_DH_Ib)
+{
+    IEMOP_MNEMONIC(mov_DH_Ib, "mov DH,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xSI | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb7
+ */
+FNIEMOP_DEF(iemOp_BH_Ib)
+{
+    IEMOP_MNEMONIC(mov_BH_Ib, "mov BH,Ib");
+    return FNIEMOP_CALL_1(iemOpCommonMov_r8_Ib, X86_GREG_xDI | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * Common 'mov regX,immX' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonMov_Rv_Iv, uint8_t, iReg)
+{
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL_CONST(uint16_t, u16Value,/*=*/ u16Imm);
+            IEM_MC_STORE_GREG_U16(iReg, u16Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL_CONST(uint32_t, u32Value,/*=*/ u32Imm);
+            IEM_MC_STORE_GREG_U32(iReg, u32Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+        }
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Imm; IEM_OPCODE_GET_NEXT_U64(&u64Imm); /* 64-bit immediate! */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL_CONST(uint64_t, u64Value,/*=*/ u64Imm);
+            IEM_MC_STORE_GREG_U64(iReg, u64Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+        }
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xb8
+ */
+FNIEMOP_DEF(iemOp_eAX_Iv)
+{
+    IEMOP_MNEMONIC(mov_rAX_IV, "mov rAX,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xAX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xb9
+ */
+FNIEMOP_DEF(iemOp_eCX_Iv)
+{
+    IEMOP_MNEMONIC(mov_rCX_IV, "mov rCX,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xCX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xba
+ */
+FNIEMOP_DEF(iemOp_eDX_Iv)
+{
+    IEMOP_MNEMONIC(mov_rDX_IV, "mov rDX,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xDX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xbb
+ */
+FNIEMOP_DEF(iemOp_eBX_Iv)
+{
+    IEMOP_MNEMONIC(mov_rBX_IV, "mov rBX,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xBX | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xbc
+ */
+FNIEMOP_DEF(iemOp_eSP_Iv)
+{
+    IEMOP_MNEMONIC(mov_rSP_IV, "mov rSP,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xSP | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xbd
+ */
+FNIEMOP_DEF(iemOp_eBP_Iv)
+{
+    IEMOP_MNEMONIC(mov_rBP_IV, "mov rBP,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xBP | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xbe
+ */
+FNIEMOP_DEF(iemOp_eSI_Iv)
+{
+    IEMOP_MNEMONIC(mov_rSI_IV, "mov rSI,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xSI | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xbf
+ */
+FNIEMOP_DEF(iemOp_eDI_Iv)
+{
+    IEMOP_MNEMONIC(mov_rDI_IV, "mov rDI,IV");
+    return FNIEMOP_CALL_1(iemOpCommonMov_Rv_Iv, X86_GREG_xDI | pVCpu->iem.s.uRexB);
+}
+
+
+/**
+ * @opcode      0xc0
+ */
+FNIEMOP_DEF(iemOp_Grp2_Eb_Ib)
+{
+    IEMOP_HLP_MIN_186();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Eb_Ib, "rol Eb,Ib"); break;
+        case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Eb_Ib, "ror Eb,Ib"); break;
+        case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Eb_Ib, "rcl Eb,Ib"); break;
+        case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Eb_Ib, "rcr Eb,Ib"); break;
+        case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Eb_Ib, "shl Eb,Ib"); break;
+        case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Eb_Ib, "shr Eb,Ib"); break;
+        case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Eb_Ib, "sar Eb,Ib"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe stupid */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register */
+        uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,            0);
+        IEM_MC_ARG_CONST(uint8_t,   cShiftArg, cShift, 1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,   pu8Dst,    0);
+        IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+        uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+        IEM_MC_ASSIGN(cShiftArg, cShift);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0xc1
+ */
+FNIEMOP_DEF(iemOp_Grp2_Ev_Ib)
+{
+    IEMOP_HLP_MIN_186();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Ev_Ib, "rol Ev,Ib"); break;
+        case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Ev_Ib, "ror Ev,Ib"); break;
+        case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Ev_Ib, "rcl Ev,Ib"); break;
+        case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Ev_Ib, "rcr Ev,Ib"); break;
+        case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Ev_Ib, "shl Ev,Ib"); break;
+        case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Ev_Ib, "shr Ev,Ib"); break;
+        case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Ev_Ib, "sar Ev,Ib"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe stupid */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register */
+        uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, cShift, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, cShift, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, cShift, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,  pu16Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,  pu32Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,  pu64Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0xc2
+ */
+FNIEMOP_DEF(iemOp_retn_Iw)
+{
+    IEMOP_MNEMONIC(retn_Iw, "retn Iw");
+    uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_retn_iw_16, u16Imm);
+        case IEMMODE_32BIT:
+            return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_retn_iw_32, u16Imm);
+        case IEMMODE_64BIT:
+            return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_retn_iw_64, u16Imm);
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xc3
+ */
+FNIEMOP_DEF(iemOp_retn)
+{
+    IEMOP_MNEMONIC(retn, "retn");
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_retn_16);
+        case IEMMODE_32BIT:
+            return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_retn_32);
+        case IEMMODE_64BIT:
+            return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_retn_64);
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xc4
+ */
+FNIEMOP_DEF(iemOp_les_Gv_Mp__vex3)
+{
+    /* The LDS instruction is invalid 64-bit mode. In legacy and
+       compatability mode it is invalid with MOD=3.
+       The use as a VEX prefix is made possible by assigning the inverted
+       REX.R and REX.X to the two MOD bits, since the REX bits are ignored
+       outside of 64-bit mode.  VEX is not available in real or v86 mode. */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (   pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+        || IEM_IS_MODRM_REG_MODE(bRm) )
+    {
+        IEMOP_MNEMONIC(vex3_prefix, "vex3");
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx)
+        {
+            /* Note! The real mode, v8086 mode and invalid prefix checks are done once
+                     the instruction is fully decoded.  Even when XCR0=3 and CR4.OSXSAVE=0. */
+            uint8_t bVex2;   IEM_OPCODE_GET_NEXT_U8(&bVex2);
+            uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode);
+            pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_VEX;
+            if ((bVex2 & 0x80 /* VEX.W */) && pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+                pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_SIZE_REX_W;
+            pVCpu->iem.s.uRexReg    = (~bRm >> (7 - 3)) & 0x8;
+            pVCpu->iem.s.uRexIndex  = (~bRm >> (6 - 3)) & 0x8;
+            pVCpu->iem.s.uRexB      = (~bRm >> (5 - 3)) & 0x8;
+            pVCpu->iem.s.uVex3rdReg = (~bVex2 >> 3) & 0xf;
+            pVCpu->iem.s.uVexLength = (bVex2 >> 2) & 1;
+            pVCpu->iem.s.idxPrefix  = bVex2 & 0x3;
+
+            switch (bRm & 0x1f)
+            {
+                case 1: /* 0x0f lead opcode byte. */
+#ifdef IEM_WITH_VEX
+                    return FNIEMOP_CALL(g_apfnVexMap1[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+
+                case 2: /* 0x0f 0x38 lead opcode bytes. */
+#ifdef IEM_WITH_VEX
+                    return FNIEMOP_CALL(g_apfnVexMap2[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+
+                case 3: /* 0x0f 0x3a lead opcode bytes. */
+#ifdef IEM_WITH_VEX
+                    return FNIEMOP_CALL(g_apfnVexMap3[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+                    IEMOP_BITCH_ABOUT_STUB();
+                    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+
+                default:
+                    Log(("VEX3: Invalid vvvv value: %#x!\n", bRm & 0x1f));
+                    return IEMOP_RAISE_INVALID_OPCODE();
+            }
+        }
+        Log(("VEX3: AVX support disabled!\n"));
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+
+    IEMOP_MNEMONIC(les_Gv_Mp, "les Gv,Mp");
+    return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_ES, bRm);
+}
+
+
+/**
+ * @opcode      0xc5
+ */
+FNIEMOP_DEF(iemOp_lds_Gv_Mp__vex2)
+{
+    /* The LES instruction is invalid 64-bit mode. In legacy and
+       compatability mode it is invalid with MOD=3.
+       The use as a VEX prefix is made possible by assigning the inverted
+       REX.R to the top MOD bit, and the top bit in the inverted register
+       specifier to the bottom MOD bit, thereby effectively limiting 32-bit
+       to accessing registers 0..7 in this VEX form. */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (   pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+        || IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_MNEMONIC(vex2_prefix, "vex2");
+        if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fAvx)
+        {
+            /* Note! The real mode, v8086 mode and invalid prefix checks are done once
+                     the instruction is fully decoded.  Even when XCR0=3 and CR4.OSXSAVE=0. */
+            uint8_t bOpcode; IEM_OPCODE_GET_NEXT_U8(&bOpcode);
+            pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_VEX;
+            pVCpu->iem.s.uRexReg    = (~bRm >> (7 - 3)) & 0x8;
+            pVCpu->iem.s.uVex3rdReg = (~bRm >> 3) & 0xf;
+            pVCpu->iem.s.uVexLength = (bRm >> 2) & 1;
+            pVCpu->iem.s.idxPrefix  = bRm & 0x3;
+
+#ifdef IEM_WITH_VEX
+            return FNIEMOP_CALL(g_apfnVexMap1[(uintptr_t)bOpcode * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+            IEMOP_BITCH_ABOUT_STUB();
+            return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+        }
+
+        /** @todo does intel completely decode the sequence with SIB/disp before \#UD? */
+        Log(("VEX2: AVX support disabled!\n"));
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+
+    IEMOP_MNEMONIC(lds_Gv_Mp, "lds Gv,Mp");
+    return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_DS, bRm);
+}
+
+
+/**
+ * @opcode      0xc6
+ */
+FNIEMOP_DEF(iemOp_Grp11_Eb_Ib)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_REG_MASK) != (0 << X86_MODRM_REG_SHIFT)) /* only mov Eb,Ib in this group. */
+        return IEMOP_RAISE_INVALID_OPCODE();
+    IEMOP_MNEMONIC(mov_Eb_Ib, "mov Eb,Ib");
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register access */
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_RM(pVCpu, bRm), u8Imm);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory access. */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u8Imm);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0xc7
+ */
+FNIEMOP_DEF(iemOp_Grp11_Ev_Iz)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if ((bRm & X86_MODRM_REG_MASK) != (0 << X86_MODRM_REG_SHIFT)) /* only mov Eb,Ib in this group. */
+        return IEMOP_RAISE_INVALID_OPCODE();
+    IEMOP_MNEMONIC(mov_Ev_Iz, "mov Ev,Iz");
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register access */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 0);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_RM(pVCpu, bRm), u16Imm);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 0);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Imm);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 0);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Imm);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory access. */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Imm);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Imm);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Imm);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+
+
+/**
+ * @opcode      0xc8
+ */
+FNIEMOP_DEF(iemOp_enter_Iw_Ib)
+{
+    IEMOP_MNEMONIC(enter_Iw_Ib, "enter Iw,Ib");
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    uint16_t cbFrame;        IEM_OPCODE_GET_NEXT_U16(&cbFrame);
+    uint8_t  u8NestingLevel; IEM_OPCODE_GET_NEXT_U8(&u8NestingLevel);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_enter, pVCpu->iem.s.enmEffOpSize, cbFrame, u8NestingLevel);
+}
+
+
+/**
+ * @opcode      0xc9
+ */
+FNIEMOP_DEF(iemOp_leave)
+{
+    IEMOP_MNEMONIC(leave, "leave");
+    IEMOP_HLP_MIN_186();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_leave, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0xca
+ */
+FNIEMOP_DEF(iemOp_retf_Iw)
+{
+    IEMOP_MNEMONIC(retf_Iw, "retf Iw");
+    uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_retf, pVCpu->iem.s.enmEffOpSize, u16Imm);
+}
+
+
+/**
+ * @opcode      0xcb
+ */
+FNIEMOP_DEF(iemOp_retf)
+{
+    IEMOP_MNEMONIC(retf, "retf");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_retf, pVCpu->iem.s.enmEffOpSize, 0);
+}
+
+
+/**
+ * @opcode      0xcc
+ */
+FNIEMOP_DEF(iemOp_int3)
+{
+    IEMOP_MNEMONIC(int3, "int3");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_int, X86_XCPT_BP, IEMINT_INT3);
+}
+
+
+/**
+ * @opcode      0xcd
+ */
+FNIEMOP_DEF(iemOp_int_Ib)
+{
+    IEMOP_MNEMONIC(int_Ib, "int Ib");
+    uint8_t u8Int; IEM_OPCODE_GET_NEXT_U8(&u8Int);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_int, u8Int, IEMINT_INTN);
+}
+
+
+/**
+ * @opcode      0xce
+ */
+FNIEMOP_DEF(iemOp_into)
+{
+    IEMOP_MNEMONIC(into, "into");
+    IEMOP_HLP_NO_64BIT();
+
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG_CONST(uint8_t,   u8Int,  /*=*/ X86_XCPT_OF, 0);
+    IEM_MC_ARG_CONST(IEMINT,    enmInt, /*=*/ IEMINT_INTO, 1);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_int, u8Int, enmInt);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opcode      0xcf
+ */
+FNIEMOP_DEF(iemOp_iret)
+{
+    IEMOP_MNEMONIC(iret, "iret");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_iret, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0xd0
+ */
+FNIEMOP_DEF(iemOp_Grp2_Eb_1)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Eb_1, "rol Eb,1"); break;
+        case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Eb_1, "ror Eb,1"); break;
+        case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Eb_1, "rcl Eb,1"); break;
+        case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Eb_1, "rcr Eb,1"); break;
+        case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Eb_1, "shl Eb,1"); break;
+        case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Eb_1, "shr Eb,1"); break;
+        case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Eb_1, "sar Eb,1"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe, well... */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,             0);
+        IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=*/1,   1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,             0);
+        IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=*/1,   1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags,        2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+
+/**
+ * @opcode      0xd1
+ */
+FNIEMOP_DEF(iemOp_Grp2_Ev_1)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Ev_1, "rol Ev,1"); break;
+        case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Ev_1, "ror Ev,1"); break;
+        case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Ev_1, "rcl Ev,1"); break;
+        case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Ev_1, "rcr Ev,1"); break;
+        case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Ev_1, "shl Ev,1"); break;
+        case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Ev_1, "shr Ev,1"); break;
+        case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Ev_1, "sar Ev,1"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe, well... */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,           0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,           2);
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,            0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,            0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,            0);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg,/*=1*/1,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0xd2
+ */
+FNIEMOP_DEF(iemOp_Grp2_Eb_CL)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Eb_CL, "rol Eb,CL"); break;
+        case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Eb_CL, "ror Eb,CL"); break;
+        case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Eb_CL, "rcl Eb,CL"); break;
+        case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Eb_CL, "rcr Eb,CL"); break;
+        case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Eb_CL, "shl Eb,CL"); break;
+        case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Eb_CL, "shr Eb,CL"); break;
+        case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Eb_CL, "sar Eb,CL"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc, grr. */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,   pu8Dst,     0);
+        IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+        IEM_MC_ARG(uint32_t *,  pEFlags,    2);
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,   pu8Dst,          0);
+        IEM_MC_ARG(uint8_t,     cShiftArg,       1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU8, pu8Dst, cShiftArg, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0xd3
+ */
+FNIEMOP_DEF(iemOp_Grp2_Ev_CL)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPSHIFTSIZES pImpl;
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rol_eflags); IEMOP_MNEMONIC(rol_Ev_CL, "rol Ev,CL"); break;
+        case 1: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_ror_eflags); IEMOP_MNEMONIC(ror_Ev_CL, "ror Ev,CL"); break;
+        case 2: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcl_eflags); IEMOP_MNEMONIC(rcl_Ev_CL, "rcl Ev,CL"); break;
+        case 3: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_rcr_eflags); IEMOP_MNEMONIC(rcr_Ev_CL, "rcr Ev,CL"); break;
+        case 4: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shl_eflags); IEMOP_MNEMONIC(shl_Ev_CL, "shl Ev,CL"); break;
+        case 5: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shr_eflags); IEMOP_MNEMONIC(shr_Ev_CL, "shr Ev,CL"); break;
+        case 7: pImpl = IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_sar_eflags); IEMOP_MNEMONIC(sar_Ev_CL, "sar Ev,CL"); break;
+        case 6: return IEMOP_RAISE_INVALID_OPCODE();
+        IEM_NOT_REACHED_DEFAULT_CASE_RET(); /* gcc maybe stupid */
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_AF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,    0);
+                IEM_MC_ARG(uint8_t,         cShiftArg,  1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,    0);
+                IEM_MC_ARG(uint8_t,         cShiftArg,  1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,    0);
+                IEM_MC_ARG(uint8_t,         cShiftArg,  1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,  pu16Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,  pu32Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,  pu64Dst,    0);
+                IEM_MC_ARG(uint8_t,     cShiftArg,  1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+/**
+ * @opcode      0xd4
+ */
+FNIEMOP_DEF(iemOp_aam_Ib)
+{
+    IEMOP_MNEMONIC(aam_Ib, "aam Ib");
+    uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_NO_64BIT();
+    if (!bImm)
+        return IEMOP_RAISE_DIVIDE_ERROR();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_aam, bImm);
+}
+
+
+/**
+ * @opcode      0xd5
+ */
+FNIEMOP_DEF(iemOp_aad_Ib)
+{
+    IEMOP_MNEMONIC(aad_Ib, "aad Ib");
+    uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_NO_64BIT();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_aad, bImm);
+}
+
+
+/**
+ * @opcode      0xd6
+ */
+FNIEMOP_DEF(iemOp_salc)
+{
+    IEMOP_MNEMONIC(salc, "salc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_NO_64BIT();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+        IEM_MC_STORE_GREG_U8_CONST(X86_GREG_xAX, 0xff);
+    } IEM_MC_ELSE() {
+        IEM_MC_STORE_GREG_U8_CONST(X86_GREG_xAX, 0x00);
+    } IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0xd7
+ */
+FNIEMOP_DEF(iemOp_xlat)
+{
+    IEMOP_MNEMONIC(xlat, "xlat");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_LOCAL(uint8_t,  u8Tmp);
+            IEM_MC_LOCAL(uint16_t, u16Addr);
+            IEM_MC_FETCH_GREG_U8_ZX_U16(u16Addr, X86_GREG_xAX);
+            IEM_MC_ADD_GREG_U16_TO_LOCAL(u16Addr, X86_GREG_xBX);
+            IEM_MC_FETCH_MEM16_U8(u8Tmp, pVCpu->iem.s.iEffSeg, u16Addr);
+            IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_LOCAL(uint8_t,  u8Tmp);
+            IEM_MC_LOCAL(uint32_t, u32Addr);
+            IEM_MC_FETCH_GREG_U8_ZX_U32(u32Addr, X86_GREG_xAX);
+            IEM_MC_ADD_GREG_U32_TO_LOCAL(u32Addr, X86_GREG_xBX);
+            IEM_MC_FETCH_MEM32_U8(u8Tmp, pVCpu->iem.s.iEffSeg, u32Addr);
+            IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_LOCAL(uint8_t,  u8Tmp);
+            IEM_MC_LOCAL(uint64_t, u64Addr);
+            IEM_MC_FETCH_GREG_U8_ZX_U64(u64Addr, X86_GREG_xAX);
+            IEM_MC_ADD_GREG_U64_TO_LOCAL(u64Addr, X86_GREG_xBX);
+            IEM_MC_FETCH_MEM_U8(u8Tmp, pVCpu->iem.s.iEffSeg, u64Addr);
+            IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Tmp);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+         IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and STn, and storing the
+ * result in ST0.
+ *
+ * @param   bRm         Mod R/M byte.
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_st0_stN, uint8_t, bRm, PFNIEMAIMPLFPUR80, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, IEM_GET_MODRM_RM_8(bRm))
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr80Value2);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and STn, and only affecting
+ * flags.
+ *
+ * @param   bRm         Mod R/M byte.
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpuNoStore_st0_stN, uint8_t, bRm, PFNIEMAIMPLFPUR80FSW, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, IEM_GET_MODRM_RM_8(bRm))
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pu16Fsw, pr80Value1, pr80Value2);
+        IEM_MC_UPDATE_FSW(u16Fsw);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(UINT8_MAX);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and STn, only affecting
+ * flags, and popping when done.
+ *
+ * @param   bRm         Mod R/M byte.
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpuNoStore_st0_stN_pop, uint8_t, bRm, PFNIEMAIMPLFPUR80FSW, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, IEM_GET_MODRM_RM_8(bRm))
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pu16Fsw, pr80Value1, pr80Value2);
+        IEM_MC_UPDATE_FSW_THEN_POP(u16Fsw);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(UINT8_MAX);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd8 11/0. */
+FNIEMOP_DEF_1(iemOp_fadd_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fadd_st0_stN, "fadd st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fadd_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/1. */
+FNIEMOP_DEF_1(iemOp_fmul_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fmul_st0_stN, "fmul st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fmul_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/2. */
+FNIEMOP_DEF_1(iemOp_fcom_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcom_st0_stN, "fcom st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN, bRm, iemAImpl_fcom_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/3. */
+FNIEMOP_DEF_1(iemOp_fcomp_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcomp_st0_stN, "fcomp st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN_pop, bRm, iemAImpl_fcom_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/4. */
+FNIEMOP_DEF_1(iemOp_fsub_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsub_st0_stN, "fsub st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fsub_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/5. */
+FNIEMOP_DEF_1(iemOp_fsubr_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubr_st0_stN, "fsubr st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fsubr_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/6. */
+FNIEMOP_DEF_1(iemOp_fdiv_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdiv_st0_stN, "fdiv st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fdiv_r80_by_r80);
+}
+
+
+/** Opcode 0xd8 11/7. */
+FNIEMOP_DEF_1(iemOp_fdivr_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivr_st0_stN, "fdivr st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, bRm, iemAImpl_fdivr_r80_by_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and an m32r, and storing
+ * the result in ST0.
+ *
+ * @param   bRm         Mod R/M byte.
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_st0_m32r, uint8_t, bRm, PFNIEMAIMPLFPUR32, pfnAImpl)
+{
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_LOCAL(RTFLOAT32U,            r32Val2);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U,  pr32Val2,       r32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R32(r32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr32Val2);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd8 !11/0. */
+FNIEMOP_DEF_1(iemOp_fadd_m32r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fadd_st0_m32r, "fadd st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fadd_r80_by_r32);
+}
+
+
+/** Opcode 0xd8 !11/1. */
+FNIEMOP_DEF_1(iemOp_fmul_m32r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fmul_st0_m32r, "fmul st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fmul_r80_by_r32);
+}
+
+
+/** Opcode 0xd8 !11/2. */
+FNIEMOP_DEF_1(iemOp_fcom_m32r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcom_st0_m32r, "fcom st0,m32r");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_LOCAL(RTFLOAT32U,            r32Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U,  pr32Val2,       r32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R32(r32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r32, pu16Fsw, pr80Value1, pr32Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd8 !11/3. */
+FNIEMOP_DEF_1(iemOp_fcomp_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcomp_st0_m32r, "fcomp st0,m32r");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_LOCAL(RTFLOAT32U,            r32Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U,  pr32Val2,       r32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R32(r32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r32, pu16Fsw, pr80Value1, pr32Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd8 !11/4. */
+FNIEMOP_DEF_1(iemOp_fsub_m32r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsub_st0_m32r, "fsub st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fsub_r80_by_r32);
+}
+
+
+/** Opcode 0xd8 !11/5. */
+FNIEMOP_DEF_1(iemOp_fsubr_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubr_st0_m32r, "fsubr st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fsubr_r80_by_r32);
+}
+
+
+/** Opcode 0xd8 !11/6. */
+FNIEMOP_DEF_1(iemOp_fdiv_m32r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdiv_st0_m32r, "fdiv st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fdiv_r80_by_r32);
+}
+
+
+/** Opcode 0xd8 !11/7. */
+FNIEMOP_DEF_1(iemOp_fdivr_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivr_st0_m32r, "fdivr st0,m32r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32r, bRm, iemAImpl_fdivr_r80_by_r32);
+}
+
+
+/**
+ * @opcode      0xd8
+ */
+FNIEMOP_DEF(iemOp_EscF0)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xd8 & 0x7);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fadd_stN,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fmul_stN,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcom_stN,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fcomp_stN, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fsub_stN,  bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fsubr_stN, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fdiv_stN,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fdivr_stN, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fadd_m32r,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fmul_m32r,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcom_m32r,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fcomp_m32r, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fsub_m32r,  bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fsubr_m32r, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fdiv_m32r,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fdivr_m32r, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xd9 /0 mem32real
+ * @sa  iemOp_fld_m64r */
+FNIEMOP_DEF_1(iemOp_fld_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fld_m32r, "fld m32r");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_LOCAL(RTFLOAT32U,            r32Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U,  pr32Val,    r32Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R32(r32Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r80_from_r32, pFpuRes, pr32Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 !11/2 mem32real */
+FNIEMOP_DEF_1(iemOp_fst_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fst_m32r, "fst m32r");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTFLOAT32U,             pr32Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pr32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r32, pu16Fsw, pr32Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr32Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(pr32Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pr32Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 !11/3 */
+FNIEMOP_DEF_1(iemOp_fstp_m32r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fstp_m32r, "fstp m32r");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTFLOAT32U,             pr32Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pr32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r32, pu16Fsw, pr32Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr32Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_NEG_QNAN_R32_BY_REF(pr32Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pr32Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 !11/4 */
+FNIEMOP_DEF_1(iemOp_fldenv, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fldenv, "fldenv m14/28byte");
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG_CONST(IEMMODE,           enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize,  0);
+    IEM_MC_ARG(uint8_t,                 iEffSeg,                                    1);
+    IEM_MC_ARG(RTGCPTR,                 GCPtrEffSrc,                                2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_fldenv, enmEffOpSize, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 !11/5 */
+FNIEMOP_DEF_1(iemOp_fldcw, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fldcw_m2byte, "fldcw m2byte");
+    IEM_MC_BEGIN(1, 1);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_ARG(uint16_t,                u16Fsw,                                     0);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FETCH_MEM_U16(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_CALL_CIMPL_1(iemCImpl_fldcw, u16Fsw);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 !11/6 */
+FNIEMOP_DEF_1(iemOp_fnstenv, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fstenv, "fstenv m14/m28byte");
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG_CONST(IEMMODE,           enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize,  0);
+    IEM_MC_ARG(uint8_t,                 iEffSeg,                                    1);
+    IEM_MC_ARG(RTGCPTR,                 GCPtrEffDst,                                2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_fnstenv, enmEffOpSize, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xd9 !11/7 */
+FNIEMOP_DEF_1(iemOp_fnstcw, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fnstcw_m2byte, "fnstcw m2byte");
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fcw);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_FETCH_FCW(u16Fcw);
+    IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Fcw);
+    IEM_MC_ADVANCE_RIP_AND_FINISH(); /* C0-C3 are documented as undefined, we leave them unmodified. */
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 0xd0, 0xd9 0xd8-0xdf, ++?.  */
+FNIEMOP_DEF(iemOp_fnop)
+{
+    IEMOP_MNEMONIC(fnop, "fnop");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    /** @todo Testcase: looks like FNOP leaves FOP alone but updates FPUIP. Could be
+     *        intel optimizations. Investigate. */
+    IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ADVANCE_RIP_AND_FINISH(); /* C0-C3 are documented as undefined, we leave them unmodified. */
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 11/0 stN */
+FNIEMOP_DEF_1(iemOp_fld_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fld_stN, "fld stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /** @todo Testcase: Check if this raises \#MF?  Intel mentioned it not. AMD
+     *        indicates that it does. */
+    IEM_MC_BEGIN(0, 2);
+    IEM_MC_LOCAL(PCRTFLOAT80U,          pr80Value);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, IEM_GET_MODRM_RM_8(bRm))
+        IEM_MC_SET_FPU_RESULT(FpuRes, 0 /*FSW*/, pr80Value);
+        IEM_MC_PUSH_FPU_RESULT(FpuRes);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_UNDERFLOW();
+    IEM_MC_ENDIF();
+
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 11/3 stN */
+FNIEMOP_DEF_1(iemOp_fxch_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fxch_stN, "fxch stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /** @todo Testcase: Check if this raises \#MF?  Intel mentioned it not. AMD
+     *        indicates that it does. */
+    IEM_MC_BEGIN(1, 3);
+    IEM_MC_LOCAL(PCRTFLOAT80U,          pr80Value1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,          pr80Value2);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_CONST(uint8_t,           iStReg, /*=*/ IEM_GET_MODRM_RM_8(bRm), 0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, IEM_GET_MODRM_RM_8(bRm))
+        IEM_MC_SET_FPU_RESULT(FpuRes, X86_FSW_C1, pr80Value2);
+        IEM_MC_STORE_FPUREG_R80_SRC_REF(IEM_GET_MODRM_RM_8(bRm), pr80Value1);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_CALL_CIMPL_1(iemCImpl_fxch_underflow, iStReg);
+    IEM_MC_ENDIF();
+
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 11/4, 0xdd 11/2. */
+FNIEMOP_DEF_1(iemOp_fstp_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fstp_st0_stN, "fstp st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    /* fstp st0, st0 is frequently used as an official 'ffreep st0' sequence. */
+    uint8_t const iDstReg = IEM_GET_MODRM_RM_8(bRm);
+    if (!iDstReg)
+    {
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL_CONST(uint16_t,        u16Fsw, /*=*/ 0);
+        IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+        IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_IF_FPUREG_NOT_EMPTY(0)
+            IEM_MC_UPDATE_FSW_THEN_POP(u16Fsw);
+        IEM_MC_ELSE()
+            IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(0);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(PCRTFLOAT80U,          pr80Value);
+        IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+        IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+        IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+            IEM_MC_SET_FPU_RESULT(FpuRes, 0 /*FSW*/, pr80Value);
+            IEM_MC_STORE_FPU_RESULT_THEN_POP(FpuRes, iDstReg);
+        IEM_MC_ELSE()
+            IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(iDstReg);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and replaces it with the
+ * result, i.e. unary operators.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpFpu_st0, PFNIEMAIMPLFPUR80UNARY, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          1);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_2(pfnAImpl, pFpuRes, pr80Value);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 0xe0. */
+FNIEMOP_DEF(iemOp_fchs)
+{
+    IEMOP_MNEMONIC(fchs_st0, "fchs st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fchs_r80);
+}
+
+
+/** Opcode 0xd9 0xe1. */
+FNIEMOP_DEF(iemOp_fabs)
+{
+    IEMOP_MNEMONIC(fabs_st0, "fabs st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fabs_r80);
+}
+
+
+/** Opcode 0xd9 0xe4. */
+FNIEMOP_DEF(iemOp_ftst)
+{
+    IEMOP_MNEMONIC(ftst_st0, "ftst st0");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          1);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_ftst_r80, pu16Fsw, pr80Value);
+        IEM_MC_UPDATE_FSW(u16Fsw);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(UINT8_MAX);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 0xe5. */
+FNIEMOP_DEF(iemOp_fxam)
+{
+    IEMOP_MNEMONIC(fxam_st0, "fxam st0");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          1);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_REF_FPUREG(pr80Value, 0);
+    IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fxam_r80, pu16Fsw, pr80Value);
+    IEM_MC_UPDATE_FSW(u16Fsw);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/**
+ * Common worker for FPU instructions pushing a constant onto the FPU stack.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpFpuPushConstant, PFNIEMAIMPLFPUR80LDCONST, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(1, 1);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,    FpuRes, 0);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_1(pfnAImpl, pFpuRes);
+        IEM_MC_PUSH_FPU_RESULT(FpuRes);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW();
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 0xe8. */
+FNIEMOP_DEF(iemOp_fld1)
+{
+    IEMOP_MNEMONIC(fld1, "fld1");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fld1);
+}
+
+
+/** Opcode 0xd9 0xe9. */
+FNIEMOP_DEF(iemOp_fldl2t)
+{
+    IEMOP_MNEMONIC(fldl2t, "fldl2t");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldl2t);
+}
+
+
+/** Opcode 0xd9 0xea. */
+FNIEMOP_DEF(iemOp_fldl2e)
+{
+    IEMOP_MNEMONIC(fldl2e, "fldl2e");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldl2e);
+}
+
+/** Opcode 0xd9 0xeb. */
+FNIEMOP_DEF(iemOp_fldpi)
+{
+    IEMOP_MNEMONIC(fldpi, "fldpi");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldpi);
+}
+
+
+/** Opcode 0xd9 0xec. */
+FNIEMOP_DEF(iemOp_fldlg2)
+{
+    IEMOP_MNEMONIC(fldlg2, "fldlg2");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldlg2);
+}
+
+/** Opcode 0xd9 0xed. */
+FNIEMOP_DEF(iemOp_fldln2)
+{
+    IEMOP_MNEMONIC(fldln2, "fldln2");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldln2);
+}
+
+
+/** Opcode 0xd9 0xee. */
+FNIEMOP_DEF(iemOp_fldz)
+{
+    IEMOP_MNEMONIC(fldz, "fldz");
+    return FNIEMOP_CALL_1(iemOpHlpFpuPushConstant, iemAImpl_fldz);
+}
+
+
+/** Opcode 0xd9 0xf0.
+ *
+ * The f2xm1 instruction works on values +1.0 thru -1.0, currently (the range on
+ * 287 & 8087 was +0.5 thru 0.0 according to docs).  In addition is does appear
+ * to produce proper results for +Inf and -Inf.
+ *
+ * This is probably usful in the implementation pow() and similar.
+ */
+FNIEMOP_DEF(iemOp_f2xm1)
+{
+    IEMOP_MNEMONIC(f2xm1_st0, "f2xm1 st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_f2xm1_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on STn and ST0, storing the result
+ * in STn, and popping the stack unless IE, DE or ZE was raised.
+ *
+ * @param   bRm         Mod R/M byte.
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_stN_st0_pop, uint8_t, bRm, PFNIEMAIMPLFPUR80, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, IEM_GET_MODRM_RM_8(bRm), pr80Value2, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr80Value2);
+        IEM_MC_STORE_FPU_RESULT_THEN_POP(FpuRes, IEM_GET_MODRM_RM_8(bRm));
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP(IEM_GET_MODRM_RM_8(bRm));
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 0xf1. */
+FNIEMOP_DEF(iemOp_fyl2x)
+{
+    IEMOP_MNEMONIC(fyl2x_st0, "fyl2x st1,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, 1, iemAImpl_fyl2x_r80_by_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and having two outputs, one
+ * replacing ST0 and one pushed onto the stack.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpFpuReplace_st0_push, PFNIEMAIMPLFPUR80UNARYTWO, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_LOCAL(IEMFPURESULTTWO,           FpuResTwo);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULTTWO,  pFpuResTwo, FpuResTwo,  0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value,              1);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_2(pfnAImpl, pFpuResTwo, pr80Value);
+        IEM_MC_PUSH_FPU_RESULT_TWO(FpuResTwo);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_UNDERFLOW_TWO();
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 0xf2. */
+FNIEMOP_DEF(iemOp_fptan)
+{
+    IEMOP_MNEMONIC(fptan_st0, "fptan st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpuReplace_st0_push, iemAImpl_fptan_r80_r80);
+}
+
+
+/** Opcode 0xd9 0xf3. */
+FNIEMOP_DEF(iemOp_fpatan)
+{
+    IEMOP_MNEMONIC(fpatan_st1_st0, "fpatan st1,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, 1, iemAImpl_fpatan_r80_by_r80);
+}
+
+
+/** Opcode 0xd9 0xf4. */
+FNIEMOP_DEF(iemOp_fxtract)
+{
+    IEMOP_MNEMONIC(fxtract_st0, "fxtract st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpuReplace_st0_push, iemAImpl_fxtract_r80_r80);
+}
+
+
+/** Opcode 0xd9 0xf5. */
+FNIEMOP_DEF(iemOp_fprem1)
+{
+    IEMOP_MNEMONIC(fprem1_st0_st1, "fprem1 st0,st1");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, 1, iemAImpl_fprem1_r80_by_r80);
+}
+
+
+/** Opcode 0xd9 0xf6. */
+FNIEMOP_DEF(iemOp_fdecstp)
+{
+    IEMOP_MNEMONIC(fdecstp, "fdecstp");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    /* Note! C0, C2 and C3 are documented as undefined, we clear them. */
+    /** @todo Testcase: Check whether FOP, FPUIP and FPUCS are affected by
+     *        FINCSTP and FDECSTP. */
+
+    IEM_MC_BEGIN(0,0);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_STACK_DEC_TOP();
+    IEM_MC_UPDATE_FSW_CONST(0);
+
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 0xf7. */
+FNIEMOP_DEF(iemOp_fincstp)
+{
+    IEMOP_MNEMONIC(fincstp, "fincstp");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    /* Note! C0, C2 and C3 are documented as undefined, we clear them. */
+    /** @todo Testcase: Check whether FOP, FPUIP and FPUCS are affected by
+     *        FINCSTP and FDECSTP. */
+
+    IEM_MC_BEGIN(0,0);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_STACK_INC_TOP();
+    IEM_MC_UPDATE_FSW_CONST(0);
+
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xd9 0xf8. */
+FNIEMOP_DEF(iemOp_fprem)
+{
+    IEMOP_MNEMONIC(fprem_st0_st1, "fprem st0,st1");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, 1, iemAImpl_fprem_r80_by_r80);
+}
+
+
+/** Opcode 0xd9 0xf9. */
+FNIEMOP_DEF(iemOp_fyl2xp1)
+{
+    IEMOP_MNEMONIC(fyl2xp1_st1_st0, "fyl2xp1 st1,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, 1, iemAImpl_fyl2xp1_r80_by_r80);
+}
+
+
+/** Opcode 0xd9 0xfa. */
+FNIEMOP_DEF(iemOp_fsqrt)
+{
+    IEMOP_MNEMONIC(fsqrt_st0, "fsqrt st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fsqrt_r80);
+}
+
+
+/** Opcode 0xd9 0xfb. */
+FNIEMOP_DEF(iemOp_fsincos)
+{
+    IEMOP_MNEMONIC(fsincos_st0, "fsincos st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpuReplace_st0_push, iemAImpl_fsincos_r80_r80);
+}
+
+
+/** Opcode 0xd9 0xfc. */
+FNIEMOP_DEF(iemOp_frndint)
+{
+    IEMOP_MNEMONIC(frndint_st0, "frndint st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_frndint_r80);
+}
+
+
+/** Opcode 0xd9 0xfd. */
+FNIEMOP_DEF(iemOp_fscale)
+{
+    IEMOP_MNEMONIC(fscale_st0_st1, "fscale st0,st1");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_stN, 1, iemAImpl_fscale_r80_by_r80);
+}
+
+
+/** Opcode 0xd9 0xfe. */
+FNIEMOP_DEF(iemOp_fsin)
+{
+    IEMOP_MNEMONIC(fsin_st0, "fsin st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fsin_r80);
+}
+
+
+/** Opcode 0xd9 0xff. */
+FNIEMOP_DEF(iemOp_fcos)
+{
+    IEMOP_MNEMONIC(fcos_st0, "fcos st0");
+    return FNIEMOP_CALL_1(iemOpHlpFpu_st0, iemAImpl_fcos_r80);
+}
+
+
+/** Used by iemOp_EscF1. */
+IEM_STATIC const PFNIEMOP g_apfnEscF1_E0toFF[32] =
+{
+    /* 0xe0 */  iemOp_fchs,
+    /* 0xe1 */  iemOp_fabs,
+    /* 0xe2 */  iemOp_Invalid,
+    /* 0xe3 */  iemOp_Invalid,
+    /* 0xe4 */  iemOp_ftst,
+    /* 0xe5 */  iemOp_fxam,
+    /* 0xe6 */  iemOp_Invalid,
+    /* 0xe7 */  iemOp_Invalid,
+    /* 0xe8 */  iemOp_fld1,
+    /* 0xe9 */  iemOp_fldl2t,
+    /* 0xea */  iemOp_fldl2e,
+    /* 0xeb */  iemOp_fldpi,
+    /* 0xec */  iemOp_fldlg2,
+    /* 0xed */  iemOp_fldln2,
+    /* 0xee */  iemOp_fldz,
+    /* 0xef */  iemOp_Invalid,
+    /* 0xf0 */  iemOp_f2xm1,
+    /* 0xf1 */  iemOp_fyl2x,
+    /* 0xf2 */  iemOp_fptan,
+    /* 0xf3 */  iemOp_fpatan,
+    /* 0xf4 */  iemOp_fxtract,
+    /* 0xf5 */  iemOp_fprem1,
+    /* 0xf6 */  iemOp_fdecstp,
+    /* 0xf7 */  iemOp_fincstp,
+    /* 0xf8 */  iemOp_fprem,
+    /* 0xf9 */  iemOp_fyl2xp1,
+    /* 0xfa */  iemOp_fsqrt,
+    /* 0xfb */  iemOp_fsincos,
+    /* 0xfc */  iemOp_frndint,
+    /* 0xfd */  iemOp_fscale,
+    /* 0xfe */  iemOp_fsin,
+    /* 0xff */  iemOp_fcos
+};
+
+
+/**
+ * @opcode      0xd9
+ */
+FNIEMOP_DEF(iemOp_EscF1)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xd9 & 0x7);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fld_stN, bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fxch_stN, bRm);
+            case 2:
+                if (bRm == 0xd0)
+                    return FNIEMOP_CALL(iemOp_fnop);
+                return IEMOP_RAISE_INVALID_OPCODE();
+            case 3: return FNIEMOP_CALL_1(iemOp_fstp_stN, bRm); /* Reserved. Intel behavior seems to be FSTP ST(i) though. */
+            case 4:
+            case 5:
+            case 6:
+            case 7:
+                Assert((unsigned)bRm - 0xe0U < RT_ELEMENTS(g_apfnEscF1_E0toFF));
+                return FNIEMOP_CALL(g_apfnEscF1_E0toFF[bRm - 0xe0]);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fld_m32r,  bRm);
+            case 1: return IEMOP_RAISE_INVALID_OPCODE();
+            case 2: return FNIEMOP_CALL_1(iemOp_fst_m32r,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fstp_m32r, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fldenv,    bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fldcw,     bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fnstenv,    bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fnstcw,     bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xda 11/0. */
+FNIEMOP_DEF_1(iemOp_fcmovb_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovb_st0_stN, "fcmovb st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0)
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xda 11/1. */
+FNIEMOP_DEF_1(iemOp_fcmove_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmove_st0_stN, "fcmove st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0)
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xda 11/2. */
+FNIEMOP_DEF_1(iemOp_fcmovbe_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovbe_st0_stN, "fcmovbe st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0)
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xda 11/3. */
+FNIEMOP_DEF_1(iemOp_fcmovu_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovu_st0_stN, "fcmovu st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0)
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and ST1, only affecting
+ * flags, and popping twice when done.
+ *
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpFpuNoStore_st0_st1_pop_pop, PFNIEMAIMPLFPUR80FSW, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, 0, pr80Value2, 1)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pu16Fsw, pr80Value1, pr80Value2);
+        IEM_MC_UPDATE_FSW_THEN_POP_POP(u16Fsw);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_THEN_POP_POP();
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xda 0xe9. */
+FNIEMOP_DEF(iemOp_fucompp)
+{
+    IEMOP_MNEMONIC(fucompp, "fucompp");
+    return FNIEMOP_CALL_1(iemOpHlpFpuNoStore_st0_st1_pop_pop, iemAImpl_fucom_r80_by_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and an m32i, and storing
+ * the result in ST0.
+ *
+ * @param   bRm         Mod R/M byte.
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_st0_m32i, uint8_t, bRm, PFNIEMAIMPLFPUI32, pfnAImpl)
+{
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(int32_t,                   i32Val2);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int32_t const *,   pi32Val2,       i32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I32(i32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pi32Val2);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xda !11/0. */
+FNIEMOP_DEF_1(iemOp_fiadd_m32i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fiadd_m32i, "fiadd m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fiadd_r80_by_i32);
+}
+
+
+/** Opcode 0xda !11/1. */
+FNIEMOP_DEF_1(iemOp_fimul_m32i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fimul_m32i, "fimul m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fimul_r80_by_i32);
+}
+
+
+/** Opcode 0xda !11/2. */
+FNIEMOP_DEF_1(iemOp_ficom_m32i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ficom_st0_m32i, "ficom st0,m32i");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,                  u16Fsw);
+    IEM_MC_LOCAL(int32_t,                   i32Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,        pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int32_t const *,   pi32Val2,       i32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I32(i32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i32, pu16Fsw, pr80Value1, pi32Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xda !11/3. */
+FNIEMOP_DEF_1(iemOp_ficomp_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ficomp_st0_m32i, "ficomp st0,m32i");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,                  u16Fsw);
+    IEM_MC_LOCAL(int32_t,                   i32Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,        pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int32_t const *,   pi32Val2,       i32Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I32(i32Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i32, pu16Fsw, pr80Value1, pi32Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xda !11/4. */
+FNIEMOP_DEF_1(iemOp_fisub_m32i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisub_m32i, "fisub m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fisub_r80_by_i32);
+}
+
+
+/** Opcode 0xda !11/5. */
+FNIEMOP_DEF_1(iemOp_fisubr_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisubr_m32i, "fisubr m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fisubr_r80_by_i32);
+}
+
+
+/** Opcode 0xda !11/6. */
+FNIEMOP_DEF_1(iemOp_fidiv_m32i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fidiv_m32i, "fidiv m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fidiv_r80_by_i32);
+}
+
+
+/** Opcode 0xda !11/7. */
+FNIEMOP_DEF_1(iemOp_fidivr_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fidivr_m32i, "fidivr m32i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m32i, bRm, iemAImpl_fidivr_r80_by_i32);
+}
+
+
+/**
+ * @opcode      0xda
+ */
+FNIEMOP_DEF(iemOp_EscF2)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xda & 0x7);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fcmovb_stN, bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fcmove_stN, bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcmovbe_stN, bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fcmovu_stN, bRm);
+            case 4: return IEMOP_RAISE_INVALID_OPCODE();
+            case 5:
+                if (bRm == 0xe9)
+                    return FNIEMOP_CALL(iemOp_fucompp);
+                return IEMOP_RAISE_INVALID_OPCODE();
+            case 6: return IEMOP_RAISE_INVALID_OPCODE();
+            case 7: return IEMOP_RAISE_INVALID_OPCODE();
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fiadd_m32i,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fimul_m32i,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_ficom_m32i,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_ficomp_m32i, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fisub_m32i,  bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fisubr_m32i, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fidiv_m32i,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fidivr_m32i, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xdb !11/0. */
+FNIEMOP_DEF_1(iemOp_fild_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fild_m32i, "fild m32i");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(int32_t,                   i32Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(int32_t const *,   pi32Val,    i32Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I32(i32Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fild_r80_from_i32, pFpuRes, pi32Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb !11/1. */
+FNIEMOP_DEF_1(iemOp_fisttp_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisttp_m32i, "fisttp m32i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int32_t *,               pi32Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fistt_r80_to_i32, pu16Fsw, pi32Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi32Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I32_CONST_BY_REF(pi32Dst, INT32_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi32Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb !11/2. */
+FNIEMOP_DEF_1(iemOp_fist_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fist_m32i, "fist m32i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int32_t *,               pi32Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i32, pu16Fsw, pi32Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi32Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I32_CONST_BY_REF(pi32Dst, INT32_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi32Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb !11/3. */
+FNIEMOP_DEF_1(iemOp_fistp_m32i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fistp_m32i, "fistp m32i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int32_t *,               pi32Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi32Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i32, pu16Fsw, pi32Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi32Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I32_CONST_BY_REF(pi32Dst, INT32_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi32Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb !11/5. */
+FNIEMOP_DEF_1(iemOp_fld_m80r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fld_m80r, "fld m80r");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_LOCAL(RTFLOAT80U,            r80Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT80U,  pr80Val,    r80Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R80(r80Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r80_from_r80, pFpuRes, pr80Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb !11/7. */
+FNIEMOP_DEF_1(iemOp_fstp_m80r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fstp_m80r, "fstp m80r");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTFLOAT80U,             pr80Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP_EX(pr80Dst, IEM_ACCESS_DATA_W, sizeof(*pr80Dst), pVCpu->iem.s.iEffSeg, GCPtrEffDst, 7 /*cbAlign*/, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r80, pu16Fsw, pr80Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr80Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_NEG_QNAN_R80_BY_REF(pr80Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pr80Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb 11/0. */
+FNIEMOP_DEF_1(iemOp_fcmovnb_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovnb_st0_stN, "fcmovnb st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0)
+        IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_CF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb 11/1. */
+FNIEMOP_DEF_1(iemOp_fcmovne_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovne_st0_stN, "fcmovne st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0)
+        IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb 11/2. */
+FNIEMOP_DEF_1(iemOp_fcmovnbe_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovnbe_st0_stN, "fcmovnbe st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0)
+        IEM_MC_IF_EFL_NO_BITS_SET(X86_EFL_CF | X86_EFL_ZF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb 11/3. */
+FNIEMOP_DEF_1(iemOp_fcmovnnu_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcmovnnu_st0_stN, "fcmovnnu st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(PCRTFLOAT80U,      pr80ValueN);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80_FIRST(pr80ValueN, IEM_GET_MODRM_RM_8(bRm), 0)
+        IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_PF)
+            IEM_MC_STORE_FPUREG_R80_SRC_REF(0, pr80ValueN);
+        IEM_MC_ENDIF();
+        IEM_MC_UPDATE_FPU_OPCODE_IP();
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb 0xe0. */
+FNIEMOP_DEF(iemOp_fneni)
+{
+    IEMOP_MNEMONIC(fneni, "fneni (8087/ign)");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb 0xe1. */
+FNIEMOP_DEF(iemOp_fndisi)
+{
+    IEMOP_MNEMONIC(fndisi, "fndisi (8087/ign)");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb 0xe2. */
+FNIEMOP_DEF(iemOp_fnclex)
+{
+    IEMOP_MNEMONIC(fnclex, "fnclex");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_CLEAR_FSW_EX();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb 0xe3. */
+FNIEMOP_DEF(iemOp_fninit)
+{
+    IEMOP_MNEMONIC(fninit, "fninit");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_finit, false /*fCheckXcpts*/);
+}
+
+
+/** Opcode 0xdb 0xe4. */
+FNIEMOP_DEF(iemOp_fnsetpm)
+{
+    IEMOP_MNEMONIC(fnsetpm, "fnsetpm (80287/ign)");   /* set protected mode on fpu. */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdb 0xe5. */
+FNIEMOP_DEF(iemOp_frstpm)
+{
+    IEMOP_MNEMONIC(frstpm, "frstpm (80287XL/ign)"); /* reset pm, back to real mode. */
+#if 0 /* #UDs on newer CPUs */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+    return VINF_SUCCESS;
+#else
+    return IEMOP_RAISE_INVALID_OPCODE();
+#endif
+}
+
+
+/** Opcode 0xdb 11/5. */
+FNIEMOP_DEF_1(iemOp_fucomi_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fucomi_st0_stN, "fucomi st0,stN");
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, IEM_GET_MODRM_RM_8(bRm), iemAImpl_fucomi_r80_by_r80, false /*fPop*/);
+}
+
+
+/** Opcode 0xdb 11/6. */
+FNIEMOP_DEF_1(iemOp_fcomi_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcomi_st0_stN, "fcomi st0,stN");
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, IEM_GET_MODRM_RM_8(bRm), iemAImpl_fcomi_r80_by_r80, false /*fPop*/);
+}
+
+
+/**
+ * @opcode      0xdb
+ */
+FNIEMOP_DEF(iemOp_EscF3)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xdb & 0x7);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fcmovnb_stN,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fcmovne_stN,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcmovnbe_stN, bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fcmovnnu_stN, bRm);
+            case 4:
+                switch (bRm)
+                {
+                    case 0xe0:  return FNIEMOP_CALL(iemOp_fneni);
+                    case 0xe1:  return FNIEMOP_CALL(iemOp_fndisi);
+                    case 0xe2:  return FNIEMOP_CALL(iemOp_fnclex);
+                    case 0xe3:  return FNIEMOP_CALL(iemOp_fninit);
+                    case 0xe4:  return FNIEMOP_CALL(iemOp_fnsetpm);
+                    case 0xe5:  return FNIEMOP_CALL(iemOp_frstpm);
+                    case 0xe6:  return IEMOP_RAISE_INVALID_OPCODE();
+                    case 0xe7:  return IEMOP_RAISE_INVALID_OPCODE();
+                    IEM_NOT_REACHED_DEFAULT_CASE_RET();
+                }
+                break;
+            case 5: return FNIEMOP_CALL_1(iemOp_fucomi_stN, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fcomi_stN,  bRm);
+            case 7: return IEMOP_RAISE_INVALID_OPCODE();
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fild_m32i,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fisttp_m32i,bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fist_m32i,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fistp_m32i, bRm);
+            case 4: return IEMOP_RAISE_INVALID_OPCODE();
+            case 5: return FNIEMOP_CALL_1(iemOp_fld_m80r,   bRm);
+            case 6: return IEMOP_RAISE_INVALID_OPCODE();
+            case 7: return FNIEMOP_CALL_1(iemOp_fstp_m80r,  bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * Common worker for FPU instructions working on STn and ST0, and storing the
+ * result in STn unless IE, DE or ZE was raised.
+ *
+ * @param   bRm         Mod R/M byte.
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_stN_st0, uint8_t, bRm, PFNIEMAIMPLFPUR80, pfnAImpl)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value2,                 2);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_TWO_FPUREGS_NOT_EMPTY_REF_R80(pr80Value1, IEM_GET_MODRM_RM_8(bRm), pr80Value2, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pr80Value2);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, IEM_GET_MODRM_RM_8(bRm));
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(IEM_GET_MODRM_RM_8(bRm));
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdc 11/0. */
+FNIEMOP_DEF_1(iemOp_fadd_stN_st0,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fadd_stN_st0, "fadd stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fadd_r80_by_r80);
+}
+
+
+/** Opcode 0xdc 11/1. */
+FNIEMOP_DEF_1(iemOp_fmul_stN_st0,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fmul_stN_st0, "fmul stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fmul_r80_by_r80);
+}
+
+
+/** Opcode 0xdc 11/4. */
+FNIEMOP_DEF_1(iemOp_fsubr_stN_st0,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubr_stN_st0, "fsubr stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fsubr_r80_by_r80);
+}
+
+
+/** Opcode 0xdc 11/5. */
+FNIEMOP_DEF_1(iemOp_fsub_stN_st0,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsub_stN_st0, "fsub stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fsub_r80_by_r80);
+}
+
+
+/** Opcode 0xdc 11/6. */
+FNIEMOP_DEF_1(iemOp_fdivr_stN_st0,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivr_stN_st0, "fdivr stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fdivr_r80_by_r80);
+}
+
+
+/** Opcode 0xdc 11/7. */
+FNIEMOP_DEF_1(iemOp_fdiv_stN_st0,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdiv_stN_st0, "fdiv stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0, bRm, iemAImpl_fdiv_r80_by_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and a 64-bit floating point
+ * memory operand, and storing the result in ST0.
+ *
+ * @param   bRm         Mod R/M byte.
+ * @param   pfnImpl     Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_ST0_m64r, uint8_t, bRm, PFNIEMAIMPLFPUR64, pfnImpl)
+{
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_LOCAL(RTFLOAT64U,            r64Factor2);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Factor1,                1);
+    IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U,   pr64Factor2,    r64Factor2, 2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_FETCH_MEM_R64(r64Factor2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Factor1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnImpl, pFpuRes, pr80Factor1, pr64Factor2);
+        IEM_MC_STORE_FPU_RESULT_MEM_OP(FpuRes, 0, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(0, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdc !11/0. */
+FNIEMOP_DEF_1(iemOp_fadd_m64r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fadd_m64r, "fadd m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fadd_r80_by_r64);
+}
+
+
+/** Opcode 0xdc !11/1. */
+FNIEMOP_DEF_1(iemOp_fmul_m64r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fmul_m64r, "fmul m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fmul_r80_by_r64);
+}
+
+
+/** Opcode 0xdc !11/2. */
+FNIEMOP_DEF_1(iemOp_fcom_m64r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcom_st0_m64r, "fcom st0,m64r");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_LOCAL(RTFLOAT64U,            r64Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U,  pr64Val2,       r64Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R64(r64Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r64, pu16Fsw, pr80Value1, pr64Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdc !11/3. */
+FNIEMOP_DEF_1(iemOp_fcomp_m64r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcomp_st0_m64r, "fcomp st0,m64r");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_LOCAL(RTFLOAT64U,            r64Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U,  pr64Val2,       r64Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_R64(r64Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fcom_r80_by_r64, pu16Fsw, pr80Value1, pr64Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdc !11/4. */
+FNIEMOP_DEF_1(iemOp_fsub_m64r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsub_m64r, "fsub m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fsub_r80_by_r64);
+}
+
+
+/** Opcode 0xdc !11/5. */
+FNIEMOP_DEF_1(iemOp_fsubr_m64r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubr_m64r, "fsubr m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fsubr_r80_by_r64);
+}
+
+
+/** Opcode 0xdc !11/6. */
+FNIEMOP_DEF_1(iemOp_fdiv_m64r,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdiv_m64r, "fdiv m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fdiv_r80_by_r64);
+}
+
+
+/** Opcode 0xdc !11/7. */
+FNIEMOP_DEF_1(iemOp_fdivr_m64r, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivr_m64r, "fdivr m64r");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_ST0_m64r, bRm, iemAImpl_fdivr_r80_by_r64);
+}
+
+
+/**
+ * @opcode      0xdc
+ */
+FNIEMOP_DEF(iemOp_EscF4)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xdc & 0x7);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fadd_stN_st0,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fmul_stN_st0,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcom_stN,      bRm); /* Marked reserved, intel behavior is that of FCOM ST(i). */
+            case 3: return FNIEMOP_CALL_1(iemOp_fcomp_stN,     bRm); /* Marked reserved, intel behavior is that of FCOMP ST(i). */
+            case 4: return FNIEMOP_CALL_1(iemOp_fsubr_stN_st0, bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fsub_stN_st0,  bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fdivr_stN_st0, bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fdiv_stN_st0,  bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fadd_m64r,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fmul_m64r,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcom_m64r,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fcomp_m64r, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fsub_m64r,  bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fsubr_m64r, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fdiv_m64r,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fdivr_m64r, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xdd !11/0.
+ * @sa iemOp_fld_m32r */
+FNIEMOP_DEF_1(iemOp_fld_m64r,    uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fld_m64r, "fld m64r");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_LOCAL(RTFLOAT64U,            r64Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT, pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U,  pr64Val,    r64Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_FETCH_MEM_R64(r64Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r80_from_r64, pFpuRes, pr64Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_fisttp_m64i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisttp_m64i, "fisttp m64i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int64_t *,               pi64Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fistt_r80_to_i64, pu16Fsw, pi64Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi64Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I64_CONST_BY_REF(pi64Dst, INT64_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi64Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_fst_m64r,    uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fst_m64r, "fst m64r");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTFLOAT64U,             pr64Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pr64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r64, pu16Fsw, pr64Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr64Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(pr64Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pr64Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_fstp_m64r,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fstp_m64r, "fstp m64r");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTFLOAT64U,             pr64Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pr64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_r64, pu16Fsw, pr64Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pr64Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_NEG_QNAN_R64_BY_REF(pr64Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pr64Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_frstor,      uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(frstor, "frstor m94/108byte");
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG_CONST(IEMMODE,           enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize,  0);
+    IEM_MC_ARG(uint8_t,                 iEffSeg,                                    1);
+    IEM_MC_ARG(RTGCPTR,                 GCPtrEffSrc,                                2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_frstor, enmEffOpSize, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_fnsave,      uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fnsave, "fnsave m94/108byte");
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG_CONST(IEMMODE,           enmEffOpSize, /*=*/ pVCpu->iem.s.enmEffOpSize,  0);
+    IEM_MC_ARG(uint8_t,                 iEffSeg,                                    1);
+    IEM_MC_ARG(RTGCPTR,                 GCPtrEffDst,                                2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE(); /* Note! Implicit fninit after the save, do not use FOR_READ here! */
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_fnsave, enmEffOpSize, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+
+}
+
+/** Opcode 0xdd !11/0. */
+FNIEMOP_DEF_1(iemOp_fnstsw,      uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fnstsw_m16, "fnstsw m16");
+
+    IEM_MC_BEGIN(0, 2);
+    IEM_MC_LOCAL(uint16_t, u16Tmp);
+    IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_FETCH_FSW(u16Tmp);
+    IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Tmp);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+/** @todo Debug / drop a hint to the verifier that things may differ
+ * from REM. Seen 0x4020 (iem) vs 0x4000 (rem) at 0008:801c6b88 booting
+ * NT4SP1. (X86_FSW_PE) */
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdd 11/0. */
+FNIEMOP_DEF_1(iemOp_ffree_stN,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ffree_stN, "ffree stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    /* Note! C0, C1, C2 and C3 are documented as undefined, we leave the
+             unmodified. */
+
+    IEM_MC_BEGIN(0, 0);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_STACK_FREE(IEM_GET_MODRM_RM_8(bRm));
+    IEM_MC_UPDATE_FPU_OPCODE_IP();
+
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdd 11/1. */
+FNIEMOP_DEF_1(iemOp_fst_stN,     uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fst_st0_stN, "fst st0,stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 2);
+    IEM_MC_LOCAL(PCRTFLOAT80U,          pr80Value);
+    IEM_MC_LOCAL(IEMFPURESULT,          FpuRes);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_SET_FPU_RESULT(FpuRes, 0 /*FSW*/, pr80Value);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, IEM_GET_MODRM_RM_8(bRm));
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(IEM_GET_MODRM_RM_8(bRm));
+    IEM_MC_ENDIF();
+
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdd 11/3. */
+FNIEMOP_DEF_1(iemOp_fucom_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fucom_st0_stN, "fucom st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN, bRm, iemAImpl_fucom_r80_by_r80);
+}
+
+
+/** Opcode 0xdd 11/4. */
+FNIEMOP_DEF_1(iemOp_fucomp_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fucomp_st0_stN, "fucomp st0,stN");
+    return FNIEMOP_CALL_2(iemOpHlpFpuNoStore_st0_stN_pop, bRm, iemAImpl_fucom_r80_by_r80);
+}
+
+
+/**
+ * @opcode      0xdd
+ */
+FNIEMOP_DEF(iemOp_EscF5)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xdd & 0x7);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_ffree_stN,   bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fxch_stN,    bRm); /* Reserved, intel behavior is that of XCHG ST(i). */
+            case 2: return FNIEMOP_CALL_1(iemOp_fst_stN,     bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fstp_stN,    bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fucom_stN_st0,bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fucomp_stN,  bRm);
+            case 6: return IEMOP_RAISE_INVALID_OPCODE();
+            case 7: return IEMOP_RAISE_INVALID_OPCODE();
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fld_m64r,    bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fisttp_m64i, bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fst_m64r,    bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fstp_m64r,   bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_frstor,      bRm);
+            case 5: return IEMOP_RAISE_INVALID_OPCODE();
+            case 6: return FNIEMOP_CALL_1(iemOp_fnsave,      bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fnstsw,      bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xde 11/0. */
+FNIEMOP_DEF_1(iemOp_faddp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(faddp_stN_st0, "faddp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fadd_r80_by_r80);
+}
+
+
+/** Opcode 0xde 11/0. */
+FNIEMOP_DEF_1(iemOp_fmulp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fmulp_stN_st0, "fmulp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fmul_r80_by_r80);
+}
+
+
+/** Opcode 0xde 0xd9. */
+FNIEMOP_DEF(iemOp_fcompp)
+{
+    IEMOP_MNEMONIC(fcompp, "fcompp");
+    return FNIEMOP_CALL_1(iemOpHlpFpuNoStore_st0_st1_pop_pop, iemAImpl_fcom_r80_by_r80);
+}
+
+
+/** Opcode 0xde 11/4. */
+FNIEMOP_DEF_1(iemOp_fsubrp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubrp_stN_st0, "fsubrp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fsubr_r80_by_r80);
+}
+
+
+/** Opcode 0xde 11/5. */
+FNIEMOP_DEF_1(iemOp_fsubp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fsubp_stN_st0, "fsubp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fsub_r80_by_r80);
+}
+
+
+/** Opcode 0xde 11/6. */
+FNIEMOP_DEF_1(iemOp_fdivrp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivrp_stN_st0, "fdivrp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fdivr_r80_by_r80);
+}
+
+
+/** Opcode 0xde 11/7. */
+FNIEMOP_DEF_1(iemOp_fdivp_stN_st0, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fdivp_stN_st0, "fdivp stN,st0");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_stN_st0_pop, bRm, iemAImpl_fdiv_r80_by_r80);
+}
+
+
+/**
+ * Common worker for FPU instructions working on ST0 and an m16i, and storing
+ * the result in ST0.
+ *
+ * @param   bRm         Mod R/M byte.
+ * @param   pfnAImpl    Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_2(iemOpHlpFpu_st0_m16i, uint8_t, bRm, PFNIEMAIMPLFPUI16, pfnAImpl)
+{
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(int16_t,                   i16Val2);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,        FpuRes,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int16_t const *,   pi16Val2,       i16Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I16(i16Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(pfnAImpl, pFpuRes, pr80Value1, pi16Val2);
+        IEM_MC_STORE_FPU_RESULT(FpuRes, 0);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW(0);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xde !11/0. */
+FNIEMOP_DEF_1(iemOp_fiadd_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fiadd_m16i, "fiadd m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fiadd_r80_by_i16);
+}
+
+
+/** Opcode 0xde !11/1. */
+FNIEMOP_DEF_1(iemOp_fimul_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fimul_m16i, "fimul m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fimul_r80_by_i16);
+}
+
+
+/** Opcode 0xde !11/2. */
+FNIEMOP_DEF_1(iemOp_ficom_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ficom_st0_m16i, "ficom st0,m16i");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,                  u16Fsw);
+    IEM_MC_LOCAL(int16_t,                   i16Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,        pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int16_t const *,   pi16Val2,       i16Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I16(i16Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i16, pu16Fsw, pr80Value1, pi16Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xde !11/3. */
+FNIEMOP_DEF_1(iemOp_ficomp_m16i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ficomp_st0_m16i, "ficomp st0,m16i");
+
+    IEM_MC_BEGIN(3, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(uint16_t,                  u16Fsw);
+    IEM_MC_LOCAL(int16_t,                   i16Val2);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,        pu16Fsw,        u16Fsw,     0);
+    IEM_MC_ARG(PCRTFLOAT80U,                pr80Value1,                 1);
+    IEM_MC_ARG_LOCAL_REF(int16_t const *,   pi16Val2,       i16Val2,    2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I16(i16Val2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value1, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_ficom_r80_by_i16, pu16Fsw, pr80Value1, pi16Val2);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xde !11/4. */
+FNIEMOP_DEF_1(iemOp_fisub_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisub_m16i, "fisub m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fisub_r80_by_i16);
+}
+
+
+/** Opcode 0xde !11/5. */
+FNIEMOP_DEF_1(iemOp_fisubr_m16i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisubr_m16i, "fisubr m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fisubr_r80_by_i16);
+}
+
+
+/** Opcode 0xde !11/6. */
+FNIEMOP_DEF_1(iemOp_fidiv_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fidiv_m16i, "fidiv m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fidiv_r80_by_i16);
+}
+
+
+/** Opcode 0xde !11/7. */
+FNIEMOP_DEF_1(iemOp_fidivr_m16i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fidivr_m16i, "fidivr m16i");
+    return FNIEMOP_CALL_2(iemOpHlpFpu_st0_m16i, bRm, iemAImpl_fidivr_r80_by_i16);
+}
+
+
+/**
+ * @opcode      0xde
+ */
+FNIEMOP_DEF(iemOp_EscF6)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    pVCpu->iem.s.uFpuOpcode = RT_MAKE_U16(bRm, 0xde & 0x7);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_faddp_stN_st0, bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fmulp_stN_st0, bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fcomp_stN, bRm);
+            case 3: if (bRm == 0xd9)
+                        return FNIEMOP_CALL(iemOp_fcompp);
+                    return IEMOP_RAISE_INVALID_OPCODE();
+            case 4: return FNIEMOP_CALL_1(iemOp_fsubrp_stN_st0, bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fsubp_stN_st0, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fdivrp_stN_st0, bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fdivp_stN_st0, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fiadd_m16i,  bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fimul_m16i,  bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_ficom_m16i,  bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_ficomp_m16i, bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fisub_m16i,  bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fisubr_m16i, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fidiv_m16i,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fidivr_m16i, bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xdf 11/0.
+ * Undocument instruction, assumed to work like ffree + fincstp.  */
+FNIEMOP_DEF_1(iemOp_ffreep_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(ffreep_stN, "ffreep stN");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_STACK_FREE(IEM_GET_MODRM_RM_8(bRm));
+    IEM_MC_FPU_STACK_INC_TOP();
+    IEM_MC_UPDATE_FPU_OPCODE_IP();
+
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdf 0xe0. */
+FNIEMOP_DEF(iemOp_fnstsw_ax)
+{
+    IEMOP_MNEMONIC(fnstsw_ax, "fnstsw ax");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(uint16_t, u16Tmp);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_FETCH_FSW(u16Tmp);
+    IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Tmp);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdf 11/5. */
+FNIEMOP_DEF_1(iemOp_fucomip_st0_stN, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fucomip_st0_stN, "fucomip st0,stN");
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, IEM_GET_MODRM_RM_8(bRm), iemAImpl_fcomi_r80_by_r80, true /*fPop*/);
+}
+
+
+/** Opcode 0xdf 11/6. */
+FNIEMOP_DEF_1(iemOp_fcomip_st0_stN,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fcomip_st0_stN, "fcomip st0,stN");
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_fcomi_fucomi, IEM_GET_MODRM_RM_8(bRm), iemAImpl_fcomi_r80_by_r80, true /*fPop*/);
+}
+
+
+/** Opcode 0xdf !11/0. */
+FNIEMOP_DEF_1(iemOp_fild_m16i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fild_m16i, "fild m16i");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(int16_t,                   i16Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(int16_t const *,   pi16Val,    i16Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I16(i16Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fild_r80_from_i16, pFpuRes, pi16Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdf !11/1. */
+FNIEMOP_DEF_1(iemOp_fisttp_m16i, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fisttp_m16i, "fisttp m16i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int16_t *,               pi16Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi16Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fistt_r80_to_i16, pu16Fsw, pi16Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi16Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I16_CONST_BY_REF(pi16Dst, INT16_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi16Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdf !11/2. */
+FNIEMOP_DEF_1(iemOp_fist_m16i,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fist_m16i, "fist m16i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int16_t *,               pi16Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi16Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i16, pu16Fsw, pi16Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi16Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I16_CONST_BY_REF(pi16Dst, INT16_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi16Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdf !11/3. */
+FNIEMOP_DEF_1(iemOp_fistp_m16i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fistp_m16i, "fistp m16i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int16_t *,               pi16Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi16Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i16, pu16Fsw, pi16Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi16Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I16_CONST_BY_REF(pi16Dst, INT16_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi16Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdf !11/4. */
+FNIEMOP_DEF_1(iemOp_fbld_m80d,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fbld_m80d, "fbld m80d");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(RTPBCD80U,                 d80Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,    FpuRes,     0);
+    IEM_MC_ARG_LOCAL_REF(PCRTPBCD80U,       pd80Val,    d80Val,     1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_D80(d80Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fld_r80_from_d80, pFpuRes, pd80Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdf !11/5. */
+FNIEMOP_DEF_1(iemOp_fild_m64i,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fild_m64i, "fild m64i");
+
+    IEM_MC_BEGIN(2, 3);
+    IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+    IEM_MC_LOCAL(IEMFPURESULT,              FpuRes);
+    IEM_MC_LOCAL(int64_t,                   i64Val);
+    IEM_MC_ARG_LOCAL_REF(PIEMFPURESULT,     pFpuRes,    FpuRes, 0);
+    IEM_MC_ARG_LOCAL_REF(int64_t const *,   pi64Val,    i64Val, 1);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_FETCH_MEM_I64(i64Val, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_IS_EMPTY(7)
+        IEM_MC_CALL_FPU_AIMPL_2(iemAImpl_fild_r80_from_i64, pFpuRes, pi64Val);
+        IEM_MC_PUSH_FPU_RESULT_MEM_OP(FpuRes, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ELSE()
+        IEM_MC_FPU_STACK_PUSH_OVERFLOW_MEM_OP(pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdf !11/6. */
+FNIEMOP_DEF_1(iemOp_fbstp_m80d,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fbstp_m80d, "fbstp m80d");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(PRTPBCD80U,              pd80Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP_EX(pd80Dst, IEM_ACCESS_DATA_W, sizeof(*pd80Dst), pVCpu->iem.s.iEffSeg, GCPtrEffDst, 7 /*cbAlign*/, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fst_r80_to_d80, pu16Fsw, pd80Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pd80Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_INDEF_D80_BY_REF(pd80Dst);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pd80Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xdf !11/7. */
+FNIEMOP_DEF_1(iemOp_fistp_m64i,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fistp_m64i, "fistp m64i");
+    IEM_MC_BEGIN(3, 2);
+    IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+    IEM_MC_LOCAL(uint16_t,              u16Fsw);
+    IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Fsw,    u16Fsw, 0);
+    IEM_MC_ARG(int64_t *,               pi64Dst,            1);
+    IEM_MC_ARG(PCRTFLOAT80U,            pr80Value,          2);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+
+    IEM_MC_MEM_MAP(pi64Dst, IEM_ACCESS_DATA_W, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1 /*arg*/);
+    IEM_MC_PREPARE_FPU_USAGE();
+    IEM_MC_IF_FPUREG_NOT_EMPTY_REF_R80(pr80Value, 0)
+        IEM_MC_CALL_FPU_AIMPL_3(iemAImpl_fist_r80_to_i64, pu16Fsw, pi64Dst, pr80Value);
+        IEM_MC_MEM_COMMIT_AND_UNMAP_FOR_FPU_STORE(pi64Dst, IEM_ACCESS_DATA_W, u16Fsw);
+        IEM_MC_UPDATE_FSW_WITH_MEM_OP_THEN_POP(u16Fsw, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ELSE()
+        IEM_MC_IF_FCW_IM()
+            IEM_MC_STORE_MEM_I64_CONST_BY_REF(pi64Dst, INT64_MIN /* (integer indefinite) */);
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pi64Dst, IEM_ACCESS_DATA_W);
+        IEM_MC_ENDIF();
+        IEM_MC_FPU_STACK_UNDERFLOW_MEM_OP_THEN_POP(UINT8_MAX, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0xdf
+ */
+FNIEMOP_DEF(iemOp_EscF7)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_ffreep_stN, bRm); /* ffree + pop afterwards, since forever according to AMD. */
+            case 1: return FNIEMOP_CALL_1(iemOp_fxch_stN,   bRm); /* Reserved, behaves like FXCH ST(i) on intel. */
+            case 2: return FNIEMOP_CALL_1(iemOp_fstp_stN,   bRm); /* Reserved, behaves like FSTP ST(i) on intel. */
+            case 3: return FNIEMOP_CALL_1(iemOp_fstp_stN,   bRm); /* Reserved, behaves like FSTP ST(i) on intel. */
+            case 4: if (bRm == 0xe0)
+                        return FNIEMOP_CALL(iemOp_fnstsw_ax);
+                    return IEMOP_RAISE_INVALID_OPCODE();
+            case 5: return FNIEMOP_CALL_1(iemOp_fucomip_st0_stN, bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fcomip_st0_stN,  bRm);
+            case 7: return IEMOP_RAISE_INVALID_OPCODE();
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 0: return FNIEMOP_CALL_1(iemOp_fild_m16i,   bRm);
+            case 1: return FNIEMOP_CALL_1(iemOp_fisttp_m16i, bRm);
+            case 2: return FNIEMOP_CALL_1(iemOp_fist_m16i,   bRm);
+            case 3: return FNIEMOP_CALL_1(iemOp_fistp_m16i,  bRm);
+            case 4: return FNIEMOP_CALL_1(iemOp_fbld_m80d,   bRm);
+            case 5: return FNIEMOP_CALL_1(iemOp_fild_m64i,   bRm);
+            case 6: return FNIEMOP_CALL_1(iemOp_fbstp_m80d,  bRm);
+            case 7: return FNIEMOP_CALL_1(iemOp_fistp_m64i,  bRm);
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0xe0
+ */
+FNIEMOP_DEF(iemOp_loopne_Jb)
+{
+    IEMOP_MNEMONIC(loopne_Jb, "loopne Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U16(X86_GREG_xCX, 1);
+            IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_NOT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U32(X86_GREG_xCX, 1);
+            IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_NOT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U64(X86_GREG_xCX, 1);
+            IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_NOT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xe1
+ */
+FNIEMOP_DEF(iemOp_loope_Jb)
+{
+    IEMOP_MNEMONIC(loope_Jb, "loope Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U16(X86_GREG_xCX, 1);
+            IEM_MC_IF_CX_IS_NZ_AND_EFL_BIT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U32(X86_GREG_xCX, 1);
+            IEM_MC_IF_ECX_IS_NZ_AND_EFL_BIT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U64(X86_GREG_xCX, 1);
+            IEM_MC_IF_RCX_IS_NZ_AND_EFL_BIT_SET(X86_EFL_ZF) {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xe2
+ */
+FNIEMOP_DEF(iemOp_loop_Jb)
+{
+    IEMOP_MNEMONIC(loop_Jb, "loop Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    /** @todo Check out the \#GP case if EIP < CS.Base or EIP > CS.Limit when
+     * using the 32-bit operand size override.  How can that be restarted?  See
+     * weird pseudo code in intel manual. */
+
+    /* NB: At least Windows for Workgroups 3.11 (NDIS.386) and Windows 95 (NDIS.VXD, IOS)
+     * use LOOP $-2 to implement NdisStallExecution and other CPU stall APIs. Shortcutting
+     * the loop causes guest crashes, but when logging it's nice to skip a few million
+     * lines of useless output. */
+#if defined(LOG_ENABLED)
+    if ((LogIs3Enabled() || LogIs4Enabled()) && (-(int8_t)IEM_GET_INSTR_LEN(pVCpu) == i8Imm))
+        switch (pVCpu->iem.s.enmEffAddrMode)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0,0);
+                IEM_MC_STORE_GREG_U16_CONST(X86_GREG_xCX, 0);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0,0);
+                IEM_MC_STORE_GREG_U32_CONST(X86_GREG_xCX, 0);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0,0);
+                IEM_MC_STORE_GREG_U64_CONST(X86_GREG_xCX, 0);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+#endif
+
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,0);
+
+            IEM_MC_SUB_GREG_U16(X86_GREG_xCX, 1);
+            IEM_MC_IF_CX_IS_NZ() {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U32(X86_GREG_xCX, 1);
+            IEM_MC_IF_ECX_IS_NZ() {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_SUB_GREG_U64(X86_GREG_xCX, 1);
+            IEM_MC_IF_RCX_IS_NZ() {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ELSE() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xe3
+ */
+FNIEMOP_DEF(iemOp_jecxz_Jb)
+{
+    IEMOP_MNEMONIC(jecxz_Jb, "jecxz Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    switch (pVCpu->iem.s.enmEffAddrMode)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_IF_CX_IS_NZ() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ELSE() {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_IF_ECX_IS_NZ() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ELSE() {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0,0);
+            IEM_MC_IF_RCX_IS_NZ() {
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+            } IEM_MC_ELSE() {
+                IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+            } IEM_MC_ENDIF();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcode 0xe4 */
+FNIEMOP_DEF(iemOp_in_AL_Ib)
+{
+    IEMOP_MNEMONIC(in_AL_Ib, "in AL,Ib");
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_in, u8Imm, true /* fImm */, 1);
+}
+
+
+/** Opcode 0xe5 */
+FNIEMOP_DEF(iemOp_in_eAX_Ib)
+{
+    IEMOP_MNEMONIC(in_eAX_Ib, "in eAX,Ib");
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_in, u8Imm, true /* fImm */, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4);
+}
+
+
+/** Opcode 0xe6 */
+FNIEMOP_DEF(iemOp_out_Ib_AL)
+{
+    IEMOP_MNEMONIC(out_Ib_AL, "out Ib,AL");
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_out, u8Imm, true /* fImm */, 1);
+}
+
+
+/** Opcode 0xe7 */
+FNIEMOP_DEF(iemOp_out_Ib_eAX)
+{
+    IEMOP_MNEMONIC(out_Ib_eAX, "out Ib,eAX");
+    uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_out, u8Imm, true /* fImm */, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4);
+}
+
+
+/**
+ * @opcode      0xe8
+ */
+FNIEMOP_DEF(iemOp_call_Jv)
+{
+    IEMOP_MNEMONIC(call_Jv, "call Jv");
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+            return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_call_rel_16, (int16_t)u16Imm);
+        }
+
+        case IEMMODE_32BIT:
+        {
+            uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+            return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_call_rel_32, (int32_t)u32Imm);
+        }
+
+        case IEMMODE_64BIT:
+        {
+            uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+            return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_call_rel_64, u64Imm);
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xe9
+ */
+FNIEMOP_DEF(iemOp_jmp_Jv)
+{
+    IEMOP_MNEMONIC(jmp_Jv, "jmp Jv");
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+        {
+            int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+            IEM_MC_BEGIN(0, 0);
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        case IEMMODE_64BIT:
+        case IEMMODE_32BIT:
+        {
+            int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+            IEM_MC_BEGIN(0, 0);
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+        }
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xea
+ */
+FNIEMOP_DEF(iemOp_jmp_Ap)
+{
+    IEMOP_MNEMONIC(jmp_Ap, "jmp Ap");
+    IEMOP_HLP_NO_64BIT();
+
+    /* Decode the far pointer address and pass it on to the far call C implementation. */
+    uint32_t offSeg;
+    if (pVCpu->iem.s.enmEffOpSize != IEMMODE_16BIT)
+        IEM_OPCODE_GET_NEXT_U32(&offSeg);
+    else
+        IEM_OPCODE_GET_NEXT_U16_ZX_U32(&offSeg);
+    uint16_t uSel;  IEM_OPCODE_GET_NEXT_U16(&uSel);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_3(iemCImpl_FarJmp, uSel, offSeg, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/**
+ * @opcode      0xeb
+ */
+FNIEMOP_DEF(iemOp_jmp_Jb)
+{
+    IEMOP_MNEMONIC(jmp_Jb, "jmp Jb");
+    int8_t i8Imm; IEM_OPCODE_GET_NEXT_S8(&i8Imm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_REL_JMP_S8_AND_FINISH(i8Imm);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0xec */
+FNIEMOP_DEF(iemOp_in_AL_DX)
+{
+    IEMOP_MNEMONIC(in_AL_DX, "in  AL,DX");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_in_eAX_DX, 1);
+}
+
+
+/** Opcode 0xed */
+FNIEMOP_DEF(iemOp_in_eAX_DX)
+{
+    IEMOP_MNEMONIC(in_eAX_DX, "in  eAX,DX");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_in_eAX_DX, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4);
+}
+
+
+/** Opcode 0xee */
+FNIEMOP_DEF(iemOp_out_DX_AL)
+{
+    IEMOP_MNEMONIC(out_DX_AL, "out DX,AL");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_out_DX_eAX, 1);
+}
+
+
+/** Opcode 0xef */
+FNIEMOP_DEF(iemOp_out_DX_eAX)
+{
+    IEMOP_MNEMONIC(out_DX_eAX, "out DX,eAX");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_out_DX_eAX, pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT ? 2 : 4);
+}
+
+
+/**
+ * @opcode      0xf0
+ */
+FNIEMOP_DEF(iemOp_lock)
+{
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("lock");
+    if (!pVCpu->iem.s.fDisregardLock)
+        pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_LOCK;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0xf1
+ */
+FNIEMOP_DEF(iemOp_int1)
+{
+    IEMOP_MNEMONIC(int1, "int1"); /* icebp */
+    /** @todo Does not generate \#UD on 286, or so they say...  Was allegedly a
+     * prefix byte on 8086 and/or/maybe 80286 without meaning according to the 286
+     * LOADALL memo.  Needs some testing. */
+    IEMOP_HLP_MIN_386();
+    /** @todo testcase! */
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_int, X86_XCPT_DB, IEMINT_INT1);
+}
+
+
+/**
+ * @opcode      0xf2
+ */
+FNIEMOP_DEF(iemOp_repne)
+{
+    /* This overrides any previous REPE prefix. */
+    pVCpu->iem.s.fPrefixes &= ~IEM_OP_PRF_REPZ;
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("repne");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REPNZ;
+
+    /* For the 4 entry opcode tables, REPNZ overrides any previous
+       REPZ and operand size prefixes. */
+    pVCpu->iem.s.idxPrefix = 3;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0xf3
+ */
+FNIEMOP_DEF(iemOp_repe)
+{
+    /* This overrides any previous REPNE prefix. */
+    pVCpu->iem.s.fPrefixes &= ~IEM_OP_PRF_REPNZ;
+    IEMOP_HLP_CLEAR_REX_NOT_BEFORE_OPCODE("repe");
+    pVCpu->iem.s.fPrefixes |= IEM_OP_PRF_REPZ;
+
+    /* For the 4 entry opcode tables, REPNZ overrides any previous
+       REPNZ and operand size prefixes. */
+    pVCpu->iem.s.idxPrefix = 2;
+
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnOneByteMap[b]);
+}
+
+
+/**
+ * @opcode      0xf4
+ */
+FNIEMOP_DEF(iemOp_hlt)
+{
+    IEMOP_MNEMONIC(hlt, "hlt");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_hlt);
+}
+
+
+/**
+ * @opcode      0xf5
+ */
+FNIEMOP_DEF(iemOp_cmc)
+{
+    IEMOP_MNEMONIC(cmc, "cmc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_FLIP_EFL_BIT(X86_EFL_CF);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * Common implementation of 'inc/dec/not/neg Eb'.
+ *
+ * @param   bRm             The RM byte.
+ * @param   pImpl           The instruction implementation.
+ */
+FNIEMOP_DEF_2(iemOpCommonUnaryEb, uint8_t, bRm, PCIEMOPUNARYSIZES, pImpl)
+{
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register access */
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint8_t *,   pu8Dst, 0);
+        IEM_MC_ARG(uint32_t *,  pEFlags, 1);
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU8, pu8Dst, pEFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory access. */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,          0);
+        IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU8, pu8Dst, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU8, pu8Dst, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common implementation of 'inc/dec/not/neg Ev'.
+ *
+ * @param   bRm             The RM byte.
+ * @param   pImpl           The instruction implementation.
+ */
+FNIEMOP_DEF_2(iemOpCommonUnaryEv, uint8_t, bRm, PCIEMOPUNARYSIZES, pImpl)
+{
+    /* Registers are handled by a common worker. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        return FNIEMOP_CALL_2(iemOpCommonUnaryGReg, pImpl, IEM_GET_MODRM_RM(pVCpu, bRm));
+
+    /* Memory we do here. */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_ARG(uint16_t *,      pu16Dst,         0);
+            IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags, 1);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+            IEM_MC_FETCH_EFLAGS(EFlags);
+            if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU16, pu16Dst, pEFlags);
+            else
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU16, pu16Dst, pEFlags);
+
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+            IEM_MC_COMMIT_EFLAGS(EFlags);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_ARG(uint32_t *,      pu32Dst,         0);
+            IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags, 1);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+            IEM_MC_FETCH_EFLAGS(EFlags);
+            if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU32, pu32Dst, pEFlags);
+            else
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU32, pu32Dst, pEFlags);
+
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+            IEM_MC_COMMIT_EFLAGS(EFlags);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_ARG(uint64_t *,      pu64Dst,         0);
+            IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags, 1);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+            IEM_MC_FETCH_EFLAGS(EFlags);
+            if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnNormalU64, pu64Dst, pEFlags);
+            else
+                IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnLockedU64, pu64Dst, pEFlags);
+
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+            IEM_MC_COMMIT_EFLAGS(EFlags);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcode 0xf6 /0. */
+FNIEMOP_DEF_1(iemOp_grp3_test_Eb, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(test_Eb_Ib, "test Eb,Ib");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register access */
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,             0);
+        IEM_MC_ARG_CONST(uint8_t,   u8Src,/*=*/u8Imm,   1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,            2);
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u8, pu8Dst, u8Src, pEFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory access. */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,             0);
+        IEM_MC_ARG(uint8_t,         u8Src,              1);
+        IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,    2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+        uint8_t u8Imm; IEM_OPCODE_GET_NEXT_U8(&u8Imm);
+        IEM_MC_ASSIGN(u8Src, u8Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u8, pu8Dst, u8Src, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_R);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0xf7 /0. */
+FNIEMOP_DEF_1(iemOp_grp3_test_Ev, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(test_Ev_Iv, "test Ev,Iv");
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register access */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src,/*=*/u16Imm,     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u16, pu16Dst, u16Src, pEFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src,/*=*/u32Imm,     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u32, pu32Dst, u32Src, pEFlags);
+                /* No clearing the high dword here - test doesn't write back the result. */
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src,/*=*/u64Imm,     1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u64, pu64Dst, u64Src, pEFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory access. */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,            0);
+                IEM_MC_ARG(uint16_t,        u16Src,             1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,    2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 2);
+                uint16_t u16Imm; IEM_OPCODE_GET_NEXT_U16(&u16Imm);
+                IEM_MC_ASSIGN(u16Src, u16Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_R);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,            0);
+                IEM_MC_ARG(uint32_t,        u32Src,             1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,    2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint32_t u32Imm; IEM_OPCODE_GET_NEXT_U32(&u32Imm);
+                IEM_MC_ASSIGN(u32Src, u32Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_R);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,            0);
+                IEM_MC_ARG(uint64_t,        u64Src,             1);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,    2);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 4);
+                uint64_t u64Imm; IEM_OPCODE_GET_NEXT_S32_SX_U64(&u64Imm);
+                IEM_MC_ASSIGN(u64Src, u64Imm);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_R, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_test_u64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_R);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0xf6 /4, /5, /6 and /7. */
+FNIEMOP_DEF_2(iemOpCommonGrp3MulDivEb, uint8_t, bRm, PFNIEMAIMPLMULDIVU8, pfnU8)
+{
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register access */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint16_t *,      pu16AX,     0);
+        IEM_MC_ARG(uint8_t,         u8Value,    1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+        IEM_MC_LOCAL(int32_t,       rc);
+
+        IEM_MC_FETCH_GREG_U8(u8Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_AIMPL_3(rc, pfnU8, pu16AX, u8Value, pEFlags);
+        IEM_MC_IF_LOCAL_IS_Z(rc) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_RAISE_DIVIDE_ERROR();
+        } IEM_MC_ENDIF();
+
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory access. */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint16_t *,      pu16AX,     0);
+        IEM_MC_ARG(uint8_t,         u8Value,    1);
+        IEM_MC_ARG(uint32_t *,      pEFlags,    2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_LOCAL(int32_t,       rc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U8(u8Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_AIMPL_3(rc, pfnU8, pu16AX, u8Value, pEFlags);
+        IEM_MC_IF_LOCAL_IS_Z(rc) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_RAISE_DIVIDE_ERROR();
+        } IEM_MC_ENDIF();
+
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0xf7 /4, /5, /6 and /7. */
+FNIEMOP_DEF_2(iemOpCommonGrp3MulDivEv, uint8_t, bRm, PCIEMOPMULDIVSIZES, pImpl)
+{
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register access */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(4, 1);
+                IEM_MC_ARG(uint16_t *,      pu16AX,     0);
+                IEM_MC_ARG(uint16_t *,      pu16DX,     1);
+                IEM_MC_ARG(uint16_t,        u16Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U16(pu16DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU16, pu16AX, pu16DX, u16Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_ADVANCE_RIP_AND_FINISH();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(4, 1);
+                IEM_MC_ARG(uint32_t *,      pu32AX,     0);
+                IEM_MC_ARG(uint32_t *,      pu32DX,     1);
+                IEM_MC_ARG(uint32_t,        u32Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U32(pu32AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U32(pu32DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU32, pu32AX, pu32DX, u32Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32AX);
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32DX);
+                    IEM_MC_ADVANCE_RIP_AND_FINISH();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_BEGIN(4, 1);
+                IEM_MC_ARG(uint64_t *,      pu64AX,     0);
+                IEM_MC_ARG(uint64_t *,      pu64DX,     1);
+                IEM_MC_ARG(uint64_t,        u64Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U64(pu64AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U64(pu64DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU64, pu64AX, pu64DX, u64Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_ADVANCE_RIP_AND_FINISH();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                break;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory access. */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint16_t *,      pu16AX,     0);
+                IEM_MC_ARG(uint16_t *,      pu16DX,     1);
+                IEM_MC_ARG(uint16_t,        u16Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U16(pu16AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U16(pu16DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU16, pu16AX, pu16DX, u16Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_ADVANCE_RIP_AND_FINISH();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_32BIT:
+            {
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint32_t *,      pu32AX,     0);
+                IEM_MC_ARG(uint32_t *,      pu32DX,     1);
+                IEM_MC_ARG(uint32_t,        u32Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U32(pu32AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U32(pu32DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU32, pu32AX, pu32DX, u32Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32AX);
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32DX);
+                    IEM_MC_ADVANCE_RIP_AND_FINISH();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                break;
+            }
+
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint64_t *,      pu64AX,     0);
+                IEM_MC_ARG(uint64_t *,      pu64DX,     1);
+                IEM_MC_ARG(uint64_t,        u64Value,   2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,    3);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_LOCAL(int32_t,       rc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U64(pu64AX, X86_GREG_xAX);
+                IEM_MC_REF_GREG_U64(pu64DX, X86_GREG_xDX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_AIMPL_4(rc, pImpl->pfnU64, pu64AX, pu64DX, u64Value, pEFlags);
+                IEM_MC_IF_LOCAL_IS_Z(rc) {
+                    IEM_MC_ADVANCE_RIP_AND_FINISH();
+                } IEM_MC_ELSE() {
+                    IEM_MC_RAISE_DIVIDE_ERROR();
+                } IEM_MC_ENDIF();
+
+                IEM_MC_END();
+                break;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+/**
+ * @opcode      0xf6
+ */
+FNIEMOP_DEF(iemOp_Grp3_Eb)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0:
+            return FNIEMOP_CALL_1(iemOp_grp3_test_Eb, bRm);
+        case 1:
+/** @todo testcase: Present on <=386, most 486 (not early), Pentiums, and current CPUs too. CPUUNDOC.EXE */
+            return IEMOP_RAISE_INVALID_OPCODE();
+        case 2:
+            IEMOP_MNEMONIC(not_Eb, "not Eb");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_not);
+        case 3:
+            IEMOP_MNEMONIC(neg_Eb, "neg Eb");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_neg);
+        case 4:
+            IEMOP_MNEMONIC(mul_Eb, "mul Eb");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_mul_u8_eflags));
+        case 5:
+            IEMOP_MNEMONIC(imul_Eb, "imul Eb");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_u8_eflags));
+        case 6:
+            IEMOP_MNEMONIC(div_Eb, "div Eb");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_div_u8_eflags));
+        case 7:
+            IEMOP_MNEMONIC(idiv_Eb, "idiv Eb");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEb, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_idiv_u8_eflags));
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xf7
+ */
+FNIEMOP_DEF(iemOp_Grp3_Ev)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0:
+            return FNIEMOP_CALL_1(iemOp_grp3_test_Ev, bRm);
+        case 1:
+/** @todo testcase: Present on <=386, most 486 (not early), Pentiums, and current CPUs too. CPUUNDOC.EXE */
+            return IEMOP_RAISE_INVALID_OPCODE();
+        case 2:
+            IEMOP_MNEMONIC(not_Ev, "not Ev");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_not);
+        case 3:
+            IEMOP_MNEMONIC(neg_Ev, "neg Ev");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_neg);
+        case 4:
+            IEMOP_MNEMONIC(mul_Ev, "mul Ev");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_mul_eflags));
+        case 5:
+            IEMOP_MNEMONIC(imul_Ev, "imul Ev");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_eflags));
+        case 6:
+            IEMOP_MNEMONIC(div_Ev, "div Ev");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_div_eflags));
+        case 7:
+            IEMOP_MNEMONIC(idiv_Ev, "idiv Ev");
+            IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_OF | X86_EFL_CF);
+            return FNIEMOP_CALL_2(iemOpCommonGrp3MulDivEv, bRm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_idiv_eflags));
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xf8
+ */
+FNIEMOP_DEF(iemOp_clc)
+{
+    IEMOP_MNEMONIC(clc, "clc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_CLEAR_EFL_BIT(X86_EFL_CF);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0xf9
+ */
+FNIEMOP_DEF(iemOp_stc)
+{
+    IEMOP_MNEMONIC(stc, "stc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_SET_EFL_BIT(X86_EFL_CF);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0xfa
+ */
+FNIEMOP_DEF(iemOp_cli)
+{
+    IEMOP_MNEMONIC(cli, "cli");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_cli);
+}
+
+
+FNIEMOP_DEF(iemOp_sti)
+{
+    IEMOP_MNEMONIC(sti, "sti");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_sti);
+}
+
+
+/**
+ * @opcode      0xfc
+ */
+FNIEMOP_DEF(iemOp_cld)
+{
+    IEMOP_MNEMONIC(cld, "cld");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_CLEAR_EFL_BIT(X86_EFL_DF);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0xfd
+ */
+FNIEMOP_DEF(iemOp_std)
+{
+    IEMOP_MNEMONIC(std, "std");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(0, 0);
+    IEM_MC_SET_EFL_BIT(X86_EFL_DF);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/**
+ * @opcode      0xfe
+ */
+FNIEMOP_DEF(iemOp_Grp4)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0:
+            IEMOP_MNEMONIC(inc_Eb, "inc Eb");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_inc);
+        case 1:
+            IEMOP_MNEMONIC(dec_Eb, "dec Eb");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEb, bRm, &g_iemAImpl_dec);
+        default:
+            IEMOP_MNEMONIC(grp4_ud, "grp4-ud");
+            return IEMOP_RAISE_INVALID_OPCODE();
+    }
+}
+
+
+/**
+ * Opcode 0xff /2.
+ * @param   bRm             The RM byte.
+ */
+FNIEMOP_DEF_1(iemOp_Grp5_calln_Ev, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(calln_Ev, "calln Ev");
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* The new RIP is taken from a register. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(1, 0);
+                IEM_MC_ARG(uint16_t, u16Target, 0);
+                IEM_MC_FETCH_GREG_U16(u16Target, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_16, u16Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(1, 0);
+                IEM_MC_ARG(uint32_t, u32Target, 0);
+                IEM_MC_FETCH_GREG_U32(u32Target, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_32, u32Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(1, 0);
+                IEM_MC_ARG(uint64_t, u64Target, 0);
+                IEM_MC_FETCH_GREG_U64(u64Target, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_64, u64Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* The new RIP is taken from a register. */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(1, 1);
+                IEM_MC_ARG(uint16_t,  u16Target, 0);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_16, u16Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(1, 1);
+                IEM_MC_ARG(uint32_t,  u32Target, 0);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_32, u32Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(1, 1);
+                IEM_MC_ARG(uint64_t,  u64Target, 0);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_CALL_CIMPL_1(iemCImpl_call_64, u64Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+typedef IEM_CIMPL_DECL_TYPE_3(FNIEMCIMPLFARBRANCH, uint16_t, uSel, uint64_t, offSeg, IEMMODE, enmOpSize);
+
+FNIEMOP_DEF_2(iemOpHlp_Grp5_far_Ep, uint8_t, bRm, FNIEMCIMPLFARBRANCH *, pfnCImpl)
+{
+    /* Registers? How?? */
+    if (RT_LIKELY(IEM_IS_MODRM_MEM_MODE(bRm)))
+    { /* likely */ }
+    else
+        return IEMOP_RAISE_INVALID_OPCODE(); /* callf eax is not legal */
+
+    /* 64-bit mode: Default is 32-bit, but only intel respects a REX.W prefix. */
+    /** @todo what does VIA do? */
+    if (pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT || pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT || IEM_IS_GUEST_CPU_INTEL(pVCpu))
+    { /* likely */ }
+    else
+        pVCpu->iem.s.enmEffOpSize = IEMMODE_32BIT;
+
+    /* Far pointer loaded from memory. */
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(3, 1);
+            IEM_MC_ARG(uint16_t,        u16Sel,                         0);
+            IEM_MC_ARG(uint16_t,        offSeg,                         1);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize, IEMMODE_16BIT,    2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_MEM_U16_DISP(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 2);
+            IEM_MC_CALL_CIMPL_3(pfnCImpl, u16Sel, offSeg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(3, 1);
+            IEM_MC_ARG(uint16_t,        u16Sel,                         0);
+            IEM_MC_ARG(uint32_t,        offSeg,                         1);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize, IEMMODE_32BIT,    2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U32(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_MEM_U16_DISP(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 4);
+            IEM_MC_CALL_CIMPL_3(pfnCImpl, u16Sel, offSeg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            Assert(!IEM_IS_GUEST_CPU_AMD(pVCpu));
+            IEM_MC_BEGIN(3, 1);
+            IEM_MC_ARG(uint16_t,        u16Sel,                         0);
+            IEM_MC_ARG(uint64_t,        offSeg,                         1);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize, IEMMODE_64BIT,    2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U64(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_MEM_U16_DISP(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc, 8);
+            IEM_MC_CALL_CIMPL_3(pfnCImpl, u16Sel, offSeg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * Opcode 0xff /3.
+ * @param   bRm             The RM byte.
+ */
+FNIEMOP_DEF_1(iemOp_Grp5_callf_Ep, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(callf_Ep, "callf Ep");
+    return FNIEMOP_CALL_2(iemOpHlp_Grp5_far_Ep, bRm, iemCImpl_callf);
+}
+
+
+/**
+ * Opcode 0xff /4.
+ * @param   bRm             The RM byte.
+ */
+FNIEMOP_DEF_1(iemOp_Grp5_jmpn_Ev, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(jmpn_Ev, "jmpn Ev");
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* The new RIP is taken from a register. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Target);
+                IEM_MC_FETCH_GREG_U16(u16Target, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_SET_RIP_U16_AND_FINISH(u16Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Target);
+                IEM_MC_FETCH_GREG_U32(u32Target, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_SET_RIP_U32_AND_FINISH(u32Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Target);
+                IEM_MC_FETCH_GREG_U64(u64Target, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_SET_RIP_U64_AND_FINISH(u64Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* The new RIP is taken from a memory location. */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Target);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_SET_RIP_U16_AND_FINISH(u16Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Target);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_SET_RIP_U32_AND_FINISH(u32Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Target);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Target, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_SET_RIP_U64_AND_FINISH(u64Target);
+                IEM_MC_END()
+                return VINF_SUCCESS;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * Opcode 0xff /5.
+ * @param   bRm             The RM byte.
+ */
+FNIEMOP_DEF_1(iemOp_Grp5_jmpf_Ep, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(jmpf_Ep, "jmpf Ep");
+    return FNIEMOP_CALL_2(iemOpHlp_Grp5_far_Ep, bRm, iemCImpl_FarJmp);
+}
+
+
+/**
+ * Opcode 0xff /6.
+ * @param   bRm             The RM byte.
+ */
+FNIEMOP_DEF_1(iemOp_Grp5_push_Ev, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(push_Ev, "push Ev");
+
+    /* Registers are handled by a common worker. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        return FNIEMOP_CALL_1(iemOpCommonPushGReg, IEM_GET_MODRM_RM(pVCpu, bRm));
+
+    /* Memory we do here. */
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint16_t,  u16Src);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_PUSH_U16(u16Src);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint32_t,  u32Src);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_PUSH_U32(u32Src);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t,  u64Src);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_PUSH_U64(u64Src);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/**
+ * @opcode      0xff
+ */
+FNIEMOP_DEF(iemOp_Grp5)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0:
+            IEMOP_MNEMONIC(inc_Ev, "inc Ev");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_inc);
+        case 1:
+            IEMOP_MNEMONIC(dec_Ev, "dec Ev");
+            return FNIEMOP_CALL_2(iemOpCommonUnaryEv, bRm, &g_iemAImpl_dec);
+        case 2:
+            return FNIEMOP_CALL_1(iemOp_Grp5_calln_Ev, bRm);
+        case 3:
+            return FNIEMOP_CALL_1(iemOp_Grp5_callf_Ep, bRm);
+        case 4:
+            return FNIEMOP_CALL_1(iemOp_Grp5_jmpn_Ev, bRm);
+        case 5:
+            return FNIEMOP_CALL_1(iemOp_Grp5_jmpf_Ep, bRm);
+        case 6:
+            return FNIEMOP_CALL_1(iemOp_Grp5_push_Ev, bRm);
+        case 7:
+            IEMOP_MNEMONIC(grp5_ud, "grp5-ud");
+            return IEMOP_RAISE_INVALID_OPCODE();
+    }
+    AssertFailedReturn(VERR_IEM_IPE_3);
+}
+
+
+
+const PFNIEMOP g_apfnOneByteMap[256] =
+{
+    /* 0x00 */  iemOp_add_Eb_Gb,        iemOp_add_Ev_Gv,        iemOp_add_Gb_Eb,        iemOp_add_Gv_Ev,
+    /* 0x04 */  iemOp_add_Al_Ib,        iemOp_add_eAX_Iz,       iemOp_push_ES,          iemOp_pop_ES,
+    /* 0x08 */  iemOp_or_Eb_Gb,         iemOp_or_Ev_Gv,         iemOp_or_Gb_Eb,         iemOp_or_Gv_Ev,
+    /* 0x0c */  iemOp_or_Al_Ib,         iemOp_or_eAX_Iz,        iemOp_push_CS,          iemOp_2byteEscape,
+    /* 0x10 */  iemOp_adc_Eb_Gb,        iemOp_adc_Ev_Gv,        iemOp_adc_Gb_Eb,        iemOp_adc_Gv_Ev,
+    /* 0x14 */  iemOp_adc_Al_Ib,        iemOp_adc_eAX_Iz,       iemOp_push_SS,          iemOp_pop_SS,
+    /* 0x18 */  iemOp_sbb_Eb_Gb,        iemOp_sbb_Ev_Gv,        iemOp_sbb_Gb_Eb,        iemOp_sbb_Gv_Ev,
+    /* 0x1c */  iemOp_sbb_Al_Ib,        iemOp_sbb_eAX_Iz,       iemOp_push_DS,          iemOp_pop_DS,
+    /* 0x20 */  iemOp_and_Eb_Gb,        iemOp_and_Ev_Gv,        iemOp_and_Gb_Eb,        iemOp_and_Gv_Ev,
+    /* 0x24 */  iemOp_and_Al_Ib,        iemOp_and_eAX_Iz,       iemOp_seg_ES,           iemOp_daa,
+    /* 0x28 */  iemOp_sub_Eb_Gb,        iemOp_sub_Ev_Gv,        iemOp_sub_Gb_Eb,        iemOp_sub_Gv_Ev,
+    /* 0x2c */  iemOp_sub_Al_Ib,        iemOp_sub_eAX_Iz,       iemOp_seg_CS,           iemOp_das,
+    /* 0x30 */  iemOp_xor_Eb_Gb,        iemOp_xor_Ev_Gv,        iemOp_xor_Gb_Eb,        iemOp_xor_Gv_Ev,
+    /* 0x34 */  iemOp_xor_Al_Ib,        iemOp_xor_eAX_Iz,       iemOp_seg_SS,           iemOp_aaa,
+    /* 0x38 */  iemOp_cmp_Eb_Gb,        iemOp_cmp_Ev_Gv,        iemOp_cmp_Gb_Eb,        iemOp_cmp_Gv_Ev,
+    /* 0x3c */  iemOp_cmp_Al_Ib,        iemOp_cmp_eAX_Iz,       iemOp_seg_DS,           iemOp_aas,
+    /* 0x40 */  iemOp_inc_eAX,          iemOp_inc_eCX,          iemOp_inc_eDX,          iemOp_inc_eBX,
+    /* 0x44 */  iemOp_inc_eSP,          iemOp_inc_eBP,          iemOp_inc_eSI,          iemOp_inc_eDI,
+    /* 0x48 */  iemOp_dec_eAX,          iemOp_dec_eCX,          iemOp_dec_eDX,          iemOp_dec_eBX,
+    /* 0x4c */  iemOp_dec_eSP,          iemOp_dec_eBP,          iemOp_dec_eSI,          iemOp_dec_eDI,
+    /* 0x50 */  iemOp_push_eAX,         iemOp_push_eCX,         iemOp_push_eDX,         iemOp_push_eBX,
+    /* 0x54 */  iemOp_push_eSP,         iemOp_push_eBP,         iemOp_push_eSI,         iemOp_push_eDI,
+    /* 0x58 */  iemOp_pop_eAX,          iemOp_pop_eCX,          iemOp_pop_eDX,          iemOp_pop_eBX,
+    /* 0x5c */  iemOp_pop_eSP,          iemOp_pop_eBP,          iemOp_pop_eSI,          iemOp_pop_eDI,
+    /* 0x60 */  iemOp_pusha,            iemOp_popa__mvex,       iemOp_bound_Gv_Ma__evex, iemOp_arpl_Ew_Gw_movsx_Gv_Ev,
+    /* 0x64 */  iemOp_seg_FS,           iemOp_seg_GS,           iemOp_op_size,          iemOp_addr_size,
+    /* 0x68 */  iemOp_push_Iz,          iemOp_imul_Gv_Ev_Iz,    iemOp_push_Ib,          iemOp_imul_Gv_Ev_Ib,
+    /* 0x6c */  iemOp_insb_Yb_DX,       iemOp_inswd_Yv_DX,      iemOp_outsb_Yb_DX,      iemOp_outswd_Yv_DX,
+    /* 0x70 */  iemOp_jo_Jb,            iemOp_jno_Jb,           iemOp_jc_Jb,            iemOp_jnc_Jb,
+    /* 0x74 */  iemOp_je_Jb,            iemOp_jne_Jb,           iemOp_jbe_Jb,           iemOp_jnbe_Jb,
+    /* 0x78 */  iemOp_js_Jb,            iemOp_jns_Jb,           iemOp_jp_Jb,            iemOp_jnp_Jb,
+    /* 0x7c */  iemOp_jl_Jb,            iemOp_jnl_Jb,           iemOp_jle_Jb,           iemOp_jnle_Jb,
+    /* 0x80 */  iemOp_Grp1_Eb_Ib_80,    iemOp_Grp1_Ev_Iz,       iemOp_Grp1_Eb_Ib_82,    iemOp_Grp1_Ev_Ib,
+    /* 0x84 */  iemOp_test_Eb_Gb,       iemOp_test_Ev_Gv,       iemOp_xchg_Eb_Gb,       iemOp_xchg_Ev_Gv,
+    /* 0x88 */  iemOp_mov_Eb_Gb,        iemOp_mov_Ev_Gv,        iemOp_mov_Gb_Eb,        iemOp_mov_Gv_Ev,
+    /* 0x8c */  iemOp_mov_Ev_Sw,        iemOp_lea_Gv_M,         iemOp_mov_Sw_Ev,        iemOp_Grp1A__xop,
+    /* 0x90 */  iemOp_nop,              iemOp_xchg_eCX_eAX,     iemOp_xchg_eDX_eAX,     iemOp_xchg_eBX_eAX,
+    /* 0x94 */  iemOp_xchg_eSP_eAX,     iemOp_xchg_eBP_eAX,     iemOp_xchg_eSI_eAX,     iemOp_xchg_eDI_eAX,
+    /* 0x98 */  iemOp_cbw,              iemOp_cwd,              iemOp_call_Ap,          iemOp_wait,
+    /* 0x9c */  iemOp_pushf_Fv,         iemOp_popf_Fv,          iemOp_sahf,             iemOp_lahf,
+    /* 0xa0 */  iemOp_mov_AL_Ob,        iemOp_mov_rAX_Ov,       iemOp_mov_Ob_AL,        iemOp_mov_Ov_rAX,
+    /* 0xa4 */  iemOp_movsb_Xb_Yb,      iemOp_movswd_Xv_Yv,     iemOp_cmpsb_Xb_Yb,      iemOp_cmpswd_Xv_Yv,
+    /* 0xa8 */  iemOp_test_AL_Ib,       iemOp_test_eAX_Iz,      iemOp_stosb_Yb_AL,      iemOp_stoswd_Yv_eAX,
+    /* 0xac */  iemOp_lodsb_AL_Xb,      iemOp_lodswd_eAX_Xv,    iemOp_scasb_AL_Xb,      iemOp_scaswd_eAX_Xv,
+    /* 0xb0 */  iemOp_mov_AL_Ib,        iemOp_CL_Ib,            iemOp_DL_Ib,            iemOp_BL_Ib,
+    /* 0xb4 */  iemOp_mov_AH_Ib,        iemOp_CH_Ib,            iemOp_DH_Ib,            iemOp_BH_Ib,
+    /* 0xb8 */  iemOp_eAX_Iv,           iemOp_eCX_Iv,           iemOp_eDX_Iv,           iemOp_eBX_Iv,
+    /* 0xbc */  iemOp_eSP_Iv,           iemOp_eBP_Iv,           iemOp_eSI_Iv,           iemOp_eDI_Iv,
+    /* 0xc0 */  iemOp_Grp2_Eb_Ib,       iemOp_Grp2_Ev_Ib,       iemOp_retn_Iw,          iemOp_retn,
+    /* 0xc4 */  iemOp_les_Gv_Mp__vex3,  iemOp_lds_Gv_Mp__vex2,  iemOp_Grp11_Eb_Ib,      iemOp_Grp11_Ev_Iz,
+    /* 0xc8 */  iemOp_enter_Iw_Ib,      iemOp_leave,            iemOp_retf_Iw,          iemOp_retf,
+    /* 0xcc */  iemOp_int3,             iemOp_int_Ib,           iemOp_into,             iemOp_iret,
+    /* 0xd0 */  iemOp_Grp2_Eb_1,        iemOp_Grp2_Ev_1,        iemOp_Grp2_Eb_CL,       iemOp_Grp2_Ev_CL,
+    /* 0xd4 */  iemOp_aam_Ib,           iemOp_aad_Ib,           iemOp_salc,             iemOp_xlat,
+    /* 0xd8 */  iemOp_EscF0,            iemOp_EscF1,            iemOp_EscF2,            iemOp_EscF3,
+    /* 0xdc */  iemOp_EscF4,            iemOp_EscF5,            iemOp_EscF6,            iemOp_EscF7,
+    /* 0xe0 */  iemOp_loopne_Jb,        iemOp_loope_Jb,         iemOp_loop_Jb,          iemOp_jecxz_Jb,
+    /* 0xe4 */  iemOp_in_AL_Ib,         iemOp_in_eAX_Ib,        iemOp_out_Ib_AL,        iemOp_out_Ib_eAX,
+    /* 0xe8 */  iemOp_call_Jv,          iemOp_jmp_Jv,           iemOp_jmp_Ap,           iemOp_jmp_Jb,
+    /* 0xec */  iemOp_in_AL_DX,         iemOp_in_eAX_DX,        iemOp_out_DX_AL,        iemOp_out_DX_eAX,
+    /* 0xf0 */  iemOp_lock,             iemOp_int1,             iemOp_repne,            iemOp_repe,
+    /* 0xf4 */  iemOp_hlt,              iemOp_cmc,              iemOp_Grp3_Eb,          iemOp_Grp3_Ev,
+    /* 0xf8 */  iemOp_clc,              iemOp_stc,              iemOp_cli,              iemOp_sti,
+    /* 0xfc */  iemOp_cld,              iemOp_std,              iemOp_Grp4,             iemOp_Grp5,
+};
+
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py b/src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py
new file mode 100755
index 00000000..38ad56d7
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsPython.py
@@ -0,0 +1,3953 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id: IEMAllInstructionsPython.py $
+
+"""
+IEM instruction extractor.
+
+This script/module parses the IEMAllInstruction*.cpp.h files next to it and
+collects information about the instructions.  It can then be used to generate
+disassembler tables and tests.
+"""
+
+__copyright__ = \
+"""
+Copyright (C) 2017-2023 Oracle and/or its affiliates.
+
+This file is part of VirtualBox base platform packages, as
+available from https://www.virtualbox.org.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation, in version 3 of the
+License.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, see <https://www.gnu.org/licenses>.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
+in the VirtualBox distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+
+SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
+"""
+__version__ = "$Revision: 155244 $"
+
+# pylint: disable=anomalous-backslash-in-string
+
+# Standard python imports.
+import os
+import re
+import sys
+
+## Only the main script needs to modify the path.
+#g_ksValidationKitDir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
+#                                    'ValidationKit');
+#sys.path.append(g_ksValidationKitDir);
+#
+#from common import utils; - Windows build boxes doesn't have pywin32.
+
+# Python 3 hacks:
+if sys.version_info[0] >= 3:
+    long = int;     # pylint: disable=redefined-builtin,invalid-name
+
+
+g_kdX86EFlagsConstants = {
+    'X86_EFL_CF':          0x00000001, # RT_BIT_32(0)
+    'X86_EFL_1':           0x00000002, # RT_BIT_32(1)
+    'X86_EFL_PF':          0x00000004, # RT_BIT_32(2)
+    'X86_EFL_AF':          0x00000010, # RT_BIT_32(4)
+    'X86_EFL_ZF':          0x00000040, # RT_BIT_32(6)
+    'X86_EFL_SF':          0x00000080, # RT_BIT_32(7)
+    'X86_EFL_TF':          0x00000100, # RT_BIT_32(8)
+    'X86_EFL_IF':          0x00000200, # RT_BIT_32(9)
+    'X86_EFL_DF':          0x00000400, # RT_BIT_32(10)
+    'X86_EFL_OF':          0x00000800, # RT_BIT_32(11)
+    'X86_EFL_IOPL':        0x00003000, # (RT_BIT_32(12) | RT_BIT_32(13))
+    'X86_EFL_NT':          0x00004000, # RT_BIT_32(14)
+    'X86_EFL_RF':          0x00010000, # RT_BIT_32(16)
+    'X86_EFL_VM':          0x00020000, # RT_BIT_32(17)
+    'X86_EFL_AC':          0x00040000, # RT_BIT_32(18)
+    'X86_EFL_VIF':         0x00080000, # RT_BIT_32(19)
+    'X86_EFL_VIP':         0x00100000, # RT_BIT_32(20)
+    'X86_EFL_ID':          0x00200000, # RT_BIT_32(21)
+    'X86_EFL_LIVE_MASK':   0x003f7fd5, # UINT32_C(0x003f7fd5)
+    'X86_EFL_RA1_MASK':    0x00000002, # RT_BIT_32(1)
+};
+
+## EFlags values allowed in \@opfltest, \@opflmodify, \@opflundef, \@opflset, and \@opflclear.
+g_kdEFlagsMnemonics = {
+    # Debugger flag notation (sorted by value):
+    'cf':   'X86_EFL_CF',   ##< Carry Flag.
+    'nc':  '!X86_EFL_CF',   ##< No Carry.
+
+    'po':   'X86_EFL_PF',   ##< Parity Pdd.
+    'pe':  '!X86_EFL_PF',   ##< Parity Even.
+
+    'af':   'X86_EFL_AF',   ##< Aux Flag.
+    'na':  '!X86_EFL_AF',   ##< No Aux.
+
+    'zr':   'X86_EFL_ZF',   ##< ZeRo.
+    'nz':  '!X86_EFL_ZF',   ##< No Zero.
+
+    'ng':   'X86_EFL_SF',   ##< NeGative (sign).
+    'pl':  '!X86_EFL_SF',   ##< PLuss (sign).
+
+    'tf':   'X86_EFL_TF',   ##< Trap flag.
+
+    'ei':   'X86_EFL_IF',   ##< Enabled Interrupts.
+    'di':  '!X86_EFL_IF',   ##< Disabled Interrupts.
+
+    'dn':   'X86_EFL_DF',   ##< DowN (string op direction).
+    'up':  '!X86_EFL_DF',   ##< UP (string op direction).
+
+    'ov':   'X86_EFL_OF',   ##< OVerflow.
+    'nv':  '!X86_EFL_OF',   ##< No Overflow.
+
+    'nt':   'X86_EFL_NT',   ##< Nested Task.
+    'rf':   'X86_EFL_RF',   ##< Resume Flag.
+    'vm':   'X86_EFL_VM',   ##< Virtual-8086 Mode.
+    'ac':   'X86_EFL_AC',   ##< Alignment Check.
+    'vif':  'X86_EFL_VIF',  ##< Virtual Interrupt Flag.
+    'vip':  'X86_EFL_VIP',  ##< Virtual Interrupt Pending.
+
+    # Reference manual notation not covered above (sorted by value):
+    'pf':   'X86_EFL_PF',
+    'zf':   'X86_EFL_ZF',
+    'sf':   'X86_EFL_SF',
+    'if':   'X86_EFL_IF',
+    'df':   'X86_EFL_DF',
+    'of':   'X86_EFL_OF',
+    'iopl': 'X86_EFL_IOPL',
+    'id':   'X86_EFL_ID',
+};
+
+## Constants and values for CR0.
+g_kdX86Cr0Constants = {
+    'X86_CR0_PE':           0x00000001, # RT_BIT_32(0)
+    'X86_CR0_MP':           0x00000002, # RT_BIT_32(1)
+    'X86_CR0_EM':           0x00000004, # RT_BIT_32(2)
+    'X86_CR0_TS':           0x00000008, # RT_BIT_32(3)
+    'X86_CR0_ET':           0x00000010, # RT_BIT_32(4)
+    'X86_CR0_NE':           0x00000020, # RT_BIT_32(5)
+    'X86_CR0_WP':           0x00010000, # RT_BIT_32(16)
+    'X86_CR0_AM':           0x00040000, # RT_BIT_32(18)
+    'X86_CR0_NW':           0x20000000, # RT_BIT_32(29)
+    'X86_CR0_CD':           0x40000000, # RT_BIT_32(30)
+    'X86_CR0_PG':           0x80000000, # RT_BIT_32(31)
+};
+
+## Constants and values for CR4.
+g_kdX86Cr4Constants = {
+    'X86_CR4_VME':          0x00000001, # RT_BIT_32(0)
+    'X86_CR4_PVI':          0x00000002, # RT_BIT_32(1)
+    'X86_CR4_TSD':          0x00000004, # RT_BIT_32(2)
+    'X86_CR4_DE':           0x00000008, # RT_BIT_32(3)
+    'X86_CR4_PSE':          0x00000010, # RT_BIT_32(4)
+    'X86_CR4_PAE':          0x00000020, # RT_BIT_32(5)
+    'X86_CR4_MCE':          0x00000040, # RT_BIT_32(6)
+    'X86_CR4_PGE':          0x00000080, # RT_BIT_32(7)
+    'X86_CR4_PCE':          0x00000100, # RT_BIT_32(8)
+    'X86_CR4_OSFXSR':       0x00000200, # RT_BIT_32(9)
+    'X86_CR4_OSXMMEEXCPT':  0x00000400, # RT_BIT_32(10)
+    'X86_CR4_VMXE':         0x00002000, # RT_BIT_32(13)
+    'X86_CR4_SMXE':         0x00004000, # RT_BIT_32(14)
+    'X86_CR4_PCIDE':        0x00020000, # RT_BIT_32(17)
+    'X86_CR4_OSXSAVE':      0x00040000, # RT_BIT_32(18)
+    'X86_CR4_SMEP':         0x00100000, # RT_BIT_32(20)
+    'X86_CR4_SMAP':         0x00200000, # RT_BIT_32(21)
+    'X86_CR4_PKE':          0x00400000, # RT_BIT_32(22)
+};
+
+## XSAVE components (XCR0).
+g_kdX86XSaveCConstants = {
+    'XSAVE_C_X87':          0x00000001,
+    'XSAVE_C_SSE':          0x00000002,
+    'XSAVE_C_YMM':          0x00000004,
+    'XSAVE_C_BNDREGS':      0x00000008,
+    'XSAVE_C_BNDCSR':       0x00000010,
+    'XSAVE_C_OPMASK':       0x00000020,
+    'XSAVE_C_ZMM_HI256':    0x00000040,
+    'XSAVE_C_ZMM_16HI':     0x00000080,
+    'XSAVE_C_PKRU':         0x00000200,
+    'XSAVE_C_LWP':          0x4000000000000000,
+    'XSAVE_C_X':            0x8000000000000000,
+    'XSAVE_C_ALL_AVX':      0x000000c4, # For clearing all AVX bits.
+    'XSAVE_C_ALL_AVX_SSE':  0x000000c6, # For clearing all AVX and SSE bits.
+};
+
+
+## \@op[1-4] locations
+g_kdOpLocations = {
+    'reg':      [], ## modrm.reg
+    'rm':       [], ## modrm.rm
+    'imm':      [], ## immediate instruction data
+    'vvvv':     [], ## VEX.vvvv
+
+    # fixed registers.
+    'AL':       [],
+    'rAX':      [],
+    'rDX':      [],
+    'rSI':      [],
+    'rDI':      [],
+    'rFLAGS':   [],
+    'CS':       [],
+    'DS':       [],
+    'ES':       [],
+    'FS':       [],
+    'GS':       [],
+    'SS':       [],
+};
+
+## \@op[1-4] types
+##
+## Value fields:
+##    - 0: the normal IDX_ParseXXX handler (IDX_UseModRM == IDX_ParseModRM).
+##    - 1: the location (g_kdOpLocations).
+##    - 2: disassembler format string version of the type.
+##    - 3: disassembler OP_PARAM_XXX (XXX only).
+##    - 4: IEM form matching instruction.
+##
+## Note! See the A.2.1 in SDM vol 2 for the type names.
+g_kdOpTypes = {
+    # Fixed addresses
+    'Ap':           ( 'IDX_ParseImmAddrF',  'imm',    '%Ap',  'Ap',      'FIXED', ),
+
+    # ModR/M.rm
+    'Eb':           ( 'IDX_UseModRM',       'rm',     '%Eb',  'Eb',      'RM',    ),
+    'Ed':           ( 'IDX_UseModRM',       'rm',     '%Ed',  'Ed',      'RM',    ),
+    'Ed_WO':        ( 'IDX_UseModRM',       'rm',     '%Ed',  'Ed',      'RM',    ),
+    'Eq':           ( 'IDX_UseModRM',       'rm',     '%Eq',  'Eq',      'RM',    ),
+    'Eq_WO':        ( 'IDX_UseModRM',       'rm',     '%Eq',  'Eq',      'RM',    ),
+    'Ew':           ( 'IDX_UseModRM',       'rm',     '%Ew',  'Ew',      'RM',    ),
+    'Ev':           ( 'IDX_UseModRM',       'rm',     '%Ev',  'Ev',      'RM',    ),
+    'Ey':           ( 'IDX_UseModRM',       'rm',     '%Ey',  'Ey',      'RM',    ),
+    'Qd':           ( 'IDX_UseModRM',       'rm',     '%Qd',  'Qd',      'RM',    ),
+    'Qq':           ( 'IDX_UseModRM',       'rm',     '%Qq',  'Qq',      'RM',    ),
+    'Qq_WO':        ( 'IDX_UseModRM',       'rm',     '%Qq',  'Qq',      'RM',    ),
+    'Wss':          ( 'IDX_UseModRM',       'rm',     '%Wss', 'Wss',     'RM',    ),
+    'Wss_WO':       ( 'IDX_UseModRM',       'rm',     '%Wss', 'Wss',     'RM',    ),
+    'Wsd':          ( 'IDX_UseModRM',       'rm',     '%Wsd', 'Wsd',     'RM',    ),
+    'Wsd_WO':       ( 'IDX_UseModRM',       'rm',     '%Wsd', 'Wsd',     'RM',    ),
+    'Wps':          ( 'IDX_UseModRM',       'rm',     '%Wps', 'Wps',     'RM',    ),
+    'Wps_WO':       ( 'IDX_UseModRM',       'rm',     '%Wps', 'Wps',     'RM',    ),
+    'Wpd':          ( 'IDX_UseModRM',       'rm',     '%Wpd', 'Wpd',     'RM',    ),
+    'Wpd_WO':       ( 'IDX_UseModRM',       'rm',     '%Wpd', 'Wpd',     'RM',    ),
+    'Wdq':          ( 'IDX_UseModRM',       'rm',     '%Wdq', 'Wdq',     'RM',    ),
+    'Wdq_WO':       ( 'IDX_UseModRM',       'rm',     '%Wdq', 'Wdq',     'RM',    ),
+    'Wq':           ( 'IDX_UseModRM',       'rm',     '%Wq',  'Wq',      'RM',    ),
+    'Wq_WO':        ( 'IDX_UseModRM',       'rm',     '%Wq',  'Wq',      'RM',    ),
+    'WqZxReg_WO':   ( 'IDX_UseModRM',       'rm',     '%Wq',  'Wq',      'RM',    ),
+    'Wx':           ( 'IDX_UseModRM',       'rm',     '%Wx',  'Wx',      'RM',    ),
+    'Wx_WO':        ( 'IDX_UseModRM',       'rm',     '%Wx',  'Wx',      'RM',    ),
+
+    # ModR/M.rm - register only.
+    'Uq':           ( 'IDX_UseModRM',       'rm',     '%Uq',  'Uq',      'REG'    ),
+    'UqHi':         ( 'IDX_UseModRM',       'rm',     '%Uq',  'UqHi',    'REG'    ),
+    'Uss':          ( 'IDX_UseModRM',       'rm',     '%Uss', 'Uss',     'REG'    ),
+    'Uss_WO':       ( 'IDX_UseModRM',       'rm',     '%Uss', 'Uss',     'REG'    ),
+    'Usd':          ( 'IDX_UseModRM',       'rm',     '%Usd', 'Usd',     'REG'    ),
+    'Usd_WO':       ( 'IDX_UseModRM',       'rm',     '%Usd', 'Usd',     'REG'    ),
+    'Ux':           ( 'IDX_UseModRM',       'rm',     '%Ux',  'Ux',      'REG'    ),
+    'Nq':           ( 'IDX_UseModRM',       'rm',     '%Qq',  'Nq',      'REG'    ),
+
+    # ModR/M.rm - memory only.
+    'Ma':           ( 'IDX_UseModRM',       'rm',     '%Ma',  'Ma',      'MEM',   ), ##< Only used by BOUND.
+    'Mb_RO':        ( 'IDX_UseModRM',       'rm',     '%Mb',  'Mb',      'MEM',   ),
+    'Md':           ( 'IDX_UseModRM',       'rm',     '%Md',  'Md',      'MEM',   ),
+    'Md_RO':        ( 'IDX_UseModRM',       'rm',     '%Md',  'Md',      'MEM',   ),
+    'Md_WO':        ( 'IDX_UseModRM',       'rm',     '%Md',  'Md',      'MEM',   ),
+    'Mdq':          ( 'IDX_UseModRM',       'rm',     '%Mdq', 'Mdq',     'MEM',   ),
+    'Mdq_WO':       ( 'IDX_UseModRM',       'rm',     '%Mdq', 'Mdq',     'MEM',   ),
+    'Mq':           ( 'IDX_UseModRM',       'rm',     '%Mq',  'Mq',      'MEM',   ),
+    'Mq_WO':        ( 'IDX_UseModRM',       'rm',     '%Mq',  'Mq',      'MEM',   ),
+    'Mps_WO':       ( 'IDX_UseModRM',       'rm',     '%Mps', 'Mps',     'MEM',   ),
+    'Mpd_WO':       ( 'IDX_UseModRM',       'rm',     '%Mpd', 'Mpd',     'MEM',   ),
+    'Mx':           ( 'IDX_UseModRM',       'rm',     '%Mx',  'Mx',      'MEM',   ),
+    'Mx_WO':        ( 'IDX_UseModRM',       'rm',     '%Mx',  'Mx',      'MEM',   ),
+    'M_RO':         ( 'IDX_UseModRM',       'rm',     '%M',   'M',       'MEM',   ),
+    'M_RW':         ( 'IDX_UseModRM',       'rm',     '%M',   'M',       'MEM',   ),
+
+    # ModR/M.reg
+    'Gb':           ( 'IDX_UseModRM',       'reg',    '%Gb',  'Gb',      '',      ),
+    'Gw':           ( 'IDX_UseModRM',       'reg',    '%Gw',  'Gw',      '',      ),
+    'Gd':           ( 'IDX_UseModRM',       'reg',    '%Gd',  'Gd',      '',      ),
+    'Gv':           ( 'IDX_UseModRM',       'reg',    '%Gv',  'Gv',      '',      ),
+    'Gv_RO':        ( 'IDX_UseModRM',       'reg',    '%Gv',  'Gv',      '',      ),
+    'Gy':           ( 'IDX_UseModRM',       'reg',    '%Gy',  'Gy',      '',      ),
+    'Pd':           ( 'IDX_UseModRM',       'reg',    '%Pd',  'Pd',      '',      ),
+    'PdZx_WO':      ( 'IDX_UseModRM',       'reg',    '%Pd',  'PdZx',    '',      ),
+    'Pq':           ( 'IDX_UseModRM',       'reg',    '%Pq',  'Pq',      '',      ),
+    'Pq_WO':        ( 'IDX_UseModRM',       'reg',    '%Pq',  'Pq',      '',      ),
+    'Vd':           ( 'IDX_UseModRM',       'reg',    '%Vd',  'Vd',      '',      ),
+    'Vd_WO':        ( 'IDX_UseModRM',       'reg',    '%Vd',  'Vd',      '',      ),
+    'VdZx_WO':      ( 'IDX_UseModRM',       'reg',    '%Vd',  'Vd',      '',      ),
+    'Vdq':          ( 'IDX_UseModRM',       'reg',    '%Vdq', 'Vdq',     '',      ),
+    'Vss':          ( 'IDX_UseModRM',       'reg',    '%Vss', 'Vss',     '',      ),
+    'Vss_WO':       ( 'IDX_UseModRM',       'reg',    '%Vss', 'Vss',     '',      ),
+    'VssZx_WO':     ( 'IDX_UseModRM',       'reg',    '%Vss', 'Vss',     '',      ),
+    'Vsd':          ( 'IDX_UseModRM',       'reg',    '%Vsd', 'Vsd',     '',      ),
+    'Vsd_WO':       ( 'IDX_UseModRM',       'reg',    '%Vsd', 'Vsd',     '',      ),
+    'VsdZx_WO':     ( 'IDX_UseModRM',       'reg',    '%Vsd', 'Vsd',     '',      ),
+    'Vps':          ( 'IDX_UseModRM',       'reg',    '%Vps', 'Vps',     '',      ),
+    'Vps_WO':       ( 'IDX_UseModRM',       'reg',    '%Vps', 'Vps',     '',      ),
+    'Vpd':          ( 'IDX_UseModRM',       'reg',    '%Vpd', 'Vpd',     '',      ),
+    'Vpd_WO':       ( 'IDX_UseModRM',       'reg',    '%Vpd', 'Vpd',     '',      ),
+    'Vq':           ( 'IDX_UseModRM',       'reg',    '%Vq',  'Vq',      '',      ),
+    'Vq_WO':        ( 'IDX_UseModRM',       'reg',    '%Vq',  'Vq',      '',      ),
+    'Vdq_WO':       ( 'IDX_UseModRM',       'reg',    '%Vdq', 'Vdq',     '',      ),
+    'VqHi':         ( 'IDX_UseModRM',       'reg',    '%Vdq', 'VdqHi',   '',      ),
+    'VqHi_WO':      ( 'IDX_UseModRM',       'reg',    '%Vdq', 'VdqHi',   '',      ),
+    'VqZx_WO':      ( 'IDX_UseModRM',       'reg',    '%Vq',  'VqZx',    '',      ),
+    'Vx':           ( 'IDX_UseModRM',       'reg',    '%Vx',  'Vx',      '',      ),
+    'Vx_WO':        ( 'IDX_UseModRM',       'reg',    '%Vx',  'Vx',      '',      ),
+
+    # VEX.vvvv
+    'By':           ( 'IDX_UseModRM',       'vvvv',   '%By',  'By',      'V',     ),
+    'Hps':          ( 'IDX_UseModRM',       'vvvv',   '%Hps', 'Hps',     'V',     ),
+    'Hpd':          ( 'IDX_UseModRM',       'vvvv',   '%Hpd', 'Hpd',     'V',     ),
+    'HssHi':        ( 'IDX_UseModRM',       'vvvv',   '%Hx',  'HssHi',   'V',     ),
+    'HsdHi':        ( 'IDX_UseModRM',       'vvvv',   '%Hx',  'HsdHi',   'V',     ),
+    'Hq':           ( 'IDX_UseModRM',       'vvvv',   '%Hq',  'Hq',      'V',     ),
+    'HqHi':         ( 'IDX_UseModRM',       'vvvv',   '%Hq',  'HqHi',    'V',     ),
+    'Hx':           ( 'IDX_UseModRM',       'vvvv',   '%Hx',  'Hx',      'V',     ),
+
+    # Immediate values.
+    'Ib':           ( 'IDX_ParseImmByte',   'imm',    '%Ib',  'Ib',   '', ), ##< NB! Could be IDX_ParseImmByteSX for some instrs.
+    'Iw':           ( 'IDX_ParseImmUshort', 'imm',    '%Iw',  'Iw',      '',      ),
+    'Id':           ( 'IDX_ParseImmUlong',  'imm',    '%Id',  'Id',      '',      ),
+    'Iq':           ( 'IDX_ParseImmQword',  'imm',    '%Iq',  'Iq',      '',      ),
+    'Iv':           ( 'IDX_ParseImmV',      'imm',    '%Iv',  'Iv',      '',      ), ##< o16: word, o32: dword, o64: qword
+    'Iz':           ( 'IDX_ParseImmZ',      'imm',    '%Iz',  'Iz',      '',      ), ##< o16: word, o32|o64:dword
+
+    # Address operands (no ModR/M).
+    'Ob':           ( 'IDX_ParseImmAddr',   'imm',    '%Ob',  'Ob',      '',      ),
+    'Ov':           ( 'IDX_ParseImmAddr',   'imm',    '%Ov',  'Ov',      '',      ),
+
+    # Relative jump targets
+    'Jb':           ( 'IDX_ParseImmBRel',   'imm',    '%Jb',  'Jb',      '',      ),
+    'Jv':           ( 'IDX_ParseImmVRel',   'imm',    '%Jv',  'Jv',      '',      ),
+
+    # DS:rSI
+    'Xb':           ( 'IDX_ParseXb',        'rSI',    '%eSI', 'Xb',      '',      ),
+    'Xv':           ( 'IDX_ParseXv',        'rSI',    '%eSI', 'Xv',      '',      ),
+    # ES:rDI
+    'Yb':           ( 'IDX_ParseYb',        'rDI',    '%eDI', 'Yb',      '',      ),
+    'Yv':           ( 'IDX_ParseYv',        'rDI',    '%eDI', 'Yv',      '',      ),
+
+    'Fv':           ( 'IDX_ParseFixedReg',  'rFLAGS', '%Fv',  'Fv',      '',      ),
+
+    # Fixed registers.
+    'AL':           ( 'IDX_ParseFixedReg',  'AL',     'al',   'REG_AL',  '',      ),
+    'rAX':          ( 'IDX_ParseFixedReg',  'rAX',    '%eAX', 'REG_EAX', '',      ),
+    'rDX':          ( 'IDX_ParseFixedReg',  'rDX',    '%eDX', 'REG_EDX', '',      ),
+    'CS':           ( 'IDX_ParseFixedReg',  'CS',     'cs',   'REG_CS',  '',      ), # 8086: push CS
+    'DS':           ( 'IDX_ParseFixedReg',  'DS',     'ds',   'REG_DS',  '',      ),
+    'ES':           ( 'IDX_ParseFixedReg',  'ES',     'es',   'REG_ES',  '',      ),
+    'FS':           ( 'IDX_ParseFixedReg',  'FS',     'fs',   'REG_FS',  '',      ),
+    'GS':           ( 'IDX_ParseFixedReg',  'GS',     'gs',   'REG_GS',  '',      ),
+    'SS':           ( 'IDX_ParseFixedReg',  'SS',     'ss',   'REG_SS',  '',      ),
+};
+
+# IDX_ParseFixedReg
+# IDX_ParseVexDest
+
+
+## IEMFORM_XXX mappings.
+g_kdIemForms = {     # sEncoding,   [ sWhere1, ... ]         opcodesub      ),
+    'RM':           ( 'ModR/M',     [ 'reg', 'rm' ],         '',            ),
+    'RM_REG':       ( 'ModR/M',     [ 'reg', 'rm' ],         '11 mr/reg',   ),
+    'RM_MEM':       ( 'ModR/M',     [ 'reg', 'rm' ],         '!11 mr/reg',  ),
+    'RMI':          ( 'ModR/M',     [ 'reg', 'rm', 'imm' ],  '',            ),
+    'RMI_REG':      ( 'ModR/M',     [ 'reg', 'rm', 'imm' ],  '11 mr/reg',   ),
+    'RMI_MEM':      ( 'ModR/M',     [ 'reg', 'rm', 'imm' ],  '!11 mr/reg',  ),
+    'MR':           ( 'ModR/M',     [ 'rm', 'reg' ],         '',            ),
+    'MR_REG':       ( 'ModR/M',     [ 'rm', 'reg' ],         '11 mr/reg',   ),
+    'MR_MEM':       ( 'ModR/M',     [ 'rm', 'reg' ],         '!11 mr/reg',  ),
+    'MRI':          ( 'ModR/M',     [ 'rm', 'reg', 'imm' ],  '',            ),
+    'MRI_REG':      ( 'ModR/M',     [ 'rm', 'reg', 'imm' ],  '11 mr/reg',   ),
+    'MRI_MEM':      ( 'ModR/M',     [ 'rm', 'reg', 'imm' ],  '!11 mr/reg',  ),
+    'M':            ( 'ModR/M',     [ 'rm', ],               '',            ),
+    'M_REG':        ( 'ModR/M',     [ 'rm', ],               '',            ),
+    'M_MEM':        ( 'ModR/M',     [ 'rm', ],               '',            ),
+    'R':            ( 'ModR/M',     [ 'reg', ],              '',            ),
+
+    'VEX_RM':       ( 'VEX.ModR/M', [ 'reg', 'rm' ],         '',            ),
+    'VEX_RM_REG':   ( 'VEX.ModR/M', [ 'reg', 'rm' ],         '11 mr/reg',   ),
+    'VEX_RM_MEM':   ( 'VEX.ModR/M', [ 'reg', 'rm' ],         '!11 mr/reg',  ),
+    'VEX_MR':       ( 'VEX.ModR/M', [ 'rm', 'reg' ],         '',            ),
+    'VEX_MR_REG':   ( 'VEX.ModR/M', [ 'rm', 'reg' ],         '11 mr/reg',   ),
+    'VEX_MR_MEM':   ( 'VEX.ModR/M', [ 'rm', 'reg' ],         '!11 mr/reg',  ),
+    'VEX_M':        ( 'VEX.ModR/M', [ 'rm', ],               '' ),
+    'VEX_M_REG':    ( 'VEX.ModR/M', [ 'rm', ],               '' ),
+    'VEX_M_MEM':    ( 'VEX.ModR/M', [ 'rm', ],               '' ),
+    'VEX_R':        ( 'VEX.ModR/M', [ 'reg', ],              '' ),
+    'VEX_RVM':      ( 'VEX.ModR/M', [ 'reg', 'vvvv', 'rm' ], '',            ),
+    'VEX_RVM_REG':  ( 'VEX.ModR/M', [ 'reg', 'vvvv', 'rm' ], '11 mr/reg',   ),
+    'VEX_RVM_MEM':  ( 'VEX.ModR/M', [ 'reg', 'vvvv', 'rm' ], '!11 mr/reg',  ),
+    'VEX_RMV':      ( 'VEX.ModR/M', [ 'reg', 'rm', 'vvvv' ], '',            ),
+    'VEX_RMV_REG':  ( 'VEX.ModR/M', [ 'reg', 'rm', 'vvvv' ], '11 mr/reg',   ),
+    'VEX_RMV_MEM':  ( 'VEX.ModR/M', [ 'reg', 'rm', 'vvvv' ], '!11 mr/reg',  ),
+    'VEX_RMI':      ( 'VEX.ModR/M', [ 'reg', 'rm', 'imm' ],  '',            ),
+    'VEX_RMI_REG':  ( 'VEX.ModR/M', [ 'reg', 'rm', 'imm' ],  '11 mr/reg',   ),
+    'VEX_RMI_MEM':  ( 'VEX.ModR/M', [ 'reg', 'rm', 'imm' ],  '!11 mr/reg',  ),
+    'VEX_MVR':      ( 'VEX.ModR/M', [ 'rm', 'vvvv', 'reg' ], '',            ),
+    'VEX_MVR_REG':  ( 'VEX.ModR/M', [ 'rm', 'vvvv', 'reg' ], '11 mr/reg',   ),
+    'VEX_MVR_MEM':  ( 'VEX.ModR/M', [ 'rm', 'vvvv', 'reg' ], '!11 mr/reg',  ),
+
+    'VEX_VM':       ( 'VEX.ModR/M', [ 'vvvv', 'rm' ],        '',            ),
+    'VEX_VM_REG':   ( 'VEX.ModR/M', [ 'vvvv', 'rm' ],        '11 mr/reg',   ),
+    'VEX_VM_MEM':   ( 'VEX.ModR/M', [ 'vvvv', 'rm' ],        '!11 mr/reg',  ),
+
+    'FIXED':        ( 'fixed',      None,                    '',            ),
+};
+
+## \@oppfx values.
+g_kdPrefixes = {
+    'none': [],
+    '0x66': [],
+    '0xf3': [],
+    '0xf2': [],
+};
+
+## Special \@opcode tag values.
+g_kdSpecialOpcodes = {
+    '/reg':         [],
+    'mr/reg':       [],
+    '11 /reg':      [],
+    '!11 /reg':     [],
+    '11 mr/reg':    [],
+    '!11 mr/reg':   [],
+};
+
+## Special \@opcodesub tag values.
+## The first value is the real value for aliases.
+## The second value is for bs3cg1.
+g_kdSubOpcodes = {
+    'none':                 [ None,                 '',         ],
+    '11 mr/reg':            [ '11 mr/reg',          '',         ],
+    '11':                   [ '11 mr/reg',          '',         ], ##< alias
+    '!11 mr/reg':           [ '!11 mr/reg',         '',         ],
+    '!11':                  [ '!11 mr/reg',         '',         ], ##< alias
+    'rex.w=0':              [ 'rex.w=0',            'WZ',       ],
+    'w=0':                  [ 'rex.w=0',            '',         ], ##< alias
+    'rex.w=1':              [ 'rex.w=1',            'WNZ',      ],
+    'w=1':                  [ 'rex.w=1',            '',         ], ##< alias
+    'vex.l=0':              [ 'vex.l=0',            'L0',       ],
+    'vex.l=1':              [ 'vex.l=0',            'L1',       ],
+    '11 mr/reg vex.l=0':    [ '11 mr/reg vex.l=0',  'L0',       ],
+    '11 mr/reg vex.l=1':    [ '11 mr/reg vex.l=1',  'L1',       ],
+    '!11 mr/reg vex.l=0':   [ '!11 mr/reg vex.l=0', 'L0',       ],
+    '!11 mr/reg vex.l=1':   [ '!11 mr/reg vex.l=1', 'L1',       ],
+};
+
+## Valid values for \@openc
+g_kdEncodings = {
+    'ModR/M':       [ 'BS3CG1ENC_MODRM', ],     ##< ModR/M
+    'VEX.ModR/M':   [ 'BS3CG1ENC_VEX_MODRM', ], ##< VEX...ModR/M
+    'fixed':        [ 'BS3CG1ENC_FIXED', ],     ##< Fixed encoding (address, registers, unused, etc).
+    'VEX.fixed':    [ 'BS3CG1ENC_VEX_FIXED', ], ##< VEX + fixed encoding (address, registers, unused, etc).
+    'prefix':       [ None, ],                  ##< Prefix
+};
+
+## \@opunused, \@opinvalid, \@opinvlstyle
+g_kdInvalidStyles = {
+    'immediate':                [], ##< CPU stops decoding immediately after the opcode.
+    'vex.modrm':                [], ##< VEX+ModR/M, everyone.
+    'intel-modrm':              [], ##< Intel decodes ModR/M.
+    'intel-modrm-imm8':         [], ##< Intel decodes ModR/M and an 8-byte immediate.
+    'intel-opcode-modrm':       [], ##< Intel decodes another opcode byte followed by ModR/M. (Unused extension tables.)
+    'intel-opcode-modrm-imm8':  [], ##< Intel decodes another opcode byte followed by ModR/M and an 8-byte immediate.
+};
+
+g_kdCpuNames = {
+    '8086':     (),
+    '80186':    (),
+    '80286':    (),
+    '80386':    (),
+    '80486':    (),
+};
+
+## \@opcpuid
+g_kdCpuIdFlags = {
+    'vme':          'X86_CPUID_FEATURE_EDX_VME',
+    'tsc':          'X86_CPUID_FEATURE_EDX_TSC',
+    'msr':          'X86_CPUID_FEATURE_EDX_MSR',
+    'cx8':          'X86_CPUID_FEATURE_EDX_CX8',
+    'sep':          'X86_CPUID_FEATURE_EDX_SEP',
+    'cmov':         'X86_CPUID_FEATURE_EDX_CMOV',
+    'clfsh':        'X86_CPUID_FEATURE_EDX_CLFSH',
+    'clflushopt':   'X86_CPUID_STEXT_FEATURE_EBX_CLFLUSHOPT',
+    'mmx':          'X86_CPUID_FEATURE_EDX_MMX',
+    'fxsr':         'X86_CPUID_FEATURE_EDX_FXSR',
+    'sse':          'X86_CPUID_FEATURE_EDX_SSE',
+    'sse2':         'X86_CPUID_FEATURE_EDX_SSE2',
+    'sse3':         'X86_CPUID_FEATURE_ECX_SSE3',
+    'pclmul':       'X86_CPUID_FEATURE_ECX_DTES64',
+    'monitor':      'X86_CPUID_FEATURE_ECX_CPLDS',
+    'vmx':          'X86_CPUID_FEATURE_ECX_VMX',
+    'smx':          'X86_CPUID_FEATURE_ECX_TM2',
+    'ssse3':        'X86_CPUID_FEATURE_ECX_SSSE3',
+    'fma':          'X86_CPUID_FEATURE_ECX_FMA',
+    'cx16':         'X86_CPUID_FEATURE_ECX_CX16',
+    'pcid':         'X86_CPUID_FEATURE_ECX_PCID',
+    'sse4.1':       'X86_CPUID_FEATURE_ECX_SSE4_1',
+    'sse4.2':       'X86_CPUID_FEATURE_ECX_SSE4_2',
+    'movbe':        'X86_CPUID_FEATURE_ECX_MOVBE',
+    'popcnt':       'X86_CPUID_FEATURE_ECX_POPCNT',
+    'aes':          'X86_CPUID_FEATURE_ECX_AES',
+    'xsave':        'X86_CPUID_FEATURE_ECX_XSAVE',
+    'avx':          'X86_CPUID_FEATURE_ECX_AVX',
+    'avx2':         'X86_CPUID_STEXT_FEATURE_EBX_AVX2',
+    'f16c':         'X86_CPUID_FEATURE_ECX_F16C',
+    'rdrand':       'X86_CPUID_FEATURE_ECX_RDRAND',
+
+    'axmmx':        'X86_CPUID_AMD_FEATURE_EDX_AXMMX',
+    '3dnowext':     'X86_CPUID_AMD_FEATURE_EDX_3DNOW_EX',
+    '3dnow':        'X86_CPUID_AMD_FEATURE_EDX_3DNOW',
+    'svm':          'X86_CPUID_AMD_FEATURE_ECX_SVM',
+    'cr8l':         'X86_CPUID_AMD_FEATURE_ECX_CR8L',
+    'abm':          'X86_CPUID_AMD_FEATURE_ECX_ABM',
+    'sse4a':        'X86_CPUID_AMD_FEATURE_ECX_SSE4A',
+    '3dnowprf':     'X86_CPUID_AMD_FEATURE_ECX_3DNOWPRF',
+    'xop':          'X86_CPUID_AMD_FEATURE_ECX_XOP',
+    'fma4':         'X86_CPUID_AMD_FEATURE_ECX_FMA4',
+};
+
+## \@ophints values.
+g_kdHints = {
+    'invalid':               'DISOPTYPE_INVALID',               ##<
+    'harmless':              'DISOPTYPE_HARMLESS',              ##<
+    'controlflow':           'DISOPTYPE_CONTROLFLOW',           ##<
+    'potentially_dangerous': 'DISOPTYPE_POTENTIALLY_DANGEROUS', ##<
+    'dangerous':             'DISOPTYPE_DANGEROUS',             ##<
+    'portio':                'DISOPTYPE_PORTIO',                ##<
+    'privileged':            'DISOPTYPE_PRIVILEGED',            ##<
+    'privileged_notrap':     'DISOPTYPE_PRIVILEGED_NOTRAP',     ##<
+    'uncond_controlflow':    'DISOPTYPE_UNCOND_CONTROLFLOW',    ##<
+    'relative_controlflow':  'DISOPTYPE_RELATIVE_CONTROLFLOW',  ##<
+    'cond_controlflow':      'DISOPTYPE_COND_CONTROLFLOW',      ##<
+    'interrupt':             'DISOPTYPE_INTERRUPT',             ##<
+    'illegal':               'DISOPTYPE_ILLEGAL',               ##<
+    'rrm_dangerous':         'DISOPTYPE_RRM_DANGEROUS',         ##< Some additional dangerous ones when recompiling raw r0.
+    'rrm_dangerous_16':      'DISOPTYPE_RRM_DANGEROUS_16',      ##< Some additional dangerous ones when recompiling 16-bit raw r0.
+    'inhibit_irqs':          'DISOPTYPE_INHIBIT_IRQS',          ##< Will or can inhibit irqs (sti, pop ss, mov ss) */
+    'portio_read':           'DISOPTYPE_PORTIO_READ',           ##<
+    'portio_write':          'DISOPTYPE_PORTIO_WRITE',          ##<
+    'invalid_64':            'DISOPTYPE_INVALID_64',            ##< Invalid in 64 bits mode
+    'only_64':               'DISOPTYPE_ONLY_64',               ##< Only valid in 64 bits mode
+    'default_64_op_size':    'DISOPTYPE_DEFAULT_64_OP_SIZE',    ##< Default 64 bits operand size
+    'forced_64_op_size':     'DISOPTYPE_FORCED_64_OP_SIZE',     ##< Forced 64 bits operand size; regardless of prefix bytes
+    'rexb_extends_opreg':    'DISOPTYPE_REXB_EXTENDS_OPREG',    ##< REX.B extends the register field in the opcode byte
+    'mod_fixed_11':          'DISOPTYPE_MOD_FIXED_11',          ##< modrm.mod is always 11b
+    'forced_32_op_size_x86': 'DISOPTYPE_FORCED_32_OP_SIZE_X86', ##< Forced 32 bits operand size; regardless of prefix bytes
+                                                                ##  (only in 16 & 32 bits mode!)
+    'avx':                   'DISOPTYPE_AVX',                   ##< AVX,AVX2,++ instruction. Not implemented yet!
+    'sse':                   'DISOPTYPE_SSE',                   ##< SSE,SSE2,SSE3,++ instruction. Not implemented yet!
+    'mmx':                   'DISOPTYPE_MMX',                   ##< MMX,MMXExt,3DNow,++ instruction. Not implemented yet!
+    'fpu':                   'DISOPTYPE_FPU',                   ##< FPU instruction. Not implemented yet!
+    'ignores_oz_pfx':        '',                                ##< Ignores operand size prefix 66h.
+    'ignores_rexw':          '',                                ##< Ignores REX.W.
+    'ignores_op_sizes':      '',                                ##< Shorthand for "ignores_oz_pfx | ignores_op_sizes".
+    'vex_l_zero':            '',                                ##< VEX.L must be 0.
+    'vex_l_ignored':         '',                                ##< VEX.L is ignored.
+    'vex_v_zero':            '',                                ##< VEX.V must be 0. (generate sub-table?)
+    'lock_allowed':          '',                                ##< Lock prefix allowed.
+};
+
+## \@opxcpttype values (see SDMv2 2.4, 2.7).
+g_kdXcptTypes = {
+    'none':     [],
+    '1':        [],
+    '2':        [],
+    '3':        [],
+    '4':        [],
+    '4UA':      [],
+    '5':        [],
+    '5LZ':      [], # LZ = VEX.L must be zero.
+    '6':        [],
+    '7':        [],
+    '7LZ':      [],
+    '8':        [],
+    '11':       [],
+    '12':       [],
+    'E1':       [],
+    'E1NF':     [],
+    'E2':       [],
+    'E3':       [],
+    'E3NF':     [],
+    'E4':       [],
+    'E4NF':     [],
+    'E5':       [],
+    'E5NF':     [],
+    'E6':       [],
+    'E6NF':     [],
+    'E7NF':     [],
+    'E9':       [],
+    'E9NF':     [],
+    'E10':      [],
+    'E11':      [],
+    'E12':      [],
+    'E12NF':    [],
+};
+
+
+def _isValidOpcodeByte(sOpcode):
+    """
+    Checks if sOpcode is a valid lower case opcode byte.
+    Returns true/false.
+    """
+    if len(sOpcode) == 4:
+        if sOpcode[:2] == '0x':
+            if sOpcode[2] in '0123456789abcdef':
+                if sOpcode[3] in '0123456789abcdef':
+                    return True;
+    return False;
+
+
+class InstructionMap(object):
+    """
+    Instruction map.
+
+    The opcode map provides the lead opcode bytes (empty for the one byte
+    opcode map).  An instruction can be member of multiple opcode maps as long
+    as it uses the same opcode value within the map (because of VEX).
+    """
+
+    kdEncodings = {
+        'legacy':   [],
+        'vex1':     [], ##< VEX or EVEX prefix with vvvvv = 1
+        'vex2':     [], ##< VEX or EVEX prefix with vvvvv = 2
+        'vex3':     [], ##< VEX or EVEX prefix with vvvvv = 3
+        'xop8':     [], ##< XOP prefix with vvvvv = 8
+        'xop9':     [], ##< XOP prefix with vvvvv = 9
+        'xop10':    [], ##< XOP prefix with vvvvv = 10
+    };
+    ## Selectors.
+    ## 1. The first value is the number of table entries required by a
+    ## decoder or disassembler for this type of selector.
+    ## 2. The second value is how many entries per opcode byte if applicable.
+    kdSelectors = {
+        'byte':     [  256, 1, ], ##< next opcode byte selects the instruction (default).
+        'byte+pfx': [ 1024, 4, ], ##< next opcode byte selects the instruction together with the 0x66, 0xf2 and 0xf3 prefixes.
+        '/r':       [    8, 1, ], ##< modrm.reg selects the instruction.
+        'memreg /r':[   16, 1, ], ##< modrm.reg and (modrm.mod == 3) selects the instruction.
+        'mod /r':   [   32, 1, ], ##< modrm.reg and modrm.mod selects the instruction.
+        '!11 /r':   [    8, 1, ], ##< modrm.reg selects the instruction with modrm.mod != 0y11.
+        '11 /r':    [    8, 1, ], ##< modrm.reg select the instruction with modrm.mod == 0y11.
+        '11':       [   64, 1, ], ##< modrm.reg and modrm.rm select the instruction with modrm.mod == 0y11.
+    };
+
+    ## Define the subentry number according to the Instruction::sPrefix
+    ## value for 'byte+pfx' selected tables.
+    kiPrefixOrder = {
+        'none': 0,
+        '0x66': 1,
+        '0xf3': 2,
+        '0xf2': 3,
+    };
+
+    def __init__(self, sName, sIemName = None, asLeadOpcodes = None, sSelector = 'byte+pfx',
+                 sEncoding = 'legacy', sDisParse = None):
+        assert sSelector in self.kdSelectors;
+        assert sEncoding in self.kdEncodings;
+        if asLeadOpcodes is None:
+            asLeadOpcodes = [];
+        else:
+            for sOpcode in asLeadOpcodes:
+                assert _isValidOpcodeByte(sOpcode);
+        assert sDisParse is None or sDisParse.startswith('IDX_Parse');
+
+        self.sName          = sName;
+        self.sIemName       = sIemName;
+        self.asLeadOpcodes  = asLeadOpcodes;    ##< Lead opcode bytes formatted as hex strings like '0x0f'.
+        self.sSelector      = sSelector;        ##< The member selector, see kdSelectors.
+        self.sEncoding      = sEncoding;        ##< The encoding, see kdSelectors.
+        self.aoInstructions = []                # type: Instruction
+        self.sDisParse      = sDisParse;        ##< IDX_ParseXXX.
+
+    def copy(self, sNewName, sPrefixFilter = None):
+        """
+        Copies the table with filtering instruction by sPrefix if not None.
+        """
+        oCopy = InstructionMap(sNewName, sIemName = self.sIemName, asLeadOpcodes = self.asLeadOpcodes,
+                               sSelector = 'byte' if sPrefixFilter is not None and self.sSelector == 'byte+pfx'
+                               else self.sSelector,
+                               sEncoding = self.sEncoding, sDisParse = self.sDisParse);
+        if sPrefixFilter is None:
+            oCopy.aoInstructions = list(self.aoInstructions);
+        else:
+            oCopy.aoInstructions = [oInstr for oInstr in self.aoInstructions if oInstr.sPrefix == sPrefixFilter];
+        return oCopy;
+
+    def getTableSize(self):
+        """
+        Number of table entries.   This corresponds directly to the selector.
+        """
+        return self.kdSelectors[self.sSelector][0];
+
+    def getEntriesPerByte(self):
+        """
+        Number of table entries per opcode bytes.
+
+        This only really makes sense for the 'byte' and 'byte+pfx' selectors, for
+        the others it will just return 1.
+        """
+        return self.kdSelectors[self.sSelector][1];
+
+    def getInstructionIndex(self, oInstr):
+        """
+        Returns the table index for the instruction.
+        """
+        bOpcode = oInstr.getOpcodeByte();
+
+        # The byte selectors are simple.  We need a full opcode byte and need just return it.
+        if self.sSelector == 'byte':
+            assert oInstr.sOpcode[:2] == '0x' and len(oInstr.sOpcode) == 4, str(oInstr);
+            return bOpcode;
+
+        # The byte + prefix selector is similarly simple, though requires a prefix as well as the full opcode.
+        if self.sSelector  == 'byte+pfx':
+            assert oInstr.sOpcode[:2] == '0x' and len(oInstr.sOpcode) == 4, str(oInstr);
+            assert self.kiPrefixOrder.get(oInstr.sPrefix, -16384) >= 0;
+            return bOpcode * 4 + self.kiPrefixOrder.get(oInstr.sPrefix, -16384);
+
+        # The other selectors needs masking and shifting.
+        if self.sSelector == '/r':
+            return (bOpcode >> 3) & 0x7;
+
+        if self.sSelector == 'mod /r':
+            return (bOpcode >> 3) & 0x1f;
+
+        if self.sSelector == 'memreg /r':
+            return ((bOpcode >> 3) & 0x7) | (int((bOpcode >> 6) == 3) << 3);
+
+        if self.sSelector == '!11 /r':
+            assert (bOpcode & 0xc0) != 0xc, str(oInstr);
+            return (bOpcode >> 3) & 0x7;
+
+        if self.sSelector == '11 /r':
+            assert (bOpcode & 0xc0) == 0xc, str(oInstr);
+            return (bOpcode >> 3) & 0x7;
+
+        if self.sSelector == '11':
+            assert (bOpcode & 0xc0) == 0xc, str(oInstr);
+            return bOpcode & 0x3f;
+
+        assert False, self.sSelector;
+        return -1;
+
+    def getInstructionsInTableOrder(self):
+        """
+        Get instructions in table order.
+
+        Returns array of instructions.  Normally there is exactly one
+        instruction per entry.  However the entry could also be None if
+        not instruction was specified for that opcode value.  Or there
+        could be a list of instructions to deal with special encodings
+        where for instance prefix (e.g. REX.W) encodes a different
+        instruction or different CPUs have different instructions or
+        prefixes in the same place.
+        """
+        # Start with empty table.
+        cTable  = self.getTableSize();
+        aoTable = [None] * cTable;
+
+        # Insert the instructions.
+        for oInstr in self.aoInstructions:
+            if oInstr.sOpcode:
+                idxOpcode = self.getInstructionIndex(oInstr);
+                assert idxOpcode < cTable, str(idxOpcode);
+
+                oExisting = aoTable[idxOpcode];
+                if oExisting is None:
+                    aoTable[idxOpcode] = oInstr;
+                elif not isinstance(oExisting, list):
+                    aoTable[idxOpcode] = list([oExisting, oInstr]);
+                else:
+                    oExisting.append(oInstr);
+
+        return aoTable;
+
+
+    def getDisasTableName(self):
+        """
+        Returns the disassembler table name for this map.
+        """
+        sName = 'g_aDisas';
+        for sWord in self.sName.split('_'):
+            if sWord == 'm':            # suffix indicating modrm.mod==mem
+                sName += '_m';
+            elif sWord == 'r':          # suffix indicating modrm.mod==reg
+                sName += '_r';
+            elif len(sWord) == 2 and re.match('^[a-f0-9][a-f0-9]$', sWord):
+                sName += '_' + sWord;
+            else:
+                sWord  = sWord.replace('grp', 'Grp');
+                sWord  = sWord.replace('map', 'Map');
+                sName += sWord[0].upper() + sWord[1:];
+        return sName;
+
+    def getDisasRangeName(self):
+        """
+        Returns the disassembler table range name for this map.
+        """
+        return self.getDisasTableName().replace('g_aDisas', 'g_Disas') + 'Range';
+
+    def isVexMap(self):
+        """ Returns True if a VEX map. """
+        return self.sEncoding.startswith('vex');
+
+
+class TestType(object):
+    """
+    Test value type.
+
+    This base class deals with integer like values.  The fUnsigned constructor
+    parameter indicates the default stance on zero vs sign extending.  It is
+    possible to override fUnsigned=True by prefixing the value with '+' or '-'.
+    """
+    def __init__(self, sName, acbSizes = None, fUnsigned = True):
+        self.sName = sName;
+        self.acbSizes = [1, 2, 4, 8, 16, 32] if acbSizes is None else acbSizes;  # Normal sizes.
+        self.fUnsigned = fUnsigned;
+
+    class BadValue(Exception):
+        """ Bad value exception. """
+        def __init__(self, sMessage):
+            Exception.__init__(self, sMessage);
+            self.sMessage = sMessage;
+
+    ## For ascii ~ operator.
+    kdHexInv = {
+        '0': 'f',
+        '1': 'e',
+        '2': 'd',
+        '3': 'c',
+        '4': 'b',
+        '5': 'a',
+        '6': '9',
+        '7': '8',
+        '8': '7',
+        '9': '6',
+        'a': '5',
+        'b': '4',
+        'c': '3',
+        'd': '2',
+        'e': '1',
+        'f': '0',
+    };
+
+    def get(self, sValue):
+        """
+        Get the shortest normal sized byte representation of oValue.
+
+        Returns ((fSignExtend, bytearray), ) or ((fSignExtend, bytearray), (fSignExtend, bytearray), ).
+        The latter form is for AND+OR pairs where the first entry is what to
+        AND with the field and the second the one or OR with.
+
+        Raises BadValue if invalid value.
+        """
+        if not sValue:
+            raise TestType.BadValue('empty value');
+
+        # Deal with sign and detect hexadecimal or decimal.
+        fSignExtend = not self.fUnsigned;
+        if sValue[0] == '-' or sValue[0] == '+':
+            fSignExtend = True;
+            fHex = len(sValue) > 3 and sValue[1:3].lower() == '0x';
+        else:
+            fHex = len(sValue) > 2 and sValue[0:2].lower() == '0x';
+
+        # try convert it to long integer.
+        try:
+            iValue = long(sValue, 16 if fHex else 10);
+        except Exception as oXcpt:
+            raise TestType.BadValue('failed to convert "%s" to integer (%s)' % (sValue, oXcpt));
+
+        # Convert the hex string and pad it to a decent value.  Negative values
+        # needs to be manually converted to something non-negative (~-n + 1).
+        if iValue >= 0:
+            sHex = hex(iValue);
+            if sys.version_info[0] < 3:
+                assert sHex[-1] == 'L';
+                sHex = sHex[:-1];
+            assert sHex[:2] == '0x';
+            sHex = sHex[2:];
+        else:
+            sHex = hex(-iValue - 1);
+            if sys.version_info[0] < 3:
+                assert sHex[-1] == 'L';
+                sHex = sHex[:-1];
+            assert sHex[:2] == '0x';
+            sHex = ''.join([self.kdHexInv[sDigit] for sDigit in sHex[2:]]);
+            if fSignExtend and sHex[0] not in [ '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
+                sHex = 'f' + sHex;
+
+        cDigits = len(sHex);
+        if cDigits <= self.acbSizes[-1] * 2:
+            for cb in self.acbSizes:
+                cNaturalDigits = cb * 2;
+                if cDigits <= cNaturalDigits:
+                    break;
+        else:
+            cNaturalDigits = self.acbSizes[-1] * 2;
+            cNaturalDigits = int((cDigits + cNaturalDigits - 1) / cNaturalDigits) * cNaturalDigits;
+            assert isinstance(cNaturalDigits, int)
+
+        if cNaturalDigits != cDigits:
+            cNeeded = cNaturalDigits - cDigits;
+            if iValue >= 0:
+                sHex = ('0' * cNeeded) + sHex;
+            else:
+                sHex = ('f' * cNeeded) + sHex;
+
+        # Invert and convert to bytearray and return it.
+        abValue = bytearray([int(sHex[offHex - 2 : offHex], 16) for offHex in range(len(sHex), 0, -2)]);
+
+        return ((fSignExtend, abValue),);
+
+    def validate(self, sValue):
+        """
+        Returns True if value is okay, error message on failure.
+        """
+        try:
+            self.get(sValue);
+        except TestType.BadValue as oXcpt:
+            return oXcpt.sMessage;
+        return True;
+
+    def isAndOrPair(self, sValue):
+        """
+        Checks if sValue is a pair.
+        """
+        _ = sValue;
+        return False;
+
+
+class TestTypeEflags(TestType):
+    """
+    Special value parsing for EFLAGS/RFLAGS/FLAGS.
+    """
+
+    kdZeroValueFlags = { 'nv': 0, 'pl': 0, 'nz': 0, 'na': 0, 'pe': 0, 'nc': 0, 'di': 0, 'up': 0 };
+
+    def __init__(self, sName):
+        TestType.__init__(self, sName, acbSizes = [1, 2, 4, 8], fUnsigned = True);
+
+    def get(self, sValue):
+        fClear = 0;
+        fSet   = 0;
+        for sFlag in sValue.split(','):
+            sConstant = g_kdEFlagsMnemonics.get(sFlag, None);
+            if sConstant is None:
+                raise self.BadValue('Unknown flag "%s" in "%s"' % (sFlag, sValue))
+            if sConstant[0] == '!':
+                fClear |= g_kdX86EFlagsConstants[sConstant[1:]];
+            else:
+                fSet   |= g_kdX86EFlagsConstants[sConstant];
+
+        aoSet = TestType.get(self, '0x%x' % (fSet,));
+        if fClear != 0:
+            aoClear = TestType.get(self, '%#x' % (fClear,))
+            assert self.isAndOrPair(sValue) is True;
+            return (aoClear[0], aoSet[0]);
+        assert self.isAndOrPair(sValue) is False;
+        return aoSet;
+
+    def isAndOrPair(self, sValue):
+        for sZeroFlag in self.kdZeroValueFlags:
+            if sValue.find(sZeroFlag) >= 0:
+                return True;
+        return False;
+
+class TestTypeFromDict(TestType):
+    """
+    Special value parsing for CR0.
+    """
+
+    kdZeroValueFlags = { 'nv': 0, 'pl': 0, 'nz': 0, 'na': 0, 'pe': 0, 'nc': 0, 'di': 0, 'up': 0 };
+
+    def __init__(self, sName, kdConstantsAndValues, sConstantPrefix):
+        TestType.__init__(self, sName, acbSizes = [1, 2, 4, 8], fUnsigned = True);
+        self.kdConstantsAndValues = kdConstantsAndValues;
+        self.sConstantPrefix      = sConstantPrefix;
+
+    def get(self, sValue):
+        fValue = 0;
+        for sFlag in sValue.split(','):
+            fFlagValue = self.kdConstantsAndValues.get(self.sConstantPrefix + sFlag.upper(), None);
+            if fFlagValue is None:
+                raise self.BadValue('Unknown flag "%s" in "%s"' % (sFlag, sValue))
+            fValue |= fFlagValue;
+        return TestType.get(self, '0x%x' % (fValue,));
+
+
+class TestInOut(object):
+    """
+    One input or output state modifier.
+
+    This should be thought as values to modify BS3REGCTX and extended (needs
+    to be structured) state.
+    """
+    ## Assigned operators.
+    kasOperators = [
+        '&|=',  # Special AND(INV)+OR operator for use with EFLAGS.
+        '&~=',
+        '&=',
+        '|=',
+        '='
+    ];
+    ## Types
+    kdTypes = {
+        'uint':  TestType('uint', fUnsigned = True),
+        'int':   TestType('int'),
+        'efl':   TestTypeEflags('efl'),
+        'cr0':   TestTypeFromDict('cr0', g_kdX86Cr0Constants, 'X86_CR0_'),
+        'cr4':   TestTypeFromDict('cr4', g_kdX86Cr4Constants, 'X86_CR4_'),
+        'xcr0':  TestTypeFromDict('xcr0', g_kdX86XSaveCConstants, 'XSAVE_C_'),
+    };
+    ## CPU context fields.
+    kdFields = {
+        # name:         ( default type, [both|input|output], )
+        # Operands.
+        'op1':          ( 'uint', 'both',   ), ## \@op1
+        'op2':          ( 'uint', 'both',   ), ## \@op2
+        'op3':          ( 'uint', 'both',   ), ## \@op3
+        'op4':          ( 'uint', 'both',   ), ## \@op4
+        # Flags.
+        'efl':          ( 'efl',  'both',   ),
+        'efl_undef':    ( 'uint', 'output', ),
+        # 8-bit GPRs.
+        'al':           ( 'uint', 'both',   ),
+        'cl':           ( 'uint', 'both',   ),
+        'dl':           ( 'uint', 'both',   ),
+        'bl':           ( 'uint', 'both',   ),
+        'ah':           ( 'uint', 'both',   ),
+        'ch':           ( 'uint', 'both',   ),
+        'dh':           ( 'uint', 'both',   ),
+        'bh':           ( 'uint', 'both',   ),
+        'r8l':          ( 'uint', 'both',   ),
+        'r9l':          ( 'uint', 'both',   ),
+        'r10l':         ( 'uint', 'both',   ),
+        'r11l':         ( 'uint', 'both',   ),
+        'r12l':         ( 'uint', 'both',   ),
+        'r13l':         ( 'uint', 'both',   ),
+        'r14l':         ( 'uint', 'both',   ),
+        'r15l':         ( 'uint', 'both',   ),
+        # 16-bit GPRs.
+        'ax':           ( 'uint', 'both',   ),
+        'dx':           ( 'uint', 'both',   ),
+        'cx':           ( 'uint', 'both',   ),
+        'bx':           ( 'uint', 'both',   ),
+        'sp':           ( 'uint', 'both',   ),
+        'bp':           ( 'uint', 'both',   ),
+        'si':           ( 'uint', 'both',   ),
+        'di':           ( 'uint', 'both',   ),
+        'r8w':          ( 'uint', 'both',   ),
+        'r9w':          ( 'uint', 'both',   ),
+        'r10w':         ( 'uint', 'both',   ),
+        'r11w':         ( 'uint', 'both',   ),
+        'r12w':         ( 'uint', 'both',   ),
+        'r13w':         ( 'uint', 'both',   ),
+        'r14w':         ( 'uint', 'both',   ),
+        'r15w':         ( 'uint', 'both',   ),
+        # 32-bit GPRs.
+        'eax':          ( 'uint', 'both',   ),
+        'edx':          ( 'uint', 'both',   ),
+        'ecx':          ( 'uint', 'both',   ),
+        'ebx':          ( 'uint', 'both',   ),
+        'esp':          ( 'uint', 'both',   ),
+        'ebp':          ( 'uint', 'both',   ),
+        'esi':          ( 'uint', 'both',   ),
+        'edi':          ( 'uint', 'both',   ),
+        'r8d':          ( 'uint', 'both',   ),
+        'r9d':          ( 'uint', 'both',   ),
+        'r10d':         ( 'uint', 'both',   ),
+        'r11d':         ( 'uint', 'both',   ),
+        'r12d':         ( 'uint', 'both',   ),
+        'r13d':         ( 'uint', 'both',   ),
+        'r14d':         ( 'uint', 'both',   ),
+        'r15d':         ( 'uint', 'both',   ),
+        # 64-bit GPRs.
+        'rax':          ( 'uint', 'both',   ),
+        'rdx':          ( 'uint', 'both',   ),
+        'rcx':          ( 'uint', 'both',   ),
+        'rbx':          ( 'uint', 'both',   ),
+        'rsp':          ( 'uint', 'both',   ),
+        'rbp':          ( 'uint', 'both',   ),
+        'rsi':          ( 'uint', 'both',   ),
+        'rdi':          ( 'uint', 'both',   ),
+        'r8':           ( 'uint', 'both',   ),
+        'r9':           ( 'uint', 'both',   ),
+        'r10':          ( 'uint', 'both',   ),
+        'r11':          ( 'uint', 'both',   ),
+        'r12':          ( 'uint', 'both',   ),
+        'r13':          ( 'uint', 'both',   ),
+        'r14':          ( 'uint', 'both',   ),
+        'r15':          ( 'uint', 'both',   ),
+        # 16-bit, 32-bit or 64-bit registers according to operand size.
+        'oz.rax':       ( 'uint', 'both',   ),
+        'oz.rdx':       ( 'uint', 'both',   ),
+        'oz.rcx':       ( 'uint', 'both',   ),
+        'oz.rbx':       ( 'uint', 'both',   ),
+        'oz.rsp':       ( 'uint', 'both',   ),
+        'oz.rbp':       ( 'uint', 'both',   ),
+        'oz.rsi':       ( 'uint', 'both',   ),
+        'oz.rdi':       ( 'uint', 'both',   ),
+        'oz.r8':        ( 'uint', 'both',   ),
+        'oz.r9':        ( 'uint', 'both',   ),
+        'oz.r10':       ( 'uint', 'both',   ),
+        'oz.r11':       ( 'uint', 'both',   ),
+        'oz.r12':       ( 'uint', 'both',   ),
+        'oz.r13':       ( 'uint', 'both',   ),
+        'oz.r14':       ( 'uint', 'both',   ),
+        'oz.r15':       ( 'uint', 'both',   ),
+        # Control registers.
+        'cr0':          ( 'cr0',  'both',   ),
+        'cr4':          ( 'cr4',  'both',   ),
+        'xcr0':         ( 'xcr0', 'both',   ),
+        # FPU Registers
+        'fcw':          ( 'uint', 'both',   ),
+        'fsw':          ( 'uint', 'both',   ),
+        'ftw':          ( 'uint', 'both',   ),
+        'fop':          ( 'uint', 'both',   ),
+        'fpuip':        ( 'uint', 'both',   ),
+        'fpucs':        ( 'uint', 'both',   ),
+        'fpudp':        ( 'uint', 'both',   ),
+        'fpuds':        ( 'uint', 'both',   ),
+        'mxcsr':        ( 'uint', 'both',   ),
+        'st0':          ( 'uint', 'both',   ),
+        'st1':          ( 'uint', 'both',   ),
+        'st2':          ( 'uint', 'both',   ),
+        'st3':          ( 'uint', 'both',   ),
+        'st4':          ( 'uint', 'both',   ),
+        'st5':          ( 'uint', 'both',   ),
+        'st6':          ( 'uint', 'both',   ),
+        'st7':          ( 'uint', 'both',   ),
+        # MMX registers.
+        'mm0':          ( 'uint', 'both',   ),
+        'mm1':          ( 'uint', 'both',   ),
+        'mm2':          ( 'uint', 'both',   ),
+        'mm3':          ( 'uint', 'both',   ),
+        'mm4':          ( 'uint', 'both',   ),
+        'mm5':          ( 'uint', 'both',   ),
+        'mm6':          ( 'uint', 'both',   ),
+        'mm7':          ( 'uint', 'both',   ),
+        # SSE registers.
+        'xmm0':         ( 'uint', 'both',   ),
+        'xmm1':         ( 'uint', 'both',   ),
+        'xmm2':         ( 'uint', 'both',   ),
+        'xmm3':         ( 'uint', 'both',   ),
+        'xmm4':         ( 'uint', 'both',   ),
+        'xmm5':         ( 'uint', 'both',   ),
+        'xmm6':         ( 'uint', 'both',   ),
+        'xmm7':         ( 'uint', 'both',   ),
+        'xmm8':         ( 'uint', 'both',   ),
+        'xmm9':         ( 'uint', 'both',   ),
+        'xmm10':        ( 'uint', 'both',   ),
+        'xmm11':        ( 'uint', 'both',   ),
+        'xmm12':        ( 'uint', 'both',   ),
+        'xmm13':        ( 'uint', 'both',   ),
+        'xmm14':        ( 'uint', 'both',   ),
+        'xmm15':        ( 'uint', 'both',   ),
+        'xmm0.lo':      ( 'uint', 'both',   ),
+        'xmm1.lo':      ( 'uint', 'both',   ),
+        'xmm2.lo':      ( 'uint', 'both',   ),
+        'xmm3.lo':      ( 'uint', 'both',   ),
+        'xmm4.lo':      ( 'uint', 'both',   ),
+        'xmm5.lo':      ( 'uint', 'both',   ),
+        'xmm6.lo':      ( 'uint', 'both',   ),
+        'xmm7.lo':      ( 'uint', 'both',   ),
+        'xmm8.lo':      ( 'uint', 'both',   ),
+        'xmm9.lo':      ( 'uint', 'both',   ),
+        'xmm10.lo':     ( 'uint', 'both',   ),
+        'xmm11.lo':     ( 'uint', 'both',   ),
+        'xmm12.lo':     ( 'uint', 'both',   ),
+        'xmm13.lo':     ( 'uint', 'both',   ),
+        'xmm14.lo':     ( 'uint', 'both',   ),
+        'xmm15.lo':     ( 'uint', 'both',   ),
+        'xmm0.hi':      ( 'uint', 'both',   ),
+        'xmm1.hi':      ( 'uint', 'both',   ),
+        'xmm2.hi':      ( 'uint', 'both',   ),
+        'xmm3.hi':      ( 'uint', 'both',   ),
+        'xmm4.hi':      ( 'uint', 'both',   ),
+        'xmm5.hi':      ( 'uint', 'both',   ),
+        'xmm6.hi':      ( 'uint', 'both',   ),
+        'xmm7.hi':      ( 'uint', 'both',   ),
+        'xmm8.hi':      ( 'uint', 'both',   ),
+        'xmm9.hi':      ( 'uint', 'both',   ),
+        'xmm10.hi':     ( 'uint', 'both',   ),
+        'xmm11.hi':     ( 'uint', 'both',   ),
+        'xmm12.hi':     ( 'uint', 'both',   ),
+        'xmm13.hi':     ( 'uint', 'both',   ),
+        'xmm14.hi':     ( 'uint', 'both',   ),
+        'xmm15.hi':     ( 'uint', 'both',   ),
+        'xmm0.lo.zx':   ( 'uint', 'both',   ),
+        'xmm1.lo.zx':   ( 'uint', 'both',   ),
+        'xmm2.lo.zx':   ( 'uint', 'both',   ),
+        'xmm3.lo.zx':   ( 'uint', 'both',   ),
+        'xmm4.lo.zx':   ( 'uint', 'both',   ),
+        'xmm5.lo.zx':   ( 'uint', 'both',   ),
+        'xmm6.lo.zx':   ( 'uint', 'both',   ),
+        'xmm7.lo.zx':   ( 'uint', 'both',   ),
+        'xmm8.lo.zx':   ( 'uint', 'both',   ),
+        'xmm9.lo.zx':   ( 'uint', 'both',   ),
+        'xmm10.lo.zx':  ( 'uint', 'both',   ),
+        'xmm11.lo.zx':  ( 'uint', 'both',   ),
+        'xmm12.lo.zx':  ( 'uint', 'both',   ),
+        'xmm13.lo.zx':  ( 'uint', 'both',   ),
+        'xmm14.lo.zx':  ( 'uint', 'both',   ),
+        'xmm15.lo.zx':  ( 'uint', 'both',   ),
+        'xmm0.dw0':     ( 'uint', 'both',   ),
+        'xmm1.dw0':     ( 'uint', 'both',   ),
+        'xmm2.dw0':     ( 'uint', 'both',   ),
+        'xmm3.dw0':     ( 'uint', 'both',   ),
+        'xmm4.dw0':     ( 'uint', 'both',   ),
+        'xmm5.dw0':     ( 'uint', 'both',   ),
+        'xmm6.dw0':     ( 'uint', 'both',   ),
+        'xmm7.dw0':     ( 'uint', 'both',   ),
+        'xmm8.dw0':     ( 'uint', 'both',   ),
+        'xmm9.dw0':     ( 'uint', 'both',   ),
+        'xmm10.dw0':    ( 'uint', 'both',   ),
+        'xmm11.dw0':    ( 'uint', 'both',   ),
+        'xmm12.dw0':    ( 'uint', 'both',   ),
+        'xmm13.dw0':    ( 'uint', 'both',   ),
+        'xmm14.dw0':    ( 'uint', 'both',   ),
+        'xmm15_dw0':    ( 'uint', 'both',   ),
+        # AVX registers.
+        'ymm0':         ( 'uint', 'both',   ),
+        'ymm1':         ( 'uint', 'both',   ),
+        'ymm2':         ( 'uint', 'both',   ),
+        'ymm3':         ( 'uint', 'both',   ),
+        'ymm4':         ( 'uint', 'both',   ),
+        'ymm5':         ( 'uint', 'both',   ),
+        'ymm6':         ( 'uint', 'both',   ),
+        'ymm7':         ( 'uint', 'both',   ),
+        'ymm8':         ( 'uint', 'both',   ),
+        'ymm9':         ( 'uint', 'both',   ),
+        'ymm10':        ( 'uint', 'both',   ),
+        'ymm11':        ( 'uint', 'both',   ),
+        'ymm12':        ( 'uint', 'both',   ),
+        'ymm13':        ( 'uint', 'both',   ),
+        'ymm14':        ( 'uint', 'both',   ),
+        'ymm15':        ( 'uint', 'both',   ),
+
+        # Special ones.
+        'value.xcpt':   ( 'uint', 'output', ),
+    };
+
+    def __init__(self, sField, sOp, sValue, sType):
+        assert sField in self.kdFields;
+        assert sOp in self.kasOperators;
+        self.sField = sField;
+        self.sOp    = sOp;
+        self.sValue = sValue;
+        self.sType  = sType;
+        assert isinstance(sField, str);
+        assert isinstance(sOp, str);
+        assert isinstance(sType, str);
+        assert isinstance(sValue, str);
+
+
+class TestSelector(object):
+    """
+    One selector for an instruction test.
+    """
+    ## Selector compare operators.
+    kasCompareOps = [ '==', '!=' ];
+    ## Selector variables and their valid values.
+    kdVariables = {
+        # Operand size.
+        'size': {
+            'o16':  'size_o16',
+            'o32':  'size_o32',
+            'o64':  'size_o64',
+        },
+        # VEX.L value.
+        'vex.l': {
+            '0':    'vexl_0',
+            '1':    'vexl_1',
+        },
+        # Execution ring.
+        'ring': {
+            '0':    'ring_0',
+            '1':    'ring_1',
+            '2':    'ring_2',
+            '3':    'ring_3',
+            '0..2': 'ring_0_thru_2',
+            '1..3': 'ring_1_thru_3',
+        },
+        # Basic code mode.
+        'codebits': {
+            '64':   'code_64bit',
+            '32':   'code_32bit',
+            '16':   'code_16bit',
+        },
+        # cpu modes.
+        'mode': {
+            'real': 'mode_real',
+            'prot': 'mode_prot',
+            'long': 'mode_long',
+            'v86':  'mode_v86',
+            'smm':  'mode_smm',
+            'vmx':  'mode_vmx',
+            'svm':  'mode_svm',
+        },
+        # paging on/off
+        'paging': {
+            'on':       'paging_on',
+            'off':      'paging_off',
+        },
+        # CPU vendor
+        'vendor': {
+            'amd':      'vendor_amd',
+            'intel':    'vendor_intel',
+            'via':      'vendor_via',
+        },
+    };
+    ## Selector shorthand predicates.
+    ## These translates into variable expressions.
+    kdPredicates = {
+        'o16':          'size==o16',
+        'o32':          'size==o32',
+        'o64':          'size==o64',
+        'ring0':        'ring==0',
+        '!ring0':       'ring==1..3',
+        'ring1':        'ring==1',
+        'ring2':        'ring==2',
+        'ring3':        'ring==3',
+        'user':         'ring==3',
+        'supervisor':   'ring==0..2',
+        '16-bit':       'codebits==16',
+        '32-bit':       'codebits==32',
+        '64-bit':       'codebits==64',
+        'real':         'mode==real',
+        'prot':         'mode==prot',
+        'long':         'mode==long',
+        'v86':          'mode==v86',
+        'smm':          'mode==smm',
+        'vmx':          'mode==vmx',
+        'svm':          'mode==svm',
+        'paging':       'paging==on',
+        '!paging':      'paging==off',
+        'amd':          'vendor==amd',
+        '!amd':         'vendor!=amd',
+        'intel':        'vendor==intel',
+        '!intel':       'vendor!=intel',
+        'via':          'vendor==via',
+        '!via':         'vendor!=via',
+    };
+
+    def __init__(self, sVariable, sOp, sValue):
+        assert sVariable in self.kdVariables;
+        assert sOp in self.kasCompareOps;
+        assert sValue in self.kdVariables[sVariable];
+        self.sVariable  = sVariable;
+        self.sOp        = sOp;
+        self.sValue     = sValue;
+
+
+class InstructionTest(object):
+    """
+    Instruction test.
+    """
+
+    def __init__(self, oInstr): # type: (InstructionTest, Instruction)
+        self.oInstr         = oInstr    # type: InstructionTest
+        self.aoInputs       = []        # type: list(TestInOut)
+        self.aoOutputs      = []        # type: list(TestInOut)
+        self.aoSelectors    = []        # type: list(TestSelector)
+
+    def toString(self, fRepr = False):
+        """
+        Converts it to string representation.
+        """
+        asWords = [];
+        if self.aoSelectors:
+            for oSelector in self.aoSelectors:
+                asWords.append('%s%s%s' % (oSelector.sVariable, oSelector.sOp, oSelector.sValue,));
+            asWords.append('/');
+
+        for oModifier in self.aoInputs:
+            asWords.append('%s%s%s:%s' % (oModifier.sField, oModifier.sOp, oModifier.sValue, oModifier.sType,));
+
+        asWords.append('->');
+
+        for oModifier in self.aoOutputs:
+            asWords.append('%s%s%s:%s' % (oModifier.sField, oModifier.sOp, oModifier.sValue, oModifier.sType,));
+
+        if fRepr:
+            return '<' + ' '.join(asWords) + '>';
+        return '  '.join(asWords);
+
+    def __str__(self):
+        """ Provide string represenation. """
+        return self.toString(False);
+
+    def __repr__(self):
+        """ Provide unambigious string representation. """
+        return self.toString(True);
+
+class Operand(object):
+    """
+    Instruction operand.
+    """
+
+    def __init__(self, sWhere, sType):
+        assert sWhere in g_kdOpLocations, sWhere;
+        assert sType  in g_kdOpTypes, sType;
+        self.sWhere = sWhere;           ##< g_kdOpLocations
+        self.sType  = sType;            ##< g_kdOpTypes
+
+    def usesModRM(self):
+        """ Returns True if using some form of ModR/M encoding. """
+        return self.sType[0] in ['E', 'G', 'M'];
+
+
+
+class Instruction(object): # pylint: disable=too-many-instance-attributes
+    """
+    Instruction.
+    """
+
+    def __init__(self, sSrcFile, iLine):
+        ## @name Core attributes.
+        ## @{
+        self.oParent        = None      # type: Instruction
+        self.sMnemonic      = None;
+        self.sBrief         = None;
+        self.asDescSections = []        # type: list(str)
+        self.aoMaps         = []        # type: list(InstructionMap)
+        self.aoOperands     = []        # type: list(Operand)
+        self.sPrefix        = None;     ##< Single prefix: None, 'none', 0x66, 0xf3, 0xf2
+        self.sOpcode        = None      # type: str
+        self.sSubOpcode     = None      # type: str
+        self.sEncoding      = None;
+        self.asFlTest       = None;
+        self.asFlModify     = None;
+        self.asFlUndefined  = None;
+        self.asFlSet        = None;
+        self.asFlClear      = None;
+        self.dHints         = {};       ##< Dictionary of instruction hints, flags, whatnot. (Dictionary for speed; dummy value).
+        self.sDisEnum       = None;     ##< OP_XXXX value.  Default is based on the uppercased mnemonic.
+        self.asCpuIds       = [];       ##< The CPUID feature bit names for this instruction. If multiple, assume AND.
+        self.asReqFeatures  = [];       ##< Which features are required to be enabled to run this instruction.
+        self.aoTests        = []        # type: list(InstructionTest)
+        self.sMinCpu        = None;     ##< Indicates the minimum CPU required for the instruction. Not set when oCpuExpr is.
+        self.oCpuExpr       = None;     ##< Some CPU restriction expression...
+        self.sGroup         = None;
+        self.fUnused        = False;    ##< Unused instruction.
+        self.fInvalid       = False;    ##< Invalid instruction (like UD2).
+        self.sInvalidStyle  = None;     ##< Invalid behviour style (g_kdInvalidStyles),
+        self.sXcptType      = None;     ##< Exception type (g_kdXcptTypes).
+        ## @}
+
+        ## @name Implementation attributes.
+        ## @{
+        self.sStats         = None;
+        self.sFunction      = None;
+        self.fStub          = False;
+        self.fUdStub        = False;
+        ## @}
+
+        ## @name Decoding info
+        ## @{
+        self.sSrcFile       = sSrcFile;
+        self.iLineCreated   = iLine;
+        self.iLineCompleted = None;
+        self.cOpTags        = 0;
+        self.iLineFnIemOpMacro  = -1;
+        self.iLineMnemonicMacro = -1;
+        ## @}
+
+        ## @name Intermediate input fields.
+        ## @{
+        self.sRawDisOpNo    = None;
+        self.asRawDisParams = [];
+        self.sRawIemOpFlags = None;
+        self.sRawOldOpcodes = None;
+        self.asCopyTests    = [];
+        ## @}
+
+    def toString(self, fRepr = False):
+        """ Turn object into a string. """
+        aasFields = [];
+
+        aasFields.append(['opcode',    self.sOpcode]);
+        if self.sPrefix:
+            aasFields.append(['prefix', self.sPrefix]);
+        aasFields.append(['mnemonic',  self.sMnemonic]);
+        for iOperand, oOperand in enumerate(self.aoOperands):
+            aasFields.append(['op%u' % (iOperand + 1,), '%s:%s' % (oOperand.sWhere, oOperand.sType,)]);
+        if self.aoMaps:         aasFields.append(['maps', ','.join([oMap.sName for oMap in self.aoMaps])]);
+        aasFields.append(['encoding',  self.sEncoding]);
+        if self.dHints:         aasFields.append(['hints', ','.join(self.dHints.keys())]);
+        aasFields.append(['disenum',   self.sDisEnum]);
+        if self.asCpuIds:       aasFields.append(['cpuid', ','.join(self.asCpuIds)]);
+        aasFields.append(['group',     self.sGroup]);
+        if self.fUnused:        aasFields.append(['unused', 'True']);
+        if self.fInvalid:       aasFields.append(['invalid', 'True']);
+        aasFields.append(['invlstyle', self.sInvalidStyle]);
+        aasFields.append(['fltest',    self.asFlTest]);
+        aasFields.append(['flmodify',  self.asFlModify]);
+        aasFields.append(['flundef',   self.asFlUndefined]);
+        aasFields.append(['flset',     self.asFlSet]);
+        aasFields.append(['flclear',   self.asFlClear]);
+        aasFields.append(['mincpu',    self.sMinCpu]);
+        aasFields.append(['stats',     self.sStats]);
+        aasFields.append(['sFunction', self.sFunction]);
+        if self.fStub:          aasFields.append(['fStub', 'True']);
+        if self.fUdStub:        aasFields.append(['fUdStub', 'True']);
+        if self.cOpTags:        aasFields.append(['optags', str(self.cOpTags)]);
+        if self.iLineFnIemOpMacro  != -1: aasFields.append(['FNIEMOP_XXX', str(self.iLineFnIemOpMacro)]);
+        if self.iLineMnemonicMacro != -1: aasFields.append(['IEMOP_MNEMMONICn', str(self.iLineMnemonicMacro)]);
+
+        sRet = '<' if fRepr else '';
+        for sField, sValue in aasFields:
+            if sValue is not None:
+                if len(sRet) > 1:
+                    sRet += '; ';
+                sRet += '%s=%s' % (sField, sValue,);
+        if fRepr:
+            sRet += '>';
+
+        return sRet;
+
+    def __str__(self):
+        """ Provide string represenation. """
+        return self.toString(False);
+
+    def __repr__(self):
+        """ Provide unambigious string representation. """
+        return self.toString(True);
+
+    def copy(self, oMap = None, sOpcode = None, sSubOpcode = None, sPrefix = None):
+        """
+        Makes a copy of the object for the purpose of putting in a different map
+        or a different place in the current map.
+        """
+        oCopy = Instruction(self.sSrcFile, self.iLineCreated);
+
+        oCopy.oParent           = self;
+        oCopy.sMnemonic         = self.sMnemonic;
+        oCopy.sBrief            = self.sBrief;
+        oCopy.asDescSections    = list(self.asDescSections);
+        oCopy.aoMaps            = [oMap,]    if oMap       else list(self.aoMaps);
+        oCopy.aoOperands        = list(self.aoOperands); ## Deeper copy?
+        oCopy.sPrefix           = sPrefix    if sPrefix    else self.sPrefix;
+        oCopy.sOpcode           = sOpcode    if sOpcode    else self.sOpcode;
+        oCopy.sSubOpcode        = sSubOpcode if sSubOpcode else self.sSubOpcode;
+        oCopy.sEncoding         = self.sEncoding;
+        oCopy.asFlTest          = self.asFlTest;
+        oCopy.asFlModify        = self.asFlModify;
+        oCopy.asFlUndefined     = self.asFlUndefined;
+        oCopy.asFlSet           = self.asFlSet;
+        oCopy.asFlClear         = self.asFlClear;
+        oCopy.dHints            = dict(self.dHints);
+        oCopy.sDisEnum          = self.sDisEnum;
+        oCopy.asCpuIds          = list(self.asCpuIds);
+        oCopy.asReqFeatures     = list(self.asReqFeatures);
+        oCopy.aoTests           = list(self.aoTests); ## Deeper copy?
+        oCopy.sMinCpu           = self.sMinCpu;
+        oCopy.oCpuExpr          = self.oCpuExpr;
+        oCopy.sGroup            = self.sGroup;
+        oCopy.fUnused           = self.fUnused;
+        oCopy.fInvalid          = self.fInvalid;
+        oCopy.sInvalidStyle     = self.sInvalidStyle;
+        oCopy.sXcptType         = self.sXcptType;
+
+        oCopy.sStats            = self.sStats;
+        oCopy.sFunction         = self.sFunction;
+        oCopy.fStub             = self.fStub;
+        oCopy.fUdStub           = self.fUdStub;
+
+        oCopy.iLineCompleted    = self.iLineCompleted;
+        oCopy.cOpTags           = self.cOpTags;
+        oCopy.iLineFnIemOpMacro = self.iLineFnIemOpMacro;
+        oCopy.iLineMnemonicMacro = self.iLineMnemonicMacro;
+
+        oCopy.sRawDisOpNo       = self.sRawDisOpNo;
+        oCopy.asRawDisParams    = list(self.asRawDisParams);
+        oCopy.sRawIemOpFlags    = self.sRawIemOpFlags;
+        oCopy.sRawOldOpcodes    = self.sRawOldOpcodes;
+        oCopy.asCopyTests       = list(self.asCopyTests);
+
+        return oCopy;
+
+    def getOpcodeByte(self):
+        """
+        Decodes sOpcode into a byte range integer value.
+        Raises exception if sOpcode is None or invalid.
+        """
+        if self.sOpcode is None:
+            raise Exception('No opcode byte for %s!' % (self,));
+        sOpcode = str(self.sOpcode);    # pylint type confusion workaround.
+
+        # Full hex byte form.
+        if sOpcode[:2] == '0x':
+            return int(sOpcode, 16);
+
+        # The /r form:
+        if len(sOpcode) == 4 and sOpcode.startswith('/') and sOpcode[-1].isdigit():
+            return int(sOpcode[-1:]) << 3;
+
+        # The 11/r form:
+        if len(sOpcode) == 4 and sOpcode.startswith('11/') and sOpcode[-1].isdigit():
+            return (int(sOpcode[-1:]) << 3) | 0xc0;
+
+        # The !11/r form (returns mod=1):
+        ## @todo this doesn't really work...
+        if len(sOpcode) == 5 and sOpcode.startswith('!11/') and sOpcode[-1].isdigit():
+            return (int(sOpcode[-1:]) << 3) | 0x80;
+
+        raise Exception('unsupported opcode byte spec "%s" for %s' % (sOpcode, self,));
+
+    @staticmethod
+    def _flagsToIntegerMask(asFlags):
+        """
+        Returns the integer mask value for asFlags.
+        """
+        uRet = 0;
+        if asFlags:
+            for sFlag in asFlags:
+                sConstant = g_kdEFlagsMnemonics[sFlag];
+                assert sConstant[0] != '!', sConstant
+                uRet |= g_kdX86EFlagsConstants[sConstant];
+        return uRet;
+
+    def getTestedFlagsMask(self):
+        """ Returns asFlTest into a integer mask value """
+        return self._flagsToIntegerMask(self.asFlTest);
+
+    def getModifiedFlagsMask(self):
+        """ Returns asFlModify into a integer mask value """
+        return self._flagsToIntegerMask(self.asFlModify);
+
+    def getUndefinedFlagsMask(self):
+        """ Returns asFlUndefined into a integer mask value """
+        return self._flagsToIntegerMask(self.asFlUndefined);
+
+    def getSetFlagsMask(self):
+        """ Returns asFlSet into a integer mask value """
+        return self._flagsToIntegerMask(self.asFlSet);
+
+    def getClearedFlagsMask(self):
+        """ Returns asFlClear into a integer mask value """
+        return self._flagsToIntegerMask(self.asFlClear);
+
+    def onlyInVexMaps(self):
+        """ Returns True if only in VEX maps, otherwise False.  (No maps -> False) """
+        if not self.aoMaps:
+            return False;
+        for oMap in self.aoMaps:
+            if not oMap.isVexMap():
+                return False;
+        return True;
+
+
+
+## All the instructions.
+g_aoAllInstructions = []            # type: list(Instruction)
+
+## All the instructions indexed by statistics name (opstat).
+g_dAllInstructionsByStat = {}       # type: dict(Instruction)
+
+## All the instructions indexed by function name (opfunction).
+g_dAllInstructionsByFunction = {}   # type: dict(list(Instruction))
+
+## Instructions tagged by oponlytest
+g_aoOnlyTestInstructions = []       # type: list(Instruction)
+
+## Instruction maps.
+g_aoInstructionMaps = [
+    InstructionMap('one',        'g_apfnOneByteMap',        sSelector = 'byte'),
+    InstructionMap('grp1_80',    asLeadOpcodes = ['0x80',], sSelector = '/r'),
+    InstructionMap('grp1_81',    asLeadOpcodes = ['0x81',], sSelector = '/r'),
+    InstructionMap('grp1_82',    asLeadOpcodes = ['0x82',], sSelector = '/r'),
+    InstructionMap('grp1_83',    asLeadOpcodes = ['0x83',], sSelector = '/r'),
+    InstructionMap('grp1a',      asLeadOpcodes = ['0x8f',], sSelector = '/r'),
+    InstructionMap('grp2_c0',    asLeadOpcodes = ['0xc0',], sSelector = '/r'),
+    InstructionMap('grp2_c1',    asLeadOpcodes = ['0xc1',], sSelector = '/r'),
+    InstructionMap('grp2_d0',    asLeadOpcodes = ['0xd0',], sSelector = '/r'),
+    InstructionMap('grp2_d1',    asLeadOpcodes = ['0xd1',], sSelector = '/r'),
+    InstructionMap('grp2_d2',    asLeadOpcodes = ['0xd2',], sSelector = '/r'),
+    InstructionMap('grp2_d3',    asLeadOpcodes = ['0xd3',], sSelector = '/r'),
+    ## @todo g_apfnEscF1_E0toFF
+    InstructionMap('grp3_f6',    asLeadOpcodes = ['0xf6',], sSelector = '/r'),
+    InstructionMap('grp3_f7',    asLeadOpcodes = ['0xf7',], sSelector = '/r'),
+    InstructionMap('grp4',       asLeadOpcodes = ['0xfe',], sSelector = '/r'),
+    InstructionMap('grp5',       asLeadOpcodes = ['0xff',], sSelector = '/r'),
+    InstructionMap('grp11_c6_m', asLeadOpcodes = ['0xc6',], sSelector = '!11 /r'),
+    InstructionMap('grp11_c6_r', asLeadOpcodes = ['0xc6',], sSelector = '11'),    # xabort
+    InstructionMap('grp11_c7_m', asLeadOpcodes = ['0xc7',], sSelector = '!11 /r'),
+    InstructionMap('grp11_c7_r', asLeadOpcodes = ['0xc7',], sSelector = '11'),    # xbegin
+
+    InstructionMap('two0f',      'g_apfnTwoByteMap',    asLeadOpcodes = ['0x0f',], sDisParse = 'IDX_ParseTwoByteEsc'),
+    InstructionMap('grp6',       'g_apfnGroup6',        asLeadOpcodes = ['0x0f', '0x00',], sSelector = '/r'),
+    InstructionMap('grp7_m',     'g_apfnGroup7Mem',     asLeadOpcodes = ['0x0f', '0x01',], sSelector = '!11 /r'),
+    InstructionMap('grp7_r',                            asLeadOpcodes = ['0x0f', '0x01',], sSelector = '11'),
+    InstructionMap('grp8',                              asLeadOpcodes = ['0x0f', '0xba',], sSelector = '/r'),
+    InstructionMap('grp9',       'g_apfnGroup9RegReg',  asLeadOpcodes = ['0x0f', '0xc7',], sSelector = 'mod /r'),
+    ## @todo What about g_apfnGroup9MemReg?
+    InstructionMap('grp10',      None,                  asLeadOpcodes = ['0x0f', '0xb9',], sSelector = '/r'), # UD1 /w modr/m
+    InstructionMap('grp12',      'g_apfnGroup12RegReg', asLeadOpcodes = ['0x0f', '0x71',], sSelector = 'mod /r'),
+    InstructionMap('grp13',      'g_apfnGroup13RegReg', asLeadOpcodes = ['0x0f', '0x72',], sSelector = 'mod /r'),
+    InstructionMap('grp14',      'g_apfnGroup14RegReg', asLeadOpcodes = ['0x0f', '0x73',], sSelector = 'mod /r'),
+    InstructionMap('grp15',      'g_apfnGroup15MemReg', asLeadOpcodes = ['0x0f', '0xae',], sSelector = 'memreg /r'),
+    ## @todo What about g_apfnGroup15RegReg?
+    InstructionMap('grp16',       asLeadOpcodes = ['0x0f', '0x18',], sSelector = 'mod /r'),
+    InstructionMap('grpA17',      asLeadOpcodes = ['0x0f', '0x78',], sSelector = '/r'), # AMD: EXTRQ weirdness
+    InstructionMap('grpP',        asLeadOpcodes = ['0x0f', '0x0d',], sSelector = '/r'), # AMD: prefetch
+
+    InstructionMap('three0f38',  'g_apfnThreeByte0f38',    asLeadOpcodes = ['0x0f', '0x38',]),
+    InstructionMap('three0f3a',  'g_apfnThreeByte0f3a',    asLeadOpcodes = ['0x0f', '0x3a',]),
+
+    InstructionMap('vexmap1',  'g_apfnVexMap1',          sEncoding = 'vex1'),
+    InstructionMap('vexgrp12', 'g_apfnVexGroup12RegReg', sEncoding = 'vex1', asLeadOpcodes = ['0x71',], sSelector = 'mod /r'),
+    InstructionMap('vexgrp13', 'g_apfnVexGroup13RegReg', sEncoding = 'vex1', asLeadOpcodes = ['0x72',], sSelector = 'mod /r'),
+    InstructionMap('vexgrp14', 'g_apfnVexGroup14RegReg', sEncoding = 'vex1', asLeadOpcodes = ['0x73',], sSelector = 'mod /r'),
+    InstructionMap('vexgrp15', 'g_apfnVexGroup15MemReg', sEncoding = 'vex1', asLeadOpcodes = ['0xae',], sSelector = 'memreg /r'),
+    InstructionMap('vexgrp17', 'g_apfnVexGroup17_f3',    sEncoding = 'vex1', asLeadOpcodes = ['0xf3',], sSelector = '/r'),
+
+    InstructionMap('vexmap2',  'g_apfnVexMap2',          sEncoding = 'vex2'),
+    InstructionMap('vexmap3',  'g_apfnVexMap3',          sEncoding = 'vex3'),
+
+    InstructionMap('3dnow',    asLeadOpcodes = ['0x0f', '0x0f',]),
+    InstructionMap('xopmap8',  sEncoding = 'xop8'),
+    InstructionMap('xopmap9',  sEncoding = 'xop9'),
+    InstructionMap('xopgrp1',  sEncoding = 'xop9',  asLeadOpcodes = ['0x01'], sSelector = '/r'),
+    InstructionMap('xopgrp2',  sEncoding = 'xop9',  asLeadOpcodes = ['0x02'], sSelector = '/r'),
+    InstructionMap('xopgrp3',  sEncoding = 'xop9',  asLeadOpcodes = ['0x12'], sSelector = '/r'),
+    InstructionMap('xopmap10', sEncoding = 'xop10'),
+    InstructionMap('xopgrp4',  sEncoding = 'xop10', asLeadOpcodes = ['0x12'], sSelector = '/r'),
+];
+g_dInstructionMaps          = { oMap.sName:    oMap for oMap in g_aoInstructionMaps };
+g_dInstructionMapsByIemName = { oMap.sIemName: oMap for oMap in g_aoInstructionMaps };
+
+
+
+class ParserException(Exception):
+    """ Parser exception """
+    def __init__(self, sMessage):
+        Exception.__init__(self, sMessage);
+
+
+class SimpleParser(object):
+    """
+    Parser of IEMAllInstruction*.cpp.h instruction specifications.
+    """
+
+    ## @name Parser state.
+    ## @{
+    kiCode              = 0;
+    kiCommentMulti      = 1;
+    ## @}
+
+    def __init__(self, sSrcFile, asLines, sDefaultMap):
+        self.sSrcFile       = sSrcFile;
+        self.asLines        = asLines;
+        self.iLine          = 0;
+        self.iState         = self.kiCode;
+        self.sComment       = '';
+        self.iCommentLine   = 0;
+        self.aoCurInstrs    = [];
+
+        assert sDefaultMap in g_dInstructionMaps;
+        self.oDefaultMap    = g_dInstructionMaps[sDefaultMap];
+
+        self.cTotalInstr    = 0;
+        self.cTotalStubs    = 0;
+        self.cTotalTagged   = 0;
+
+        self.oReMacroName   = re.compile('^[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReMnemonic    = re.compile('^[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReStatsName   = re.compile('^[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReFunctionName= re.compile('^iemOp_[A-Za-z_][A-Za-z0-9_]*$');
+        self.oReGroupName   = re.compile('^og_[a-z0-9]+(|_[a-z0-9]+|_[a-z0-9]+_[a-z0-9]+)$');
+        self.oReDisEnum     = re.compile('^OP_[A-Z0-9_]+$');
+        self.oReFunTable    = re.compile('^(IEM_STATIC|static) +const +PFNIEMOP +g_apfn[A-Za-z0-9_]+ *\[ *\d* *\] *= *$');
+        self.oReComment     = re.compile('//.*?$|/\*.*?\*/'); ## Full comments.
+        self.fDebug         = True;
+
+        self.dTagHandlers   = {
+            '@opbrief':     self.parseTagOpBrief,
+            '@opdesc':      self.parseTagOpDesc,
+            '@opmnemonic':  self.parseTagOpMnemonic,
+            '@op1':         self.parseTagOpOperandN,
+            '@op2':         self.parseTagOpOperandN,
+            '@op3':         self.parseTagOpOperandN,
+            '@op4':         self.parseTagOpOperandN,
+            '@oppfx':       self.parseTagOpPfx,
+            '@opmaps':      self.parseTagOpMaps,
+            '@opcode':      self.parseTagOpcode,
+            '@opcodesub':   self.parseTagOpcodeSub,
+            '@openc':       self.parseTagOpEnc,
+            '@opfltest':    self.parseTagOpEFlags,
+            '@opflmodify':  self.parseTagOpEFlags,
+            '@opflundef':   self.parseTagOpEFlags,
+            '@opflset':     self.parseTagOpEFlags,
+            '@opflclear':   self.parseTagOpEFlags,
+            '@ophints':     self.parseTagOpHints,
+            '@opdisenum':   self.parseTagOpDisEnum,
+            '@opmincpu':    self.parseTagOpMinCpu,
+            '@opcpuid':     self.parseTagOpCpuId,
+            '@opgroup':     self.parseTagOpGroup,
+            '@opunused':    self.parseTagOpUnusedInvalid,
+            '@opinvalid':   self.parseTagOpUnusedInvalid,
+            '@opinvlstyle': self.parseTagOpUnusedInvalid,
+            '@optest':      self.parseTagOpTest,
+            '@optestign':   self.parseTagOpTestIgnore,
+            '@optestignore': self.parseTagOpTestIgnore,
+            '@opcopytests': self.parseTagOpCopyTests,
+            '@oponly':      self.parseTagOpOnlyTest,
+            '@oponlytest':  self.parseTagOpOnlyTest,
+            '@opxcpttype':  self.parseTagOpXcptType,
+            '@opstats':     self.parseTagOpStats,
+            '@opfunction':  self.parseTagOpFunction,
+            '@opdone':      self.parseTagOpDone,
+        };
+        for i in range(48):
+            self.dTagHandlers['@optest%u' % (i,)]   = self.parseTagOpTestNum;
+            self.dTagHandlers['@optest[%u]' % (i,)] = self.parseTagOpTestNum;
+
+        self.asErrors = [];
+
+    def raiseError(self, sMessage):
+        """
+        Raise error prefixed with the source and line number.
+        """
+        raise ParserException("%s:%d: error: %s" % (self.sSrcFile, self.iLine, sMessage,));
+
+    def raiseCommentError(self, iLineInComment, sMessage):
+        """
+        Similar to raiseError, but the line number is iLineInComment + self.iCommentLine.
+        """
+        raise ParserException("%s:%d: error: %s" % (self.sSrcFile, self.iCommentLine + iLineInComment, sMessage,));
+
+    def error(self, sMessage):
+        """
+        Adds an error.
+        returns False;
+        """
+        self.asErrors.append(u'%s:%d: error: %s\n' % (self.sSrcFile, self.iLine, sMessage,));
+        return False;
+
+    def errorOnLine(self, iLine, sMessage):
+        """
+        Adds an error.
+        returns False;
+        """
+        self.asErrors.append(u'%s:%d: error: %s\n' % (self.sSrcFile, iLine, sMessage,));
+        return False;
+
+    def errorComment(self, iLineInComment, sMessage):
+        """
+        Adds a comment error.
+        returns False;
+        """
+        self.asErrors.append(u'%s:%d: error: %s\n' % (self.sSrcFile, self.iCommentLine + iLineInComment, sMessage,));
+        return False;
+
+    def printErrors(self):
+        """
+        Print the errors to stderr.
+        Returns number of errors.
+        """
+        if self.asErrors:
+            sys.stderr.write(u''.join(self.asErrors));
+        return len(self.asErrors);
+
+    def debug(self, sMessage):
+        """
+        For debugging.
+        """
+        if self.fDebug:
+            print('debug: %s' % (sMessage,));
+
+    def stripComments(self, sLine):
+        """
+        Returns sLine with comments stripped.
+
+        Complains if traces of incomplete multi-line comments are encountered.
+        """
+        sLine = self.oReComment.sub(" ", sLine);
+        if sLine.find('/*') >= 0 or sLine.find('*/') >= 0:
+            self.error('Unexpected multi-line comment will not be handled correctly. Please simplify.');
+        return sLine;
+
+    def parseFunctionTable(self, sLine):
+        """
+        Parses a PFNIEMOP table, updating/checking the @oppfx value.
+
+        Note! Updates iLine as it consumes the whole table.
+        """
+
+        #
+        # Extract the table name.
+        #
+        sName = re.search(' *([a-zA-Z_0-9]+) *\[', sLine).group(1);
+        oMap  = g_dInstructionMapsByIemName.get(sName);
+        if not oMap:
+            self.debug('No map for PFNIEMOP table: %s' % (sName,));
+            oMap = self.oDefaultMap; # This is wrong wrong wrong.
+
+        #
+        # All but the g_apfnOneByteMap & g_apfnEscF1_E0toFF tables uses four
+        # entries per byte:
+        #       no prefix, 066h prefix, f3h prefix, f2h prefix
+        # Those tables has 256 & 32 entries respectively.
+        #
+        cEntriesPerByte   = 4;
+        cValidTableLength = 1024;
+        asPrefixes        = ('none', '0x66', '0xf3', '0xf2');
+
+        oEntriesMatch = re.search('\[ *(256|32) *\]', sLine);
+        if oEntriesMatch:
+            cEntriesPerByte   = 1;
+            cValidTableLength = int(oEntriesMatch.group(1));
+            asPrefixes        = (None,);
+
+        #
+        # The next line should be '{' and nothing else.
+        #
+        if self.iLine >= len(self.asLines) or not re.match('^ *{ *$', self.asLines[self.iLine]):
+            return self.errorOnLine(self.iLine + 1, 'Expected lone "{" on line following PFNIEMOP table %s start' % (sName, ));
+        self.iLine += 1;
+
+        #
+        # Parse till we find the end of the table.
+        #
+        iEntry = 0;
+        while self.iLine < len(self.asLines):
+            # Get the next line and strip comments and spaces (assumes no
+            # multi-line comments).
+            sLine = self.asLines[self.iLine];
+            self.iLine += 1;
+            sLine = self.stripComments(sLine).strip();
+
+            # Split the line up into entries, expanding IEMOP_X4 usage.
+            asEntries = sLine.split(',');
+            for i in range(len(asEntries) - 1, -1, -1):
+                sEntry = asEntries[i].strip();
+                if sEntry.startswith('IEMOP_X4(') and sEntry[-1] == ')':
+                    sEntry = (sEntry[len('IEMOP_X4('):-1]).strip();
+                    asEntries.insert(i + 1, sEntry);
+                    asEntries.insert(i + 1, sEntry);
+                    asEntries.insert(i + 1, sEntry);
+                if sEntry:
+                    asEntries[i] = sEntry;
+                else:
+                    del asEntries[i];
+
+            # Process the entries.
+            for sEntry in asEntries:
+                if sEntry in ('};', '}'):
+                    if iEntry != cValidTableLength:
+                        return self.error('Wrong table length for %s: %#x, expected %#x' % (sName, iEntry, cValidTableLength, ));
+                    return True;
+                if sEntry.startswith('iemOp_Invalid'):
+                    pass; # skip
+                else:
+                    # Look up matching instruction by function.
+                    sPrefix = asPrefixes[iEntry % cEntriesPerByte];
+                    sOpcode = '%#04x' % (iEntry // cEntriesPerByte);
+                    aoInstr  = g_dAllInstructionsByFunction.get(sEntry);
+                    if aoInstr:
+                        if not isinstance(aoInstr, list):
+                            aoInstr = [aoInstr,];
+                        oInstr = None;
+                        for oCurInstr in aoInstr:
+                            if oCurInstr.sOpcode == sOpcode and oCurInstr.sPrefix == sPrefix:
+                                pass;
+                            elif oCurInstr.sOpcode == sOpcode and oCurInstr.sPrefix is None:
+                                oCurInstr.sPrefix = sPrefix;
+                            elif oCurInstr.sOpcode is None and oCurInstr.sPrefix is None:
+                                oCurInstr.sOpcode = sOpcode;
+                                oCurInstr.sPrefix = sPrefix;
+                            else:
+                                continue;
+                            oInstr = oCurInstr;
+                            break;
+                        if not oInstr:
+                            oInstr = aoInstr[0].copy(oMap = oMap, sOpcode = sOpcode, sPrefix = sPrefix);
+                            aoInstr.append(oInstr);
+                            g_dAllInstructionsByFunction[sEntry] = aoInstr;
+                            g_aoAllInstructions.append(oInstr);
+                            oMap.aoInstructions.append(oInstr);
+                    else:
+                        self.debug('Function "%s", entry %#04x / byte %#04x in %s, is not associated with an instruction.'
+                                   % (sEntry, iEntry, iEntry // cEntriesPerByte, sName,));
+                iEntry += 1;
+
+        return self.error('Unexpected end of file in PFNIEMOP table');
+
+    def addInstruction(self, iLine = None):
+        """
+        Adds an instruction.
+        """
+        oInstr = Instruction(self.sSrcFile, self.iLine if iLine is None else iLine);
+        g_aoAllInstructions.append(oInstr);
+        self.aoCurInstrs.append(oInstr);
+        return oInstr;
+
+    def deriveMnemonicAndOperandsFromStats(self, oInstr, sStats):
+        """
+        Derives the mnemonic and operands from a IEM stats base name like string.
+        """
+        if oInstr.sMnemonic is None:
+            asWords = sStats.split('_');
+            oInstr.sMnemonic = asWords[0].lower();
+            if len(asWords) > 1 and not oInstr.aoOperands:
+                for sType in asWords[1:]:
+                    if sType in g_kdOpTypes:
+                        oInstr.aoOperands.append(Operand(g_kdOpTypes[sType][1], sType));
+                    else:
+                        #return self.error('unknown operand type: %s (instruction: %s)' % (sType, oInstr))
+                        return False;
+        return True;
+
+    def doneInstructionOne(self, oInstr, iLine):
+        """
+        Complete the parsing by processing, validating and expanding raw inputs.
+        """
+        assert oInstr.iLineCompleted is None;
+        oInstr.iLineCompleted = iLine;
+
+        #
+        # Specified instructions.
+        #
+        if oInstr.cOpTags > 0:
+            if oInstr.sStats is None:
+                pass;
+
+        #
+        # Unspecified legacy stuff.  We generally only got a few things to go on here.
+        #   /** Opcode 0x0f 0x00 /0. */
+        #   FNIEMOPRM_DEF(iemOp_Grp6_sldt)
+        #
+        else:
+            #if oInstr.sRawOldOpcodes:
+            #
+            #if oInstr.sMnemonic:
+            pass;
+
+        #
+        # Common defaults.
+        #
+
+        # Guess mnemonic and operands from stats if the former is missing.
+        if oInstr.sMnemonic is None:
+            if oInstr.sStats is not None:
+                self.deriveMnemonicAndOperandsFromStats(oInstr, oInstr.sStats);
+            elif oInstr.sFunction is not None:
+                self.deriveMnemonicAndOperandsFromStats(oInstr, oInstr.sFunction.replace('iemOp_', ''));
+
+        # Derive the disassembler op enum constant from the mnemonic.
+        if oInstr.sDisEnum is None and oInstr.sMnemonic is not None:
+            oInstr.sDisEnum = 'OP_' + oInstr.sMnemonic.upper();
+
+        # Derive the IEM statistics base name from mnemonic and operand types.
+        if oInstr.sStats is None:
+            if oInstr.sFunction is not None:
+                oInstr.sStats = oInstr.sFunction.replace('iemOp_', '');
+            elif oInstr.sMnemonic is not None:
+                oInstr.sStats = oInstr.sMnemonic;
+                for oOperand in oInstr.aoOperands:
+                    if oOperand.sType:
+                        oInstr.sStats += '_' + oOperand.sType;
+
+        # Derive the IEM function name from mnemonic and operand types.
+        if oInstr.sFunction is None:
+            if oInstr.sMnemonic is not None:
+                oInstr.sFunction = 'iemOp_' + oInstr.sMnemonic;
+                for oOperand in oInstr.aoOperands:
+                    if oOperand.sType:
+                        oInstr.sFunction += '_' + oOperand.sType;
+            elif oInstr.sStats:
+                oInstr.sFunction = 'iemOp_' + oInstr.sStats;
+
+        #
+        # Apply default map and then add the instruction to all it's groups.
+        #
+        if not oInstr.aoMaps:
+            oInstr.aoMaps = [ self.oDefaultMap, ];
+        for oMap in oInstr.aoMaps:
+            oMap.aoInstructions.append(oInstr);
+
+        #
+        # Derive encoding from operands and maps.
+        #
+        if oInstr.sEncoding is None:
+            if not oInstr.aoOperands:
+                if oInstr.fUnused and oInstr.sSubOpcode:
+                    oInstr.sEncoding = 'VEX.ModR/M' if oInstr.onlyInVexMaps() else 'ModR/M';
+                else:
+                    oInstr.sEncoding = 'VEX.fixed' if oInstr.onlyInVexMaps() else 'fixed';
+            elif oInstr.aoOperands[0].usesModRM():
+                if     (len(oInstr.aoOperands) >= 2 and oInstr.aoOperands[1].sWhere == 'vvvv') \
+                    or oInstr.onlyInVexMaps():
+                    oInstr.sEncoding = 'VEX.ModR/M';
+                else:
+                    oInstr.sEncoding = 'ModR/M';
+
+        #
+        # Check the opstat value and add it to the opstat indexed dictionary.
+        #
+        if oInstr.sStats:
+            if oInstr.sStats not in g_dAllInstructionsByStat:
+                g_dAllInstructionsByStat[oInstr.sStats] = oInstr;
+            else:
+                self.error('Duplicate opstat value "%s"\nnew: %s\nold: %s'
+                           % (oInstr.sStats, oInstr, g_dAllInstructionsByStat[oInstr.sStats],));
+
+        #
+        # Add to function indexed dictionary.  We allow multiple instructions per function.
+        #
+        if oInstr.sFunction:
+            if oInstr.sFunction not in g_dAllInstructionsByFunction:
+                g_dAllInstructionsByFunction[oInstr.sFunction] = [oInstr,];
+            else:
+                g_dAllInstructionsByFunction[oInstr.sFunction].append(oInstr);
+
+        #self.debug('%d..%d: %s; %d @op tags' % (oInstr.iLineCreated, oInstr.iLineCompleted, oInstr.sFunction, oInstr.cOpTags));
+        return True;
+
+    def doneInstructions(self, iLineInComment = None):
+        """
+        Done with current instruction.
+        """
+        for oInstr in self.aoCurInstrs:
+            self.doneInstructionOne(oInstr, self.iLine if iLineInComment is None else self.iCommentLine + iLineInComment);
+            if oInstr.fStub:
+                self.cTotalStubs += 1;
+
+        self.cTotalInstr += len(self.aoCurInstrs);
+
+        self.sComment     = '';
+        self.aoCurInstrs  = [];
+        return True;
+
+    def setInstrunctionAttrib(self, sAttrib, oValue, fOverwrite = False):
+        """
+        Sets the sAttrib of all current instruction to oValue.  If fOverwrite
+        is False, only None values and empty strings are replaced.
+        """
+        for oInstr in self.aoCurInstrs:
+            if fOverwrite is not True:
+                oOldValue = getattr(oInstr, sAttrib);
+                if oOldValue is not None:
+                    continue;
+            setattr(oInstr, sAttrib, oValue);
+
+    def setInstrunctionArrayAttrib(self, sAttrib, iEntry, oValue, fOverwrite = False):
+        """
+        Sets the iEntry of the array sAttrib of all current instruction to oValue.
+        If fOverwrite is False, only None values and empty strings are replaced.
+        """
+        for oInstr in self.aoCurInstrs:
+            aoArray = getattr(oInstr, sAttrib);
+            while len(aoArray) <= iEntry:
+                aoArray.append(None);
+            if fOverwrite is True or aoArray[iEntry] is None:
+                aoArray[iEntry] = oValue;
+
+    def parseCommentOldOpcode(self, asLines):
+        """ Deals with 'Opcode 0xff /4' like comments """
+        asWords = asLines[0].split();
+        if    len(asWords) >= 2  \
+          and asWords[0] == 'Opcode'  \
+          and (   asWords[1].startswith('0x')
+               or asWords[1].startswith('0X')):
+            asWords = asWords[:1];
+            for iWord, sWord in enumerate(asWords):
+                if sWord.startswith('0X'):
+                    sWord = '0x' + sWord[:2];
+                    asWords[iWord] = asWords;
+            self.setInstrunctionAttrib('sRawOldOpcodes', ' '.join(asWords));
+
+        return False;
+
+    def ensureInstructionForOpTag(self, iTagLine):
+        """ Ensure there is an instruction for the op-tag being parsed. """
+        if not self.aoCurInstrs:
+            self.addInstruction(self.iCommentLine + iTagLine);
+        for oInstr in self.aoCurInstrs:
+            oInstr.cOpTags += 1;
+            if oInstr.cOpTags == 1:
+                self.cTotalTagged += 1;
+        return self.aoCurInstrs[-1];
+
+    @staticmethod
+    def flattenSections(aasSections):
+        """
+        Flattens multiline sections into stripped single strings.
+        Returns list of strings, on section per string.
+        """
+        asRet = [];
+        for asLines in aasSections:
+            if asLines:
+                asRet.append(' '.join([sLine.strip() for sLine in asLines]));
+        return asRet;
+
+    @staticmethod
+    def flattenAllSections(aasSections, sLineSep = ' ', sSectionSep = '\n'):
+        """
+        Flattens sections into a simple stripped string with newlines as
+        section breaks.  The final section does not sport a trailing newline.
+        """
+        # Typical: One section with a single line.
+        if len(aasSections) == 1 and len(aasSections[0]) == 1:
+            return aasSections[0][0].strip();
+
+        sRet = '';
+        for iSection, asLines in enumerate(aasSections):
+            if asLines:
+                if iSection > 0:
+                    sRet += sSectionSep;
+                sRet += sLineSep.join([sLine.strip() for sLine in asLines]);
+        return sRet;
+
+
+
+    ## @name Tag parsers
+    ## @{
+
+    def parseTagOpBrief(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    \@opbrief
+        Value:  Text description, multiple sections, appended.
+
+        Brief description.  If not given, it's the first sentence from @opdesc.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sBrief = self.flattenAllSections(aasSections);
+        if not sBrief:
+            return self.errorComment(iTagLine, '%s: value required' % (sTag,));
+        if sBrief[-1] != '.':
+            sBrief = sBrief + '.';
+        if len(sBrief) > 180:
+            return self.errorComment(iTagLine, '%s: value too long (max 180 chars): %s' % (sTag, sBrief));
+        offDot = sBrief.find('.');
+        while 0 <= offDot < len(sBrief) - 1 and sBrief[offDot + 1] != ' ':
+            offDot = sBrief.find('.', offDot + 1);
+        if offDot >= 0 and offDot != len(sBrief) - 1:
+            return self.errorComment(iTagLine, '%s: only one sentence: %s' % (sTag, sBrief));
+
+        # Update the instruction.
+        if oInstr.sBrief is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite brief "%s" with "%s"'
+                                               % (sTag, oInstr.sBrief, sBrief,));
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpDesc(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    \@opdesc
+        Value:  Text description, multiple sections, appended.
+
+        It is used to describe instructions.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+        if aasSections:
+            oInstr.asDescSections.extend(self.flattenSections(aasSections));
+            return True;
+
+        _ = sTag; _ = iEndLine;
+        return True;
+
+    def parseTagOpMnemonic(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    @opmenmonic
+        Value:  mnemonic
+
+        The 'mnemonic' value must be a valid C identifier string.  Because of
+        prefixes, groups and whatnot, there times when the mnemonic isn't that
+        of an actual assembler mnemonic.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sMnemonic = self.flattenAllSections(aasSections);
+        if not self.oReMnemonic.match(sMnemonic):
+            return self.errorComment(iTagLine, '%s: invalid menmonic name: "%s"' % (sTag, sMnemonic,));
+        if oInstr.sMnemonic is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite menmonic "%s" with "%s"'
+                                     % (sTag, oInstr.sMnemonic, sMnemonic,));
+        oInstr.sMnemonic = sMnemonic
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpOperandN(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tags:  \@op1, \@op2, \@op3, \@op4
+        Value: [where:]type
+
+        The 'where' value indicates where the operand is found, like the 'reg'
+        part of the ModR/M encoding. See Instruction.kdOperandLocations for
+        a list.
+
+        The 'type' value indicates the operand type.  These follow the types
+        given in the opcode tables in the CPU reference manuals.
+        See Instruction.kdOperandTypes for a list.
+
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+        idxOp = int(sTag[-1]) - 1;
+        assert 0 <= idxOp < 4;
+
+        # flatten, split up, and validate the "where:type" value.
+        sFlattened = self.flattenAllSections(aasSections);
+        asSplit = sFlattened.split(':');
+        if len(asSplit) == 1:
+            sType  = asSplit[0];
+            sWhere = None;
+        elif len(asSplit) == 2:
+            (sWhere, sType) = asSplit;
+        else:
+            return self.errorComment(iTagLine, 'expected %s value on format "[<where>:]<type>" not "%s"' % (sTag, sFlattened,));
+
+        if sType not in g_kdOpTypes:
+            return self.errorComment(iTagLine, '%s: invalid where value "%s", valid: %s'
+                                               % (sTag, sType, ', '.join(g_kdOpTypes.keys()),));
+        if sWhere is None:
+            sWhere = g_kdOpTypes[sType][1];
+        elif sWhere not in g_kdOpLocations:
+            return self.errorComment(iTagLine, '%s: invalid where value "%s", valid: %s'
+                                               % (sTag, sWhere, ', '.join(g_kdOpLocations.keys()),));
+
+        # Insert the operand, refusing to overwrite an existing one.
+        while idxOp >= len(oInstr.aoOperands):
+            oInstr.aoOperands.append(None);
+        if oInstr.aoOperands[idxOp] is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s:%s" with "%s:%s"'
+                                               % ( sTag, oInstr.aoOperands[idxOp].sWhere, oInstr.aoOperands[idxOp].sType,
+                                                   sWhere, sType,));
+        oInstr.aoOperands[idxOp] = Operand(sWhere, sType);
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpMaps(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    \@opmaps
+        Value:  map[,map2]
+
+        Indicates which maps the instruction is in.  There is a default map
+        associated with each input file.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten, split up and validate the value.
+        sFlattened = self.flattenAllSections(aasSections, sLineSep = ',', sSectionSep = ',');
+        asMaps = sFlattened.split(',');
+        if not asMaps:
+            return self.errorComment(iTagLine, '%s: value required' % (sTag,));
+        for sMap in asMaps:
+            if sMap not in g_dInstructionMaps:
+                return self.errorComment(iTagLine, '%s: invalid map value: %s  (valid values: %s)'
+                                                   % (sTag, sMap, ', '.join(g_dInstructionMaps.keys()),));
+
+        # Add the maps to the current list.  Throw errors on duplicates.
+        for oMap in oInstr.aoMaps:
+            if oMap.sName in asMaps:
+                return self.errorComment(iTagLine, '%s: duplicate map assignment: %s' % (sTag, oMap.sName));
+
+        for sMap in asMaps:
+            oMap = g_dInstructionMaps[sMap];
+            if oMap not in oInstr.aoMaps:
+                oInstr.aoMaps.append(oMap);
+            else:
+                self.errorComment(iTagLine, '%s: duplicate map assignment (input): %s' % (sTag, sMap));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpPfx(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@oppfx
+        Value:      n/a|none|0x66|0xf3|0xf2
+
+        Required prefix for the instruction.  (In a (E)VEX context this is the
+        value of the 'pp' field rather than an actual prefix.)
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sFlattened = self.flattenAllSections(aasSections);
+        asPrefixes = sFlattened.split();
+        if len(asPrefixes) > 1:
+            return self.errorComment(iTagLine, '%s: max one prefix: %s' % (sTag, asPrefixes,));
+
+        sPrefix = asPrefixes[0].lower();
+        if sPrefix == 'none':
+            sPrefix = 'none';
+        elif sPrefix == 'n/a':
+            sPrefix = None;
+        else:
+            if len(sPrefix) == 2:
+                sPrefix = '0x' + sPrefix;
+            if not _isValidOpcodeByte(sPrefix):
+                return self.errorComment(iTagLine, '%s: invalid prefix: %s' % (sTag, sPrefix,));
+
+        if sPrefix is not None and sPrefix not in g_kdPrefixes:
+            return self.errorComment(iTagLine, '%s: invalid prefix: %s (valid %s)' % (sTag, sPrefix, g_kdPrefixes,));
+
+        # Set it.
+        if oInstr.sPrefix is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, oInstr.sPrefix, sPrefix,));
+        oInstr.sPrefix = sPrefix;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpcode(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opcode
+        Value:      0x?? | /reg (TODO: | mr/reg | 11 /reg | !11 /reg | 11 mr/reg | !11 mr/reg)
+
+        The opcode byte or sub-byte for the instruction in the context of a map.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sOpcode = self.flattenAllSections(aasSections);
+        if _isValidOpcodeByte(sOpcode):
+            pass;
+        elif len(sOpcode) == 2 and sOpcode.startswith('/') and sOpcode[-1] in '012345678':
+            pass;
+        elif len(sOpcode) == 4 and sOpcode.startswith('11/') and sOpcode[-1] in '012345678':
+            pass;
+        elif len(sOpcode) == 5 and sOpcode.startswith('!11/') and sOpcode[-1] in '012345678':
+            pass;
+        else:
+            return self.errorComment(iTagLine, '%s: invalid opcode: %s' % (sTag, sOpcode,));
+
+        # Set it.
+        if oInstr.sOpcode is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, oInstr.sOpcode, sOpcode,));
+        oInstr.sOpcode = sOpcode;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpcodeSub(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opcodesub
+        Value:      none | 11 mr/reg | !11 mr/reg | rex.w=0 | rex.w=1 | vex.l=0 | vex.l=1
+                    | 11 mr/reg vex.l=0 | 11 mr/reg vex.l=1 | !11 mr/reg vex.l=0 | !11 mr/reg vex.l=1
+
+        This is a simple way of dealing with encodings where the mod=3 and mod!=3
+        represents exactly two different instructions.  The more proper way would
+        be to go via maps with two members, but this is faster.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sSubOpcode = self.flattenAllSections(aasSections);
+        if sSubOpcode not in g_kdSubOpcodes:
+            return self.errorComment(iTagLine, '%s: invalid sub opcode: %s  (valid: 11, !11, none)' % (sTag, sSubOpcode,));
+        sSubOpcode = g_kdSubOpcodes[sSubOpcode][0];
+
+        # Set it.
+        if oInstr.sSubOpcode is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"'
+                                               % ( sTag, oInstr.sSubOpcode, sSubOpcode,));
+        oInstr.sSubOpcode = sSubOpcode;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpEnc(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@openc
+        Value:      ModR/M|fixed|prefix|<map name>
+
+        The instruction operand encoding style.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sEncoding = self.flattenAllSections(aasSections);
+        if sEncoding in g_kdEncodings:
+            pass;
+        elif sEncoding in g_dInstructionMaps:
+            pass;
+        elif not _isValidOpcodeByte(sEncoding):
+            return self.errorComment(iTagLine, '%s: invalid encoding: %s' % (sTag, sEncoding,));
+
+        # Set it.
+        if oInstr.sEncoding is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"'
+                                               % ( sTag, oInstr.sEncoding, sEncoding,));
+        oInstr.sEncoding = sEncoding;
+
+        _ = iEndLine;
+        return True;
+
+    ## EFlags tag to Instruction attribute name.
+    kdOpFlagToAttr = {
+        '@opfltest':    'asFlTest',
+        '@opflmodify':  'asFlModify',
+        '@opflundef':   'asFlUndefined',
+        '@opflset':     'asFlSet',
+        '@opflclear':   'asFlClear',
+    };
+
+    def parseTagOpEFlags(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tags:   \@opfltest, \@opflmodify, \@opflundef, \@opflset, \@opflclear
+        Value:  <eflags specifier>
+
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten, split up and validate the values.
+        asFlags = self.flattenAllSections(aasSections, sLineSep = ',', sSectionSep = ',').split(',');
+        if len(asFlags) == 1 and asFlags[0].lower() == 'none':
+            asFlags = [];
+        else:
+            fRc = True;
+            for iFlag, sFlag in enumerate(asFlags):
+                if sFlag not in g_kdEFlagsMnemonics:
+                    if sFlag.strip() in g_kdEFlagsMnemonics:
+                        asFlags[iFlag] = sFlag.strip();
+                    else:
+                        fRc = self.errorComment(iTagLine, '%s: invalid EFLAGS value: %s' % (sTag, sFlag,));
+            if not fRc:
+                return False;
+
+        # Set them.
+        asOld = getattr(oInstr, self.kdOpFlagToAttr[sTag]);
+        if asOld is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, asOld, asFlags,));
+        setattr(oInstr, self.kdOpFlagToAttr[sTag], asFlags);
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpHints(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@ophints
+        Value:      Comma or space separated list of flags and hints.
+
+        This covers the disassembler flags table and more.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten as a space separated list, split it up and validate the values.
+        asHints = self.flattenAllSections(aasSections, sLineSep = ' ', sSectionSep = ' ').replace(',', ' ').split();
+        if len(asHints) == 1 and asHints[0].lower() == 'none':
+            asHints = [];
+        else:
+            fRc = True;
+            for iHint, sHint in enumerate(asHints):
+                if sHint not in g_kdHints:
+                    if sHint.strip() in g_kdHints:
+                        sHint[iHint] = sHint.strip();
+                    else:
+                        fRc = self.errorComment(iTagLine, '%s: invalid hint value: %s' % (sTag, sHint,));
+            if not fRc:
+                return False;
+
+        # Append them.
+        for sHint in asHints:
+            if sHint not in oInstr.dHints:
+                oInstr.dHints[sHint] = True; # (dummy value, using dictionary for speed)
+            else:
+                self.errorComment(iTagLine, '%s: duplicate hint: %s' % ( sTag, sHint,));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpDisEnum(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opdisenum
+        Value:      OP_XXXX
+
+        This is for select a specific (legacy) disassembler enum value for the
+        instruction.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and split.
+        asWords = self.flattenAllSections(aasSections).split();
+        if len(asWords) != 1:
+            self.errorComment(iTagLine, '%s: expected exactly one value: %s' % (sTag, asWords,));
+            if not asWords:
+                return False;
+        sDisEnum = asWords[0];
+        if not self.oReDisEnum.match(sDisEnum):
+            return self.errorComment(iTagLine, '%s: invalid disassembler OP_XXXX enum: %s (pattern: %s)'
+                                               % (sTag, sDisEnum, self.oReDisEnum.pattern));
+
+        # Set it.
+        if oInstr.sDisEnum is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % (sTag, oInstr.sDisEnum, sDisEnum,));
+        oInstr.sDisEnum = sDisEnum;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpMinCpu(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opmincpu
+        Value:      <simple CPU name>
+
+        Indicates when this instruction was introduced.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten the value, split into words, make sure there's just one, valid it.
+        asCpus = self.flattenAllSections(aasSections).split();
+        if len(asCpus) > 1:
+            self.errorComment(iTagLine, '%s: exactly one CPU name, please: %s' % (sTag, ' '.join(asCpus),));
+
+        sMinCpu = asCpus[0];
+        if sMinCpu in g_kdCpuNames:
+            oInstr.sMinCpu = sMinCpu;
+        else:
+            return self.errorComment(iTagLine, '%s: invalid CPU name: %s  (names: %s)'
+                                               % (sTag, sMinCpu, ','.join(sorted(g_kdCpuNames)),));
+
+        # Set it.
+        if oInstr.sMinCpu is None:
+            oInstr.sMinCpu = sMinCpu;
+        elif oInstr.sMinCpu != sMinCpu:
+            self.errorComment(iTagLine, '%s: attemting to overwrite "%s" with "%s"' % (sTag, oInstr.sMinCpu, sMinCpu,));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpCpuId(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opcpuid
+        Value:      none | <CPUID flag specifier>
+
+        CPUID feature bit which is required for the instruction to be present.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten as a space separated list, split it up and validate the values.
+        asCpuIds = self.flattenAllSections(aasSections, sLineSep = ' ', sSectionSep = ' ').replace(',', ' ').split();
+        if len(asCpuIds) == 1 and asCpuIds[0].lower() == 'none':
+            asCpuIds = [];
+        else:
+            fRc = True;
+            for iCpuId, sCpuId in enumerate(asCpuIds):
+                if sCpuId not in g_kdCpuIdFlags:
+                    if sCpuId.strip() in g_kdCpuIdFlags:
+                        sCpuId[iCpuId] = sCpuId.strip();
+                    else:
+                        fRc = self.errorComment(iTagLine, '%s: invalid CPUID value: %s' % (sTag, sCpuId,));
+            if not fRc:
+                return False;
+
+        # Append them.
+        for sCpuId in asCpuIds:
+            if sCpuId not in oInstr.asCpuIds:
+                oInstr.asCpuIds.append(sCpuId);
+            else:
+                self.errorComment(iTagLine, '%s: duplicate CPUID: %s' % ( sTag, sCpuId,));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpGroup(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opgroup
+        Value:      op_grp1[_subgrp2[_subsubgrp3]]
+
+        Instruction grouping.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten as a space separated list, split it up and validate the values.
+        asGroups = self.flattenAllSections(aasSections).split();
+        if len(asGroups) != 1:
+            return self.errorComment(iTagLine, '%s: exactly one group, please: %s' % (sTag, asGroups,));
+        sGroup = asGroups[0];
+        if not self.oReGroupName.match(sGroup):
+            return self.errorComment(iTagLine, '%s: invalid group name: %s (valid: %s)'
+                                               % (sTag, sGroup, self.oReGroupName.pattern));
+
+        # Set it.
+        if oInstr.sGroup is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite "%s" with "%s"' % ( sTag, oInstr.sGroup, sGroup,));
+        oInstr.sGroup = sGroup;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpUnusedInvalid(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    \@opunused, \@opinvalid, \@opinvlstyle
+        Value:  <invalid opcode behaviour style>
+
+        The \@opunused indicates the specification is for a currently unused
+        instruction encoding.
+
+        The \@opinvalid indicates the specification is for an invalid currently
+        instruction encoding (like UD2).
+
+        The \@opinvlstyle just indicates how CPUs decode the instruction when
+        not supported (\@opcpuid, \@opmincpu) or disabled.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten as a space separated list, split it up and validate the values.
+        asStyles = self.flattenAllSections(aasSections).split();
+        if len(asStyles) != 1:
+            return self.errorComment(iTagLine, '%s: exactly one invalid behviour style, please: %s' % (sTag, asStyles,));
+        sStyle = asStyles[0];
+        if sStyle not in g_kdInvalidStyles:
+            return self.errorComment(iTagLine, '%s: invalid invalid behaviour style: %s (valid: %s)'
+                                               % (sTag, sStyle, g_kdInvalidStyles.keys(),));
+        # Set it.
+        if oInstr.sInvalidStyle is not None:
+            return self.errorComment(iTagLine,
+                                     '%s: attempting to overwrite "%s" with "%s" (only one @opunused, @opinvalid, @opinvlstyle)'
+                                     % ( sTag, oInstr.sInvalidStyle, sStyle,));
+        oInstr.sInvalidStyle = sStyle;
+        if sTag == '@opunused':
+            oInstr.fUnused = True;
+        elif sTag == '@opinvalid':
+            oInstr.fInvalid = True;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpTest(self, sTag, aasSections, iTagLine, iEndLine): # pylint: disable=too-many-locals
+        """
+        Tag:        \@optest
+        Value:      [<selectors>[ ]?] <inputs> -> <outputs>
+        Example:    mode==64bit / in1=0xfffffffe:dw in2=1:dw -> out1=0xffffffff:dw outfl=a?,p?
+
+        The main idea here is to generate basic instruction tests.
+
+        The probably simplest way of handling the diverse input, would be to use
+        it to produce size optimized byte code for a simple interpreter that
+        modifies the register input and output states.
+
+        An alternative to the interpreter would be creating multiple tables,
+        but that becomes rather complicated wrt what goes where and then to use
+        them in an efficient manner.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        #
+        # Do it section by section.
+        #
+        for asSectionLines in aasSections:
+            #
+            # Sort the input into outputs, inputs and selector conditions.
+            #
+            sFlatSection = self.flattenAllSections([asSectionLines,]);
+            if not sFlatSection:
+                self.errorComment(iTagLine, '%s: missing value (dbg: aasSections=%s)' % ( sTag, aasSections));
+                continue;
+            oTest = InstructionTest(oInstr);
+
+            asSelectors = [];
+            asInputs    = [];
+            asOutputs   = [];
+            asCur   = asOutputs;
+            fRc     = True;
+            asWords = sFlatSection.split();
+            for iWord in range(len(asWords) - 1, -1, -1):
+                sWord = asWords[iWord];
+                # Check for array switchers.
+                if sWord == '->':
+                    if asCur != asOutputs:
+                        fRc = self.errorComment(iTagLine, '%s: "->" shall only occure once: %s' % (sTag, sFlatSection,));
+                        break;
+                    asCur = asInputs;
+                elif sWord == '/':
+                    if asCur != asInputs:
+                        fRc = self.errorComment(iTagLine, '%s: "/" shall only occure once: %s' % (sTag, sFlatSection,));
+                        break;
+                    asCur = asSelectors;
+                else:
+                    asCur.insert(0, sWord);
+
+            #
+            # Validate and add selectors.
+            #
+            for sCond in asSelectors:
+                sCondExp = TestSelector.kdPredicates.get(sCond, sCond);
+                oSelector = None;
+                for sOp in TestSelector.kasCompareOps:
+                    off = sCondExp.find(sOp);
+                    if off >= 0:
+                        sVariable = sCondExp[:off];
+                        sValue    = sCondExp[off + len(sOp):];
+                        if sVariable in TestSelector.kdVariables:
+                            if sValue in TestSelector.kdVariables[sVariable]:
+                                oSelector = TestSelector(sVariable, sOp, sValue);
+                            else:
+                                self.errorComment(iTagLine, '%s: invalid condition value "%s" in "%s" (valid: %s)'
+                                                             % ( sTag, sValue, sCond,
+                                                                 TestSelector.kdVariables[sVariable].keys(),));
+                        else:
+                            self.errorComment(iTagLine, '%s: invalid condition variable "%s" in "%s" (valid: %s)'
+                                                         % ( sTag, sVariable, sCond, TestSelector.kdVariables.keys(),));
+                        break;
+                if oSelector is not None:
+                    for oExisting in oTest.aoSelectors:
+                        if oExisting.sVariable == oSelector.sVariable:
+                            self.errorComment(iTagLine, '%s: already have a selector for variable "%s" (existing: %s, new: %s)'
+                                                         % ( sTag, oSelector.sVariable, oExisting, oSelector,));
+                    oTest.aoSelectors.append(oSelector);
+                else:
+                    fRc = self.errorComment(iTagLine, '%s: failed to parse selector: %s' % ( sTag, sCond,));
+
+            #
+            # Validate outputs and inputs, adding them to the test as we go along.
+            #
+            for asItems, sDesc, aoDst in [ (asInputs, 'input', oTest.aoInputs), (asOutputs, 'output', oTest.aoOutputs)]:
+                asValidFieldKinds = [ 'both', sDesc, ];
+                for sItem in asItems:
+                    oItem = None;
+                    for sOp in TestInOut.kasOperators:
+                        off = sItem.find(sOp);
+                        if off < 0:
+                            continue;
+                        sField     = sItem[:off];
+                        sValueType = sItem[off + len(sOp):];
+                        if     sField in TestInOut.kdFields \
+                           and TestInOut.kdFields[sField][1] in asValidFieldKinds:
+                            asSplit = sValueType.split(':', 1);
+                            sValue  = asSplit[0];
+                            sType   = asSplit[1] if len(asSplit) > 1 else TestInOut.kdFields[sField][0];
+                            if sType in TestInOut.kdTypes:
+                                oValid = TestInOut.kdTypes[sType].validate(sValue);
+                                if oValid is True:
+                                    if not TestInOut.kdTypes[sType].isAndOrPair(sValue) or sOp == '&|=':
+                                        oItem = TestInOut(sField, sOp, sValue, sType);
+                                    else:
+                                        self.errorComment(iTagLine, '%s: and-or %s value "%s" can only be used with "&|="'
+                                                                    % ( sTag, sDesc, sItem, ));
+                                else:
+                                    self.errorComment(iTagLine, '%s: invalid %s value "%s" in "%s" (type: %s): %s'
+                                                                % ( sTag, sDesc, sValue, sItem, sType, oValid, ));
+                            else:
+                                self.errorComment(iTagLine, '%s: invalid %s type "%s" in "%s" (valid types: %s)'
+                                                             % ( sTag, sDesc, sType, sItem, TestInOut.kdTypes.keys(),));
+                        else:
+                            self.errorComment(iTagLine, '%s: invalid %s field "%s" in "%s"\nvalid fields: %s'
+                                                         % ( sTag, sDesc, sField, sItem,
+                                                             ', '.join([sKey for sKey, asVal in TestInOut.kdFields.items()
+                                                                        if asVal[1] in asValidFieldKinds]),));
+                        break;
+                    if oItem is not None:
+                        for oExisting in aoDst:
+                            if oExisting.sField == oItem.sField and oExisting.sOp == oItem.sOp:
+                                self.errorComment(iTagLine,
+                                                  '%s: already have a "%s" assignment for field "%s" (existing: %s, new: %s)'
+                                                  % ( sTag, oItem.sOp, oItem.sField, oExisting, oItem,));
+                        aoDst.append(oItem);
+                    else:
+                        fRc = self.errorComment(iTagLine, '%s: failed to parse assignment: %s' % ( sTag, sItem,));
+
+            #
+            # .
+            #
+            if fRc:
+                oInstr.aoTests.append(oTest);
+            else:
+                self.errorComment(iTagLine, '%s: failed to parse test: %s' % (sTag, ' '.join(asWords),));
+                self.errorComment(iTagLine, '%s: asSelectors=%s / asInputs=%s -> asOutputs=%s'
+                                            % (sTag, asSelectors, asInputs, asOutputs,));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpTestNum(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Numbered \@optest tag.  Either \@optest42 or \@optest[42].
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        iTest = 0;
+        if sTag[-1] == ']':
+            iTest = int(sTag[8:-1]);
+        else:
+            iTest = int(sTag[7:]);
+
+        if iTest != len(oInstr.aoTests):
+            self.errorComment(iTagLine, '%s: incorrect test number: %u, actual %u' % (sTag, iTest, len(oInstr.aoTests),));
+        return self.parseTagOpTest(sTag, aasSections, iTagLine, iEndLine);
+
+    def parseTagOpTestIgnore(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@optestign | \@optestignore
+        Value:      <value is ignored>
+
+        This is a simple trick to ignore a test while debugging another.
+
+        See also \@oponlytest.
+        """
+        _ = sTag; _ = aasSections; _ = iTagLine; _ = iEndLine;
+        return True;
+
+    def parseTagOpCopyTests(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opcopytests
+        Value:      <opstat | function> [..]
+        Example:    \@opcopytests add_Eb_Gb
+
+        Trick to avoid duplicating tests for different encodings of the same
+        operation.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten, validate and append the copy job to the instruction.  We execute
+        # them after parsing all the input so we can handle forward references.
+        asToCopy = self.flattenAllSections(aasSections).split();
+        if not asToCopy:
+            return self.errorComment(iTagLine, '%s: requires at least on reference value' % (sTag,));
+        for sToCopy in asToCopy:
+            if sToCopy not in oInstr.asCopyTests:
+                if self.oReStatsName.match(sToCopy) or self.oReFunctionName.match(sToCopy):
+                    oInstr.asCopyTests.append(sToCopy);
+                else:
+                    self.errorComment(iTagLine, '%s: invalid instruction reference (opstat or function) "%s" (valid: %s or %s)'
+                                                % (sTag, sToCopy, self.oReStatsName.pattern, self.oReFunctionName.pattern));
+            else:
+                self.errorComment(iTagLine, '%s: ignoring duplicate "%s"' % (sTag, sToCopy,));
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpOnlyTest(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@oponlytest | \@oponly
+        Value:      none
+
+        Only test instructions with this tag.  This is a trick that is handy
+        for singling out one or two new instructions or tests.
+
+        See also \@optestignore.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Validate and add instruction to only test dictionary.
+        sValue = self.flattenAllSections(aasSections).strip();
+        if sValue:
+            return self.errorComment(iTagLine, '%s: does not take any value: %s' % (sTag, sValue));
+
+        if oInstr not in g_aoOnlyTestInstructions:
+            g_aoOnlyTestInstructions.append(oInstr);
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpXcptType(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opxcpttype
+        Value:      [none|1|2|3|4|4UA|5|6|7|8|11|12|E1|E1NF|E2|E3|E3NF|E4|E4NF|E5|E5NF|E6|E6NF|E7NF|E9|E9NF|E10|E11|E12|E12NF]
+
+        Sets the SSE or AVX exception type (see SDMv2 2.4, 2.7).
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten as a space separated list, split it up and validate the values.
+        asTypes = self.flattenAllSections(aasSections).split();
+        if len(asTypes) != 1:
+            return self.errorComment(iTagLine, '%s: exactly one invalid exception type, please: %s' % (sTag, asTypes,));
+        sType = asTypes[0];
+        if sType not in g_kdXcptTypes:
+            return self.errorComment(iTagLine, '%s: invalid invalid exception type: %s (valid: %s)'
+                                               % (sTag, sType, sorted(g_kdXcptTypes.keys()),));
+        # Set it.
+        if oInstr.sXcptType is not None:
+            return self.errorComment(iTagLine,
+                                     '%s: attempting to overwrite "%s" with "%s" (only one @opxcpttype)'
+                                     % ( sTag, oInstr.sXcptType, sType,));
+        oInstr.sXcptType = sType;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpFunction(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opfunction
+        Value:      <VMM function name>
+
+        This is for explicitly setting the IEM function name.  Normally we pick
+        this up from the FNIEMOP_XXX macro invocation after the description, or
+        generate it from the mnemonic and operands.
+
+        It it thought it maybe necessary to set it when specifying instructions
+        which implementation isn't following immediately or aren't implemented yet.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sFunction = self.flattenAllSections(aasSections);
+        if not self.oReFunctionName.match(sFunction):
+            return self.errorComment(iTagLine, '%s: invalid VMM function name: "%s" (valid: %s)'
+                                               % (sTag, sFunction, self.oReFunctionName.pattern));
+
+        if oInstr.sFunction is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite VMM function name "%s" with "%s"'
+                                     % (sTag, oInstr.sFunction, sFunction,));
+        oInstr.sFunction = sFunction;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpStats(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:        \@opstats
+        Value:      <VMM statistics base name>
+
+        This is for explicitly setting the statistics name.  Normally we pick
+        this up from the IEMOP_MNEMONIC macro invocation, or generate it from
+        the mnemonic and operands.
+
+        It it thought it maybe necessary to set it when specifying instructions
+        which implementation isn't following immediately or aren't implemented yet.
+        """
+        oInstr = self.ensureInstructionForOpTag(iTagLine);
+
+        # Flatten and validate the value.
+        sStats = self.flattenAllSections(aasSections);
+        if not self.oReStatsName.match(sStats):
+            return self.errorComment(iTagLine, '%s: invalid VMM statistics name: "%s" (valid: %s)'
+                                               % (sTag, sStats, self.oReStatsName.pattern));
+
+        if oInstr.sStats is not None:
+            return self.errorComment(iTagLine, '%s: attempting to overwrite VMM statistics base name "%s" with "%s"'
+                                     % (sTag, oInstr.sStats, sStats,));
+        oInstr.sStats = sStats;
+
+        _ = iEndLine;
+        return True;
+
+    def parseTagOpDone(self, sTag, aasSections, iTagLine, iEndLine):
+        """
+        Tag:    \@opdone
+        Value:  none
+
+        Used to explictily flush the instructions that have been specified.
+        """
+        sFlattened = self.flattenAllSections(aasSections);
+        if sFlattened != '':
+            return self.errorComment(iTagLine, '%s: takes no value, found: "%s"' % (sTag, sFlattened,));
+        _ = sTag; _ = iEndLine;
+        return self.doneInstructions();
+
+    ## @}
+
+
+    def parseComment(self):
+        """
+        Parse the current comment (self.sComment).
+
+        If it's a opcode specifiying comment, we reset the macro stuff.
+        """
+        #
+        # Reject if comment doesn't seem to contain anything interesting.
+        #
+        if    self.sComment.find('Opcode') < 0 \
+          and self.sComment.find('@') < 0:
+            return False;
+
+        #
+        # Split the comment into lines, removing leading asterisks and spaces.
+        # Also remove leading and trailing empty lines.
+        #
+        asLines = self.sComment.split('\n');
+        for iLine, sLine in enumerate(asLines):
+            asLines[iLine] = sLine.lstrip().lstrip('*').lstrip();
+
+        while asLines and not asLines[0]:
+            self.iCommentLine += 1;
+            asLines.pop(0);
+
+        while asLines and not asLines[-1]:
+            asLines.pop(len(asLines) - 1);
+
+        #
+        # Check for old style: Opcode 0x0f 0x12
+        #
+        if asLines[0].startswith('Opcode '):
+            self.parseCommentOldOpcode(asLines);
+
+        #
+        # Look for @op* tagged data.
+        #
+        cOpTags      = 0;
+        sFlatDefault = None;
+        sCurTag      = '@default';
+        iCurTagLine  = 0;
+        asCurSection = [];
+        aasSections  = [ asCurSection, ];
+        for iLine, sLine in enumerate(asLines):
+            if not sLine.startswith('@'):
+                if sLine:
+                    asCurSection.append(sLine);
+                elif asCurSection:
+                    asCurSection = [];
+                    aasSections.append(asCurSection);
+            else:
+                #
+                # Process the previous tag.
+                #
+                if not asCurSection and len(aasSections) > 1:
+                    aasSections.pop(-1);
+                if sCurTag in self.dTagHandlers:
+                    self.dTagHandlers[sCurTag](sCurTag, aasSections, iCurTagLine, iLine);
+                    cOpTags += 1;
+                elif sCurTag.startswith('@op'):
+                    self.errorComment(iCurTagLine, 'Unknown tag: %s' % (sCurTag));
+                elif sCurTag == '@default':
+                    sFlatDefault = self.flattenAllSections(aasSections);
+                elif '@op' + sCurTag[1:] in self.dTagHandlers:
+                    self.errorComment(iCurTagLine, 'Did you mean "@op%s" rather than "%s"?' % (sCurTag[1:], sCurTag));
+                elif sCurTag in ['@encoding', '@opencoding']:
+                    self.errorComment(iCurTagLine, 'Did you mean "@openc" rather than "%s"?' % (sCurTag,));
+
+                #
+                # New tag.
+                #
+                asSplit = sLine.split(None, 1);
+                sCurTag = asSplit[0].lower();
+                if len(asSplit) > 1:
+                    asCurSection = [asSplit[1],];
+                else:
+                    asCurSection = [];
+                aasSections = [asCurSection, ];
+                iCurTagLine = iLine;
+
+        #
+        # Process the final tag.
+        #
+        if not asCurSection and len(aasSections) > 1:
+            aasSections.pop(-1);
+        if sCurTag in self.dTagHandlers:
+            self.dTagHandlers[sCurTag](sCurTag, aasSections, iCurTagLine, iLine);
+            cOpTags += 1;
+        elif sCurTag.startswith('@op'):
+            self.errorComment(iCurTagLine, 'Unknown tag: %s' % (sCurTag));
+        elif sCurTag == '@default':
+            sFlatDefault = self.flattenAllSections(aasSections);
+
+        #
+        # Don't allow default text in blocks containing @op*.
+        #
+        if cOpTags > 0 and sFlatDefault:
+            self.errorComment(0, 'Untagged comment text is not allowed with @op*: %s' % (sFlatDefault,));
+
+        return True;
+
+    def parseMacroInvocation(self, sInvocation):
+        """
+        Parses a macro invocation.
+
+        Returns a tuple, first element is the offset following the macro
+        invocation. The second element is a list of macro arguments, where the
+        zero'th is the macro name.
+        """
+        # First the name.
+        offOpen = sInvocation.find('(');
+        if offOpen <= 0:
+            self.raiseError("macro invocation open parenthesis not found");
+        sName = sInvocation[:offOpen].strip();
+        if not self.oReMacroName.match(sName):
+            return self.error("invalid macro name '%s'" % (sName,));
+        asRet = [sName, ];
+
+        # Arguments.
+        iLine    = self.iLine;
+        cDepth   = 1;
+        off      = offOpen + 1;
+        offStart = off;
+        chQuote  = None;
+        while cDepth > 0:
+            if off >= len(sInvocation):
+                if iLine >= len(self.asLines):
+                    self.error('macro invocation beyond end of file');
+                    return (off, asRet);
+                sInvocation += self.asLines[iLine];
+                iLine += 1;
+            ch = sInvocation[off];
+
+            if chQuote:
+                if ch == '\\' and off + 1 < len(sInvocation):
+                    off += 1;
+                elif ch == chQuote:
+                    chQuote = None;
+            elif ch in ('"', '\'',):
+                chQuote = ch;
+            elif ch in (',', ')',):
+                if cDepth == 1:
+                    asRet.append(sInvocation[offStart:off].strip());
+                    offStart = off + 1;
+                if ch == ')':
+                    cDepth -= 1;
+            elif ch == '(':
+                cDepth += 1;
+            off += 1;
+
+        return (off, asRet);
+
+    def findAndParseMacroInvocationEx(self, sCode, sMacro):
+        """
+        Returns (len(sCode), None) if not found, parseMacroInvocation result if found.
+        """
+        offHit = sCode.find(sMacro);
+        if offHit >= 0 and sCode[offHit + len(sMacro):].strip()[0] == '(':
+            offAfter, asRet = self.parseMacroInvocation(sCode[offHit:])
+            return (offHit + offAfter, asRet);
+        return (len(sCode), None);
+
+    def findAndParseMacroInvocation(self, sCode, sMacro):
+        """
+        Returns None if not found, arguments as per parseMacroInvocation if found.
+        """
+        return self.findAndParseMacroInvocationEx(sCode, sMacro)[1];
+
+    def findAndParseFirstMacroInvocation(self, sCode, asMacro):
+        """
+        Returns same as findAndParseMacroInvocation.
+        """
+        for sMacro in asMacro:
+            asRet = self.findAndParseMacroInvocation(sCode, sMacro);
+            if asRet is not None:
+                return asRet;
+        return None;
+
+    def workerIemOpMnemonicEx(self, sMacro, sStats, sAsm, sForm, sUpper, sLower,  # pylint: disable=too-many-arguments
+                              sDisHints, sIemHints, asOperands):
+        """
+        Processes one of the a IEMOP_MNEMONIC0EX, IEMOP_MNEMONIC1EX, IEMOP_MNEMONIC2EX,
+        IEMOP_MNEMONIC3EX, and IEMOP_MNEMONIC4EX macros.
+        """
+        #
+        # Some invocation checks.
+        #
+        if sUpper != sUpper.upper():
+            self.error('%s: bad a_Upper parameter: %s' % (sMacro, sUpper,));
+        if sLower != sLower.lower():
+            self.error('%s: bad a_Lower parameter: %s' % (sMacro, sLower,));
+        if sUpper.lower() != sLower:
+            self.error('%s: a_Upper and a_Lower parameters does not match: %s vs %s' % (sMacro, sUpper, sLower,));
+        if not self.oReMnemonic.match(sLower):
+            self.error('%s: invalid a_Lower: %s  (valid: %s)' % (sMacro, sLower, self.oReMnemonic.pattern,));
+
+        #
+        # Check if sIemHints tells us to not consider this macro invocation.
+        #
+        if sIemHints.find('IEMOPHINT_SKIP_PYTHON') >= 0:
+            return True;
+
+        # Apply to the last instruction only for now.
+        if not self.aoCurInstrs:
+            self.addInstruction();
+        oInstr = self.aoCurInstrs[-1];
+        if oInstr.iLineMnemonicMacro == -1:
+            oInstr.iLineMnemonicMacro = self.iLine;
+        else:
+            self.error('%s: already saw a IEMOP_MNEMONIC* macro on line %u for this instruction'
+                       % (sMacro, oInstr.iLineMnemonicMacro,));
+
+        # Mnemonic
+        if oInstr.sMnemonic is None:
+            oInstr.sMnemonic = sLower;
+        elif oInstr.sMnemonic != sLower:
+            self.error('%s: current instruction and a_Lower does not match: %s vs %s' % (sMacro, oInstr.sMnemonic, sLower,));
+
+        # Process operands.
+        if len(oInstr.aoOperands) not in [0, len(asOperands)]:
+            self.error('%s: number of operands given by @opN does not match macro: %s vs %s'
+                       % (sMacro, len(oInstr.aoOperands), len(asOperands),));
+        for iOperand, sType in enumerate(asOperands):
+            sWhere = g_kdOpTypes.get(sType, [None, None])[1];
+            if sWhere is None:
+                self.error('%s: unknown a_Op%u value: %s' % (sMacro, iOperand + 1, sType));
+                if iOperand < len(oInstr.aoOperands): # error recovery.
+                    sWhere = oInstr.aoOperands[iOperand].sWhere;
+                    sType  = oInstr.aoOperands[iOperand].sType;
+                else:
+                    sWhere = 'reg';
+                    sType  = 'Gb';
+            if iOperand == len(oInstr.aoOperands):
+                oInstr.aoOperands.append(Operand(sWhere, sType))
+            elif oInstr.aoOperands[iOperand].sWhere != sWhere or oInstr.aoOperands[iOperand].sType != sType:
+                self.error('%s: @op%u and a_Op%u mismatch: %s:%s vs %s:%s'
+                           % (sMacro, iOperand + 1, iOperand + 1, oInstr.aoOperands[iOperand].sWhere,
+                              oInstr.aoOperands[iOperand].sType, sWhere, sType,));
+
+        # Encoding.
+        if sForm not in g_kdIemForms:
+            self.error('%s: unknown a_Form value: %s' % (sMacro, sForm,));
+        else:
+            if oInstr.sEncoding is None:
+                oInstr.sEncoding = g_kdIemForms[sForm][0];
+            elif g_kdIemForms[sForm][0] != oInstr.sEncoding:
+                self.error('%s: current instruction @openc and a_Form does not match: %s vs %s (%s)'
+                           % (sMacro, oInstr.sEncoding, g_kdIemForms[sForm], sForm));
+
+            # Check the parameter locations for the encoding.
+            if g_kdIemForms[sForm][1] is not None:
+                if len(g_kdIemForms[sForm][1]) > len(oInstr.aoOperands):
+                    self.error('%s: The a_Form=%s has a different operand count: %s (form) vs %s'
+                               % (sMacro, sForm, len(g_kdIemForms[sForm][1]), len(oInstr.aoOperands) ));
+                else:
+                    for iOperand, sWhere in enumerate(g_kdIemForms[sForm][1]):
+                        if oInstr.aoOperands[iOperand].sWhere != sWhere:
+                            self.error('%s: current instruction @op%u and a_Form location does not match: %s vs %s (%s)'
+                                       % (sMacro, iOperand + 1, oInstr.aoOperands[iOperand].sWhere, sWhere, sForm,));
+                        sOpFormMatch = g_kdOpTypes[oInstr.aoOperands[iOperand].sType][4];
+                        if    (sOpFormMatch in [ 'REG', 'MEM', ] and sForm.find('_' + sOpFormMatch) < 0) \
+                           or (sOpFormMatch in [ 'FIXED', ]      and sForm.find(sOpFormMatch) < 0) \
+                           or (sOpFormMatch == 'RM' and (sForm.find('_MEM') > 0 or sForm.find('_REG') > 0) ) \
+                           or (sOpFormMatch == 'V'  and (   not (sForm.find('VEX') > 0 or sForm.find('XOP')) \
+                                                         or sForm.replace('VEX','').find('V') < 0) ):
+                            self.error('%s: current instruction @op%u and a_Form type does not match: %s/%s vs %s'
+                                       % (sMacro, iOperand + 1, oInstr.aoOperands[iOperand].sType, sOpFormMatch, sForm, ));
+                    if len(g_kdIemForms[sForm][1]) < len(oInstr.aoOperands):
+                        for iOperand in range(len(g_kdIemForms[sForm][1]), len(oInstr.aoOperands)):
+                            if    oInstr.aoOperands[iOperand].sType != 'FIXED' \
+                              and g_kdOpTypes[oInstr.aoOperands[iOperand].sType][0] != 'IDX_ParseFixedReg':
+                                self.error('%s: Expected FIXED type operand #%u following operands given by a_Form=%s: %s (%s)'
+                                           % (sMacro, iOperand, sForm, oInstr.aoOperands[iOperand].sType,
+                                              oInstr.aoOperands[iOperand].sWhere));
+
+
+            # Check @opcodesub
+            if oInstr.sSubOpcode \
+              and g_kdIemForms[sForm][2] \
+              and oInstr.sSubOpcode.find(g_kdIemForms[sForm][2]) < 0:
+                self.error('%s: current instruction @opcodesub and a_Form does not match: %s vs %s (%s)'
+                            % (sMacro, oInstr.sSubOpcode, g_kdIemForms[sForm][2], sForm,));
+
+        # Stats.
+        if not self.oReStatsName.match(sStats):
+            self.error('%s: invalid a_Stats value: %s' % (sMacro, sStats,));
+        elif oInstr.sStats is None:
+            oInstr.sStats = sStats;
+        elif oInstr.sStats != sStats:
+            self.error('%s: mismatching @opstats and a_Stats value: %s vs %s'
+                       % (sMacro, oInstr.sStats, sStats,));
+
+        # Process the hints (simply merge with @ophints w/o checking anything).
+        for sHint in sDisHints.split('|'):
+            sHint = sHint.strip();
+            if sHint.startswith('DISOPTYPE_'):
+                sShortHint = sHint[len('DISOPTYPE_'):].lower();
+                if sShortHint in g_kdHints:
+                    oInstr.dHints[sShortHint] = True; # (dummy value, using dictionary for speed)
+                else:
+                    self.error('%s: unknown a_fDisHints value: %s' % (sMacro, sHint,));
+            elif sHint != '0':
+                self.error('%s: expected a_fDisHints value: %s' % (sMacro, sHint,));
+
+        for sHint in sIemHints.split('|'):
+            sHint = sHint.strip();
+            if sHint.startswith('IEMOPHINT_'):
+                sShortHint = sHint[len('IEMOPHINT_'):].lower();
+                if sShortHint in g_kdHints:
+                    oInstr.dHints[sShortHint] = True; # (dummy value, using dictionary for speed)
+                else:
+                    self.error('%s: unknown a_fIemHints value: %s' % (sMacro, sHint,));
+            elif sHint != '0':
+                self.error('%s: expected a_fIemHints value: %s' % (sMacro, sHint,));
+
+        _ = sAsm;
+        return True;
+
+    def workerIemOpMnemonic(self, sMacro, sForm, sUpper, sLower, sDisHints, sIemHints, asOperands):
+        """
+        Processes one of the a IEMOP_MNEMONIC0, IEMOP_MNEMONIC1, IEMOP_MNEMONIC2,
+        IEMOP_MNEMONIC3, and IEMOP_MNEMONIC4 macros.
+        """
+        if not asOperands:
+            return self.workerIemOpMnemonicEx(sMacro, sLower, sLower, sForm, sUpper, sLower, sDisHints, sIemHints, asOperands);
+        return self.workerIemOpMnemonicEx(sMacro, sLower + '_' + '_'.join(asOperands), sLower + ' ' + ','.join(asOperands),
+                                          sForm, sUpper, sLower, sDisHints, sIemHints, asOperands);
+
+    def checkCodeForMacro(self, sCode):
+        """
+        Checks code for relevant macro invocation.
+        """
+        #
+        # Scan macro invocations.
+        #
+        if sCode.find('(') > 0:
+            # Look for instruction decoder function definitions. ASSUME single line.
+            asArgs = self.findAndParseFirstMacroInvocation(sCode,
+                                                           [ 'FNIEMOP_DEF',
+                                                             'FNIEMOP_STUB',
+                                                             'FNIEMOP_STUB_1',
+                                                             'FNIEMOP_UD_STUB',
+                                                             'FNIEMOP_UD_STUB_1' ]);
+            if asArgs is not None:
+                sFunction = asArgs[1];
+
+                if not self.aoCurInstrs:
+                    self.addInstruction();
+                for oInstr in self.aoCurInstrs:
+                    if oInstr.iLineFnIemOpMacro == -1:
+                        oInstr.iLineFnIemOpMacro = self.iLine;
+                    else:
+                        self.error('%s: already seen a FNIEMOP_XXX macro for %s' % (asArgs[0], oInstr,) );
+                self.setInstrunctionAttrib('sFunction', sFunction);
+                self.setInstrunctionAttrib('fStub', asArgs[0].find('STUB') > 0, fOverwrite = True);
+                self.setInstrunctionAttrib('fUdStub', asArgs[0].find('UD_STUB') > 0, fOverwrite = True);
+                if asArgs[0].find('STUB') > 0:
+                    self.doneInstructions();
+                return True;
+
+            # IEMOP_HLP_DONE_VEX_DECODING_*
+            asArgs = self.findAndParseFirstMacroInvocation(sCode,
+                                                           [ 'IEMOP_HLP_DONE_VEX_DECODING',
+                                                             'IEMOP_HLP_DONE_VEX_DECODING_L0',
+                                                             'IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV',
+                                                             'IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV',
+                                                             ]);
+            if asArgs is not None:
+                sMacro = asArgs[0];
+                if sMacro in ('IEMOP_HLP_DONE_VEX_DECODING_L0', 'IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV', ):
+                    for oInstr in self.aoCurInstrs:
+                        if 'vex_l_zero' not in oInstr.dHints:
+                            if oInstr.iLineMnemonicMacro >= 0:
+                                self.errorOnLine(oInstr.iLineMnemonicMacro,
+                                                 'Missing IEMOPHINT_VEX_L_ZERO! (%s on line %d)' % (sMacro, self.iLine,));
+                            oInstr.dHints['vex_l_zero'] = True;
+                return True;
+
+            #
+            # IEMOP_MNEMONIC*
+            #
+
+            # IEMOP_MNEMONIC(a_Stats, a_szMnemonic) IEMOP_INC_STATS(a_Stats)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC');
+            if asArgs is not None:
+                if len(self.aoCurInstrs) == 1:
+                    oInstr = self.aoCurInstrs[0];
+                    if oInstr.sStats is None:
+                        oInstr.sStats = asArgs[1];
+                    self.deriveMnemonicAndOperandsFromStats(oInstr, asArgs[1]);
+
+            # IEMOP_MNEMONIC0EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC0EX');
+            if asArgs is not None:
+                self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[6], asArgs[7],
+                                           []);
+            # IEMOP_MNEMONIC1EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC1EX');
+            if asArgs is not None:
+                self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[7], asArgs[8],
+                                           [asArgs[6],]);
+            # IEMOP_MNEMONIC2EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC2EX');
+            if asArgs is not None:
+                self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[8], asArgs[9],
+                                           [asArgs[6], asArgs[7]]);
+            # IEMOP_MNEMONIC3EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC3EX');
+            if asArgs is not None:
+                self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[9],
+                                           asArgs[10], [asArgs[6], asArgs[7], asArgs[8],]);
+            # IEMOP_MNEMONIC4EX(a_Stats, a_szMnemonic, a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints,
+            #                   a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC4EX');
+            if asArgs is not None:
+                self.workerIemOpMnemonicEx(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], asArgs[10],
+                                           asArgs[11], [asArgs[6], asArgs[7], asArgs[8], asArgs[9],]);
+
+            # IEMOP_MNEMONIC0(a_Form, a_Upper, a_Lower, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC0');
+            if asArgs is not None:
+                self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[4], asArgs[5], []);
+            # IEMOP_MNEMONIC1(a_Form, a_Upper, a_Lower, a_Op1, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC1');
+            if asArgs is not None:
+                self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[5], asArgs[6], [asArgs[4],]);
+            # IEMOP_MNEMONIC2(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC2');
+            if asArgs is not None:
+                self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[6], asArgs[7],
+                                         [asArgs[4], asArgs[5],]);
+            # IEMOP_MNEMONIC3(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC3');
+            if asArgs is not None:
+                self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[7], asArgs[8],
+                                         [asArgs[4], asArgs[5], asArgs[6],]);
+            # IEMOP_MNEMONIC4(a_Form, a_Upper, a_Lower, a_Op1, a_Op2, a_Op3, a_Op4, a_fDisHints, a_fIemHints)
+            asArgs = self.findAndParseMacroInvocation(sCode, 'IEMOP_MNEMONIC4');
+            if asArgs is not None:
+                self.workerIemOpMnemonic(asArgs[0], asArgs[1], asArgs[2], asArgs[3], asArgs[8], asArgs[9],
+                                         [asArgs[4], asArgs[5], asArgs[6], asArgs[7],]);
+
+        return False;
+
+
+    def parse(self):
+        """
+        Parses the given file.
+        Returns number or errors.
+        Raises exception on fatal trouble.
+        """
+        #self.debug('Parsing %s' % (self.sSrcFile,));
+
+        while self.iLine < len(self.asLines):
+            sLine = self.asLines[self.iLine];
+            self.iLine  += 1;
+
+            # We only look for comments, so only lines with a slash might possibly
+            # influence the parser state.
+            offSlash = sLine.find('/');
+            if offSlash >= 0:
+                if offSlash + 1 >= len(sLine)  or  sLine[offSlash + 1] != '/'  or  self.iState != self.kiCode:
+                    offLine = 0;
+                    while offLine < len(sLine):
+                        if self.iState == self.kiCode:
+                            offHit = sLine.find('/*', offLine); # only multiline comments for now.
+                            if offHit >= 0:
+                                self.checkCodeForMacro(sLine[offLine:offHit]);
+                                self.sComment     = '';
+                                self.iCommentLine = self.iLine;
+                                self.iState       = self.kiCommentMulti;
+                                offLine = offHit + 2;
+                            else:
+                                self.checkCodeForMacro(sLine[offLine:]);
+                                offLine = len(sLine);
+
+                        elif self.iState == self.kiCommentMulti:
+                            offHit = sLine.find('*/', offLine);
+                            if offHit >= 0:
+                                self.sComment += sLine[offLine:offHit];
+                                self.iState    = self.kiCode;
+                                offLine = offHit + 2;
+                                self.parseComment();
+                            else:
+                                self.sComment += sLine[offLine:];
+                                offLine = len(sLine);
+                        else:
+                            assert False;
+                # C++ line comment.
+                elif offSlash > 0:
+                    self.checkCodeForMacro(sLine[:offSlash]);
+
+            # No slash, but append the line if in multi-line comment.
+            elif self.iState == self.kiCommentMulti:
+                #self.debug('line %d: multi' % (self.iLine,));
+                self.sComment += sLine;
+
+            # No slash, but check code line for relevant macro.
+            elif self.iState == self.kiCode and sLine.find('IEMOP_') >= 0:
+                #self.debug('line %d: macro' % (self.iLine,));
+                self.checkCodeForMacro(sLine);
+
+            # If the line is a '}' in the first position, complete the instructions.
+            elif self.iState == self.kiCode and sLine[0] == '}':
+                #self.debug('line %d: }' % (self.iLine,));
+                self.doneInstructions();
+
+            # Look for instruction table on the form 'IEM_STATIC const PFNIEMOP g_apfnVexMap3'
+            # so we can check/add @oppfx info from it.
+            elif self.iState == self.kiCode and sLine.find('PFNIEMOP') > 0 and self.oReFunTable.match(sLine):
+                self.parseFunctionTable(sLine);
+
+        self.doneInstructions();
+        self.debug('%3s%% / %3s stubs out of %4s instructions in %s'
+                   % (self.cTotalStubs * 100 // self.cTotalInstr, self.cTotalStubs, self.cTotalInstr,
+                      os.path.basename(self.sSrcFile),));
+        return self.printErrors();
+
+
+def __parseFileByName(sSrcFile, sDefaultMap):
+    """
+    Parses one source file for instruction specfications.
+    """
+    #
+    # Read sSrcFile into a line array.
+    #
+    try:
+        oFile = open(sSrcFile, "r");    # pylint: disable=consider-using-with
+    except Exception as oXcpt:
+        raise Exception("failed to open %s for reading: %s" % (sSrcFile, oXcpt,));
+    try:
+        asLines = oFile.readlines();
+    except Exception as oXcpt:
+        raise Exception("failed to read %s: %s" % (sSrcFile, oXcpt,));
+    finally:
+        oFile.close();
+
+    #
+    # Do the parsing.
+    #
+    try:
+        cErrors = SimpleParser(sSrcFile, asLines, sDefaultMap).parse();
+    except ParserException as oXcpt:
+        print(str(oXcpt));
+        raise;
+
+    return cErrors;
+
+
+def __doTestCopying():
+    """
+    Executes the asCopyTests instructions.
+    """
+    asErrors = [];
+    for oDstInstr in g_aoAllInstructions:
+        if oDstInstr.asCopyTests:
+            for sSrcInstr in oDstInstr.asCopyTests:
+                oSrcInstr = g_dAllInstructionsByStat.get(sSrcInstr, None);
+                if oSrcInstr:
+                    aoSrcInstrs = [oSrcInstr,];
+                else:
+                    aoSrcInstrs = g_dAllInstructionsByFunction.get(sSrcInstr, []);
+                if aoSrcInstrs:
+                    for oSrcInstr in aoSrcInstrs:
+                        if oSrcInstr != oDstInstr:
+                            oDstInstr.aoTests.extend(oSrcInstr.aoTests);
+                        else:
+                            asErrors.append('%s:%s: error: @opcopytests reference "%s" matches the destination\n'
+                                            % ( oDstInstr.sSrcFile, oDstInstr.iLineCreated, sSrcInstr));
+                else:
+                    asErrors.append('%s:%s: error: @opcopytests reference "%s" not found\n'
+                                    % ( oDstInstr.sSrcFile, oDstInstr.iLineCreated, sSrcInstr));
+
+    if asErrors:
+        sys.stderr.write(u''.join(asErrors));
+    return len(asErrors);
+
+
+def __applyOnlyTest():
+    """
+    If g_aoOnlyTestInstructions contains any instructions, drop aoTests from
+    all other instructions so that only these get tested.
+    """
+    if g_aoOnlyTestInstructions:
+        for oInstr in g_aoAllInstructions:
+            if oInstr.aoTests:
+                if oInstr not in g_aoOnlyTestInstructions:
+                    oInstr.aoTests = [];
+    return 0;
+
+def __parseAll():
+    """
+    Parses all the IEMAllInstruction*.cpp.h files.
+
+    Raises exception on failure.
+    """
+    sSrcDir = os.path.dirname(os.path.abspath(__file__));
+    cErrors = 0;
+    for sDefaultMap, sName in [
+        ( 'one',        'IEMAllInstructionsOneByte.cpp.h'),
+        ( 'two0f',      'IEMAllInstructionsTwoByte0f.cpp.h'),
+        ( 'three0f38',  'IEMAllInstructionsThree0f38.cpp.h'),
+        ( 'three0f3a',  'IEMAllInstructionsThree0f3a.cpp.h'),
+        ( 'vexmap1',    'IEMAllInstructionsVexMap1.cpp.h'),
+        ( 'vexmap2',    'IEMAllInstructionsVexMap2.cpp.h'),
+        ( 'vexmap3',    'IEMAllInstructionsVexMap3.cpp.h'),
+        ( '3dnow',      'IEMAllInstructions3DNow.cpp.h'),
+    ]:
+        cErrors += __parseFileByName(os.path.join(sSrcDir, sName), sDefaultMap);
+    cErrors += __doTestCopying();
+    cErrors += __applyOnlyTest();
+
+    # Total stub stats:
+    cTotalStubs = 0;
+    for oInstr in g_aoAllInstructions:
+        cTotalStubs += oInstr.fStub;
+    print('debug: %3s%% / %3s stubs out of %4s instructions in total'
+          % (cTotalStubs * 100 // len(g_aoAllInstructions), cTotalStubs, len(g_aoAllInstructions),));
+
+    if cErrors != 0:
+        #raise Exception('%d parse errors' % (cErrors,));
+        sys.exit(1);
+    return True;
+
+
+
+__parseAll();
+
+
+#
+# Generators (may perhaps move later).
+#
+def __formatDisassemblerTableEntry(oInstr):
+    """
+    """
+    sMacro = 'OP';
+    cMaxOperands = 3;
+    if len(oInstr.aoOperands) > 3:
+        sMacro = 'OPVEX'
+        cMaxOperands = 4;
+        assert len(oInstr.aoOperands) <= cMaxOperands;
+
+    #
+    # Format string.
+    #
+    sTmp = '%s("%s' % (sMacro, oInstr.sMnemonic,);
+    for iOperand, oOperand in enumerate(oInstr.aoOperands):
+        sTmp += ' ' if iOperand == 0 else ',';
+        if g_kdOpTypes[oOperand.sType][2][0] != '%':        ## @todo remove upper() later.
+            sTmp += g_kdOpTypes[oOperand.sType][2].upper(); ## @todo remove upper() later.
+        else:
+            sTmp += g_kdOpTypes[oOperand.sType][2];
+    sTmp += '",';
+    asColumns = [ sTmp, ];
+
+    #
+    # Decoders.
+    #
+    iStart = len(asColumns);
+    if oInstr.sEncoding is None:
+        pass;
+    elif oInstr.sEncoding == 'ModR/M':
+        # ASSUME the first operand is using the ModR/M encoding
+        assert len(oInstr.aoOperands) >= 1 and oInstr.aoOperands[0].usesModRM();
+        asColumns.append('IDX_ParseModRM,');
+    elif oInstr.sEncoding in [ 'prefix', ]:
+        for oOperand in oInstr.aoOperands:
+            asColumns.append('0,');
+    elif oInstr.sEncoding in [ 'fixed', 'VEX.fixed' ]:
+        pass;
+    elif oInstr.sEncoding == 'VEX.ModR/M':
+        asColumns.append('IDX_ParseModRM,');
+    elif oInstr.sEncoding == 'vex2':
+        asColumns.append('IDX_ParseVex2b,')
+    elif oInstr.sEncoding == 'vex3':
+        asColumns.append('IDX_ParseVex3b,')
+    elif oInstr.sEncoding in g_dInstructionMaps:
+        asColumns.append(g_dInstructionMaps[oInstr.sEncoding].sDisParse + ',');
+    else:
+        ## @todo
+        #IDX_ParseTwoByteEsc,
+        #IDX_ParseGrp1,
+        #IDX_ParseShiftGrp2,
+        #IDX_ParseGrp3,
+        #IDX_ParseGrp4,
+        #IDX_ParseGrp5,
+        #IDX_Parse3DNow,
+        #IDX_ParseGrp6,
+        #IDX_ParseGrp7,
+        #IDX_ParseGrp8,
+        #IDX_ParseGrp9,
+        #IDX_ParseGrp10,
+        #IDX_ParseGrp12,
+        #IDX_ParseGrp13,
+        #IDX_ParseGrp14,
+        #IDX_ParseGrp15,
+        #IDX_ParseGrp16,
+        #IDX_ParseThreeByteEsc4,
+        #IDX_ParseThreeByteEsc5,
+        #IDX_ParseModFence,
+        #IDX_ParseEscFP,
+        #IDX_ParseNopPause,
+        #IDX_ParseInvOpModRM,
+        assert False, str(oInstr);
+
+    # Check for immediates and stuff in the remaining operands.
+    for oOperand in oInstr.aoOperands[len(asColumns) - iStart:]:
+        sIdx = g_kdOpTypes[oOperand.sType][0];
+        #if sIdx != 'IDX_UseModRM':
+        asColumns.append(sIdx + ',');
+    asColumns.extend(['0,'] * (cMaxOperands - (len(asColumns) - iStart)));
+
+    #
+    # Opcode and operands.
+    #
+    assert oInstr.sDisEnum, str(oInstr);
+    asColumns.append(oInstr.sDisEnum + ',');
+    iStart = len(asColumns)
+    for oOperand in oInstr.aoOperands:
+        asColumns.append('OP_PARM_' + g_kdOpTypes[oOperand.sType][3] + ',');
+    asColumns.extend(['OP_PARM_NONE,'] * (cMaxOperands - (len(asColumns) - iStart)));
+
+    #
+    # Flags.
+    #
+    sTmp = '';
+    for sHint in sorted(oInstr.dHints.keys()):
+        sDefine = g_kdHints[sHint];
+        if sDefine.startswith('DISOPTYPE_'):
+            if sTmp:
+                sTmp += ' | ' + sDefine;
+            else:
+                sTmp += sDefine;
+    if sTmp:
+        sTmp += '),';
+    else:
+        sTmp += '0),';
+    asColumns.append(sTmp);
+
+    #
+    # Format the columns into a line.
+    #
+    aoffColumns = [4, 29, 49, 65, 77, 89, 109, 125, 141, 157, 183, 199];
+    sLine = '';
+    for i, s in enumerate(asColumns):
+        if len(sLine) < aoffColumns[i]:
+            sLine += ' ' * (aoffColumns[i] - len(sLine));
+        else:
+            sLine += ' ';
+        sLine += s;
+
+    # OP("psrlw %Vdq,%Wdq", IDX_ParseModRM, IDX_UseModRM, 0, OP_PSRLW, OP_PARM_Vdq, OP_PARM_Wdq, OP_PARM_NONE,
+    # DISOPTYPE_HARMLESS),
+    # define OP(pszOpcode, idxParse1, idxParse2, idxParse3, opcode, param1, param2, param3, optype) \
+    # { pszOpcode, idxParse1, idxParse2, idxParse3, 0, opcode, param1, param2, param3, 0, 0, optype }
+    return sLine;
+
+def __checkIfShortTable(aoTableOrdered, oMap):
+    """
+    Returns (iInstr, cInstructions, fShortTable)
+    """
+
+    # Determin how much we can trim off.
+    cInstructions = len(aoTableOrdered);
+    while cInstructions > 0 and aoTableOrdered[cInstructions - 1] is None:
+        cInstructions -= 1;
+
+    iInstr = 0;
+    while iInstr < cInstructions and aoTableOrdered[iInstr] is None:
+        iInstr += 1;
+
+    # If we can save more than 30%, we go for the short table version.
+    if iInstr + len(aoTableOrdered) - cInstructions >= len(aoTableOrdered) // 30:
+        return (iInstr, cInstructions, True);
+    _ = oMap; # Use this for overriding.
+
+    # Output the full table.
+    return (0, len(aoTableOrdered), False);
+
+def generateDisassemblerTables(oDstFile = sys.stdout):
+    """
+    Generates disassembler tables.
+    """
+
+    #
+    # The disassembler uses a slightly different table layout to save space,
+    # since several of the prefix varia
+    #
+    aoDisasmMaps = [];
+    for sName, oMap in sorted(iter(g_dInstructionMaps.items()),
+                              key = lambda aKV: aKV[1].sEncoding + ''.join(aKV[1].asLeadOpcodes)):
+        if oMap.sSelector != 'byte+pfx':
+            aoDisasmMaps.append(oMap);
+        else:
+            # Split the map by prefix.
+            aoDisasmMaps.append(oMap.copy(oMap.sName,         'none'));
+            aoDisasmMaps.append(oMap.copy(oMap.sName + '_66', '0x66'));
+            aoDisasmMaps.append(oMap.copy(oMap.sName + '_F3', '0xf3'));
+            aoDisasmMaps.append(oMap.copy(oMap.sName + '_F2', '0xf2'));
+
+    #
+    # Dump each map.
+    #
+    asHeaderLines = [];
+    print("debug: maps=%s\n" % (', '.join([oMap.sName for oMap in aoDisasmMaps]),));
+    for oMap in aoDisasmMaps:
+        sName = oMap.sName;
+
+        if not sName.startswith("vex"): continue; # only looking at the vex maps at the moment.
+
+        #
+        # Get the instructions for the map and see if we can do a short version or not.
+        #
+        aoTableOrder    = oMap.getInstructionsInTableOrder();
+        cEntriesPerByte = oMap.getEntriesPerByte();
+        (iInstrStart, iInstrEnd, fShortTable) = __checkIfShortTable(aoTableOrder, oMap);
+
+        #
+        # Output the table start.
+        # Note! Short tables are static and only accessible via the map range record.
+        #
+        asLines = [];
+        asLines.append('/* Generated from: %-11s  Selector: %-7s  Encoding: %-7s  Lead bytes opcodes: %s */'
+                       % ( oMap.sName, oMap.sSelector, oMap.sEncoding, ' '.join(oMap.asLeadOpcodes), ));
+        if fShortTable:
+            asLines.append('%sconst DISOPCODE %s[] =' % ('static ' if fShortTable else '', oMap.getDisasTableName(),));
+        else:
+            asHeaderLines.append('extern const DISOPCODE %s[%d];'  % (oMap.getDisasTableName(), iInstrEnd - iInstrStart,));
+            asLines.append(             'const DISOPCODE %s[%d] =' % (oMap.getDisasTableName(), iInstrEnd - iInstrStart,));
+        asLines.append('{');
+
+        if fShortTable and (iInstrStart & ((0x10 * cEntriesPerByte) - 1)) != 0:
+            asLines.append('    /* %#04x: */' % (iInstrStart,));
+
+        #
+        # Output the instructions.
+        #
+        iInstr = iInstrStart;
+        while iInstr < iInstrEnd:
+            oInstr = aoTableOrder[iInstr];
+            if (iInstr & ((0x10 * cEntriesPerByte) - 1)) == 0:
+                if iInstr != iInstrStart:
+                    asLines.append('');
+                asLines.append('    /* %x */' % ((iInstr // cEntriesPerByte) >> 4,));
+
+            if oInstr is None:
+                # Invalid. Optimize blocks of invalid instructions.
+                cInvalidInstrs = 1;
+                while iInstr + cInvalidInstrs < len(aoTableOrder) and aoTableOrder[iInstr + cInvalidInstrs] is None:
+                    cInvalidInstrs += 1;
+                if (iInstr & (0x10 * cEntriesPerByte - 1)) == 0 and cInvalidInstrs >= 0x10 * cEntriesPerByte:
+                    asLines.append('    INVALID_OPCODE_BLOCK_%u,' % (0x10 * cEntriesPerByte,));
+                    iInstr += 0x10 * cEntriesPerByte - 1;
+                elif cEntriesPerByte > 1:
+                    if (iInstr & (cEntriesPerByte - 1)) == 0 and cInvalidInstrs >= cEntriesPerByte:
+                        asLines.append('    INVALID_OPCODE_BLOCK_%u,' % (cEntriesPerByte,));
+                        iInstr += 3;
+                    else:
+                        asLines.append('    /* %#04x/%d */ INVALID_OPCODE,'
+                                       % (iInstr // cEntriesPerByte, iInstr % cEntriesPerByte));
+                else:
+                    asLines.append('    /* %#04x */ INVALID_OPCODE,' % (iInstr));
+            elif isinstance(oInstr, list):
+                if len(oInstr) != 0:
+                    asLines.append('    /* %#04x */ ComplicatedListStuffNeedingWrapper, /* \n -- %s */'
+                                   % (iInstr, '\n -- '.join([str(oItem) for oItem in oInstr]),));
+                else:
+                    asLines.append(__formatDisassemblerTableEntry(oInstr));
+            else:
+                asLines.append(__formatDisassemblerTableEntry(oInstr));
+
+            iInstr += 1;
+
+        if iInstrStart >= iInstrEnd:
+            asLines.append('    /* dummy */ INVALID_OPCODE');
+
+        asLines.append('};');
+        asLines.append('AssertCompile(RT_ELEMENTS(%s) == %s);' % (oMap.getDisasTableName(), iInstrEnd - iInstrStart,));
+
+        #
+        # We always emit a map range record, assuming the linker will eliminate the unnecessary ones.
+        #
+        asHeaderLines.append('extern const DISOPMAPDESC %sRange;'  % (oMap.getDisasRangeName()));
+        asLines.append('const DISOPMAPDESC %s = { &%s[0], %#04x, RT_ELEMENTS(%s) };'
+                       % (oMap.getDisasRangeName(), oMap.getDisasTableName(), iInstrStart, oMap.getDisasTableName(),));
+
+        #
+        # Write out the lines.
+        #
+        oDstFile.write('\n'.join(asLines));
+        oDstFile.write('\n');
+        oDstFile.write('\n');
+        #break; #for now
+
+if __name__ == '__main__':
+    generateDisassemblerTables();
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h
new file mode 100644
index 00000000..a19eb792
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f38.cpp.h
@@ -0,0 +1,2187 @@
+/* $Id: IEMAllInstructionsThree0f38.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ *
+ * @remarks IEMAllInstructionsVexMap2.cpp.h is a VEX mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/** @name Three byte opcodes with first two bytes 0x0f 0x38
+ * @{
+ */
+
+FNIEMOP_DEF_2(iemOpCommonMmx_FullFull_To_Full_Ex, PFNIEMAIMPLMEDIAF2U64, pfnU64, bool, fSupported); /* in IEMAllInstructionsTwoByteOf.cpp.h */
+
+
+/**
+ * Common worker for SSSE3 instructions on the forms:
+ *      pxxx    xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. SSSE3 cpuid checks.
+ *
+ * @sa  iemOpCommonSse2_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSsse3_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,       IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSSE3_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE4.1 instructions on the forms:
+ *      pxxx    xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. SSE4.1 cpuid checks.
+ *
+ * @sa  iemOpCommonSse2_FullFull_To_Full, iemOpCommonSsse3_FullFull_To_Full,
+ *      iemOpCommonSse42_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse41_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,       IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE4.1 instructions on the forms:
+ *      pxxx    xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. SSE4.1 cpuid checks.
+ *
+ * Unlike iemOpCommonSse41_FullFull_To_Full, the @a pfnU128 worker function
+ * takes no FXSAVE state, just the operands.
+ *
+ * @sa  iemOpCommonSse2_FullFull_To_Full, iemOpCommonSsse3_FullFull_To_Full,
+ *      iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse42_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse41Opt_FullFull_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,       IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE4.2 instructions on the forms:
+ *      pxxx    xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. SSE4.2 cpuid checks.
+ *
+ * @sa  iemOpCommonSse2_FullFull_To_Full, iemOpCommonSsse3_FullFull_To_Full,
+ *      iemOpCommonSse41_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse42_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE42_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,       IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE42_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_2(pfnU128, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE-style AES-NI instructions of the form:
+ *      aesxxx  xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. AES-NI cpuid checks.
+ *
+ * Unlike iemOpCommonSse41_FullFull_To_Full, the @a pfnU128 worker function
+ * takes no FXSAVE state, just the operands.
+ *
+ * @sa  iemOpCommonSse2_FullFull_To_Full, iemOpCommonSsse3_FullFull_To_Full,
+ *      iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse42_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonAesNi_FullFull_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc, 1);
+        IEM_MC_MAYBE_RAISE_AESNI_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,       IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_AESNI_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode      0x0f 0x38 0x00. */
+FNIEMOP_DEF(iemOp_pshufb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSHUFB, pshufb, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pshufb_u64,&iemAImpl_pshufb_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x00. */
+FNIEMOP_DEF(iemOp_pshufb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSHUFB, pshufb, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pshufb_u128, iemAImpl_pshufb_u128_fallback));
+
+}
+
+
+/*  Opcode      0x0f 0x38 0x01. */
+FNIEMOP_DEF(iemOp_phaddw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PHADDW, phaddw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddw_u64,&iemAImpl_phaddw_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x01. */
+FNIEMOP_DEF(iemOp_phaddw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PHADDW, phaddw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddw_u128, iemAImpl_phaddw_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x02. */
+FNIEMOP_DEF(iemOp_phaddd_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PHADDD, phaddd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddd_u64,&iemAImpl_phaddd_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x02. */
+FNIEMOP_DEF(iemOp_phaddd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PHADDD, phaddd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddd_u128, iemAImpl_phaddd_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x03. */
+FNIEMOP_DEF(iemOp_phaddsw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PHADDSW, phaddsw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddsw_u64,&iemAImpl_phaddsw_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x03. */
+FNIEMOP_DEF(iemOp_phaddsw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PHADDSW, phaddsw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phaddsw_u128, iemAImpl_phaddsw_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x04. */
+FNIEMOP_DEF(iemOp_pmaddubsw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMADDUBSW, pmaddubsw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pmaddubsw_u64, &iemAImpl_pmaddubsw_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x04. */
+FNIEMOP_DEF(iemOp_pmaddubsw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMADDUBSW, pmaddubsw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pmaddubsw_u128, iemAImpl_pmaddubsw_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x05. */
+FNIEMOP_DEF(iemOp_phsubw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PHSUBW, phsubw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubw_u64,&iemAImpl_phsubw_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x05. */
+FNIEMOP_DEF(iemOp_phsubw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PHSUBW, phsubw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubw_u128, iemAImpl_phsubw_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x06. */
+FNIEMOP_DEF(iemOp_phsubd_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PHSUBD, phsubd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubd_u64,&iemAImpl_phsubd_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+
+/** Opcode 0x66 0x0f 0x38 0x06. */
+FNIEMOP_DEF(iemOp_phsubd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PHSUBD, phsubd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubd_u128, iemAImpl_phsubd_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x07. */
+FNIEMOP_DEF(iemOp_phsubsw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PHSUBSW, phsubsw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubsw_u64,&iemAImpl_phsubsw_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x07. */
+FNIEMOP_DEF(iemOp_phsubsw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PHSUBSW, phsubsw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_phsubsw_u128, iemAImpl_phsubsw_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x08. */
+FNIEMOP_DEF(iemOp_psignb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSIGNB, psignb, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignb_u64, &iemAImpl_psignb_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x08. */
+FNIEMOP_DEF(iemOp_psignb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSIGNB, psignb, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignb_u128, iemAImpl_psignb_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x09. */
+FNIEMOP_DEF(iemOp_psignw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSIGNW, psignw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignw_u64, &iemAImpl_psignw_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x09. */
+FNIEMOP_DEF(iemOp_psignw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSIGNW, psignw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignw_u128, iemAImpl_psignw_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x0a. */
+FNIEMOP_DEF(iemOp_psignd_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSIGND, psignd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignd_u64, &iemAImpl_psignd_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x0a. */
+FNIEMOP_DEF(iemOp_psignd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSIGND, psignd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_psignd_u128, iemAImpl_psignd_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x0b. */
+FNIEMOP_DEF(iemOp_pmulhrsw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMULHRSW, pmulhrsw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pmulhrsw_u64, &iemAImpl_pmulhrsw_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x0b. */
+FNIEMOP_DEF(iemOp_pmulhrsw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMULHRSW, pmulhrsw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pmulhrsw_u128, iemAImpl_pmulhrsw_u128_fallback));
+
+}
+
+
+/*  Opcode      0x0f 0x38 0x0c - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x0c - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x0d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x0d - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x0e - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x0e - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x0f - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x0f - invalid (vex only). */
+
+
+/*  Opcode      0x0f 0x38 0x10 - invalid */
+
+
+/** Body for the *blend* instructions. */
+#define IEMOP_BODY_P_BLEND_X(a_Instr) \
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \
+    if (IEM_IS_MODRM_REG_MODE(bRm)) \
+    { \
+        /* \
+         * Register, register. \
+         */ \
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \
+        IEM_MC_BEGIN(3, 0); \
+        IEM_MC_ARG(PRTUINT128U,  puDst,  0); \
+        IEM_MC_ARG(PCRTUINT128U, puSrc,  1); \
+        IEM_MC_ARG(PCRTUINT128U, puMask, 2); \
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();  \
+        IEM_MC_PREPARE_SSE_USAGE();  \
+        IEM_MC_REF_XREG_U128(puDst,  IEM_GET_MODRM_REG(pVCpu, bRm)); \
+        IEM_MC_REF_XREG_U128_CONST(puSrc,  IEM_GET_MODRM_RM(pVCpu, bRm)); \
+        IEM_MC_REF_XREG_U128_CONST(puMask, 0); \
+        IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fSse41, \
+                                                             iemAImpl_ ## a_Instr ## _u128, \
+                                                             iemAImpl_ ## a_Instr ## _u128_fallback), \
+                                 puDst, puSrc, puMask); \
+        IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+        IEM_MC_END(); \
+    } \
+    else \
+    { \
+        /* \
+         * Register, memory. \
+         */ \
+        IEM_MC_BEGIN(3, 2); \
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc); \
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); \
+        IEM_MC_ARG(PRTUINT128U,            puDst,       0); \
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U, puSrc, uSrc, 1); \
+        IEM_MC_ARG(PCRTUINT128U,           puMask,      2); \
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); \
+        IEM_MC_PREPARE_SSE_USAGE(); \
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+        IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \
+        IEM_MC_REF_XREG_U128_CONST(puMask, 0); \
+        IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fSse41, \
+                                                             iemAImpl_ ## a_Instr ## _u128, \
+                                                             iemAImpl_ ## a_Instr ## _u128_fallback), \
+                                 puDst, puSrc, puMask); \
+        IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+        IEM_MC_END(); \
+    } \
+    (void)0
+
+/** Opcode 0x66 0x0f 0x38 0x10 (legacy only). */
+FNIEMOP_DEF(iemOp_pblendvb_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, PBLENDVB, pblendvb, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); /** @todo RM0 */
+    IEMOP_BODY_P_BLEND_X(pblendvb);
+}
+
+
+/*  Opcode      0x0f 0x38 0x11 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x11 - invalid */
+/*  Opcode      0x0f 0x38 0x12 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x12 - invalid */
+/*  Opcode      0x0f 0x38 0x13 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x13 - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x14 - invalid */
+
+
+/** Opcode 0x66 0x0f 0x38 0x14 (legacy only). */
+FNIEMOP_DEF(iemOp_blendvps_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, BLENDVPS, blendvps, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); /** @todo RM0 */
+    IEMOP_BODY_P_BLEND_X(blendvps);
+}
+
+
+/*  Opcode      0x0f 0x38 0x15 - invalid */
+
+
+/** Opcode 0x66 0x0f 0x38 0x15 (legacy only). */
+FNIEMOP_DEF(iemOp_blendvpd_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, BLENDVPD, blendvpd, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES); /** @todo RM0 */
+    IEMOP_BODY_P_BLEND_X(blendvpd);
+}
+
+
+/*  Opcode      0x0f 0x38 0x16 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x16 - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x17 - invalid */
+
+
+/** Opcode 0x66 0x0f 0x38 0x17 - invalid */
+FNIEMOP_DEF(iemOp_ptest_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PTEST, ptest, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc1,  0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc2,  1);
+        IEM_MC_ARG(uint32_t *,                  pEFlags, 2);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128_CONST(puSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc2,      IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_ptest_u128, puSrc1, puSrc2, pEFlags);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc1,        0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc2, uSrc2, 1);
+        IEM_MC_ARG(uint32_t *,                  pEFlags,       2);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128_CONST(puSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_ptest_u128, puSrc1, puSrc2, pEFlags);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode      0x0f 0x38 0x18 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x18 - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x19 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x19 - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x1a - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x1a - invalid (vex only). */
+/*  Opcode      0x0f 0x38 0x1b - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x1b - invalid */
+
+
+/** Opcode      0x0f 0x38 0x1c. */
+FNIEMOP_DEF(iemOp_pabsb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PABSB, pabsb, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsb_u64, &iemAImpl_pabsb_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x1c. */
+FNIEMOP_DEF(iemOp_pabsb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PABSB, pabsb, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsb_u128, iemAImpl_pabsb_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x1d. */
+FNIEMOP_DEF(iemOp_pabsw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PABSW, pabsw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsw_u64, &iemAImpl_pabsw_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x1d. */
+FNIEMOP_DEF(iemOp_pabsw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PABSW, pabsw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsw_u128, iemAImpl_pabsw_u128_fallback));
+
+}
+
+
+/** Opcode      0x0f 0x38 0x1e. */
+FNIEMOP_DEF(iemOp_pabsd_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PABSD, pabsd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsd_u64, &iemAImpl_pabsd_u64_fallback),
+                          IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x1e. */
+FNIEMOP_DEF(iemOp_pabsd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PABSD, pabsd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_pabsd_u128, iemAImpl_pabsd_u128_fallback));
+
+}
+
+
+/*  Opcode      0x0f 0x38 0x1f - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x1f - invalid */
+
+
+/** Body for the pmov{s,z}x* instructions. */
+#define IEMOP_BODY_PMOV_S_Z(a_Instr, a_SrcWidth) \
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \
+    if (IEM_IS_MODRM_REG_MODE(bRm)) \
+    { \
+        /* \
+         * Register, register. \
+         */ \
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \
+        IEM_MC_BEGIN(2, 0); \
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0); \
+        IEM_MC_ARG(uint64_t,    uSrc, 1); \
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();  \
+        IEM_MC_PREPARE_SSE_USAGE();  \
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword */); \
+        IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \
+        IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse41, \
+                                                             iemAImpl_ ## a_Instr ## _u128, \
+                                                             iemAImpl_v ## a_Instr ## _u128_fallback), \
+                                 puDst, uSrc); \
+        IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+        IEM_MC_END(); \
+    } \
+    else \
+    { \
+        /* \
+         * Register, memory. \
+         */ \
+        IEM_MC_BEGIN(2, 2); \
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc); \
+        IEM_MC_ARG(PRTUINT128U,                 puDst, 0); \
+        IEM_MC_ARG(uint ## a_SrcWidth ## _t,    uSrc, 1); \
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); \
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT(); \
+        IEM_MC_PREPARE_SSE_USAGE(); \
+        IEM_MC_FETCH_MEM_U## a_SrcWidth (uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+        IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm)); \
+        IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse41, \
+                                                             iemAImpl_ ## a_Instr ## _u128, \
+                                                             iemAImpl_v ## a_Instr ## _u128_fallback), \
+                                 puDst, uSrc); \
+        IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+        IEM_MC_END(); \
+    } \
+    (void)0
+
+
+/** Opcode 0x66 0x0f 0x38 0x20. */
+FNIEMOP_DEF(iemOp_pmovsxbw_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVSXBW, pmovsxbw, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovsxbw, 64);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x21. */
+FNIEMOP_DEF(iemOp_pmovsxbd_Vx_UxMd)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVSXBD, pmovsxbd, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovsxbd, 32);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x22. */
+FNIEMOP_DEF(iemOp_pmovsxbq_Vx_UxMw)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVSXBQ, pmovsxbq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovsxbq, 16);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x23. */
+FNIEMOP_DEF(iemOp_pmovsxwd_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVSXWD, pmovsxwd, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovsxwd, 64);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x24. */
+FNIEMOP_DEF(iemOp_pmovsxwq_Vx_UxMd)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVSXWQ, pmovsxwq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovsxwq, 32);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x25. */
+FNIEMOP_DEF(iemOp_pmovsxdq_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVSXDQ, pmovsxdq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovsxdq, 64);
+}
+
+
+/*  Opcode 0x66 0x0f 0x38 0x26 - invalid */
+/*  Opcode 0x66 0x0f 0x38 0x27 - invalid */
+
+
+/** Opcode 0x66 0x0f 0x38 0x28. */
+FNIEMOP_DEF(iemOp_pmuldq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMULDQ, pmuldq, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41Opt_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmuldq_u128, iemAImpl_pmuldq_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x29. */
+FNIEMOP_DEF(iemOp_pcmpeqq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PCMPEQQ, pcmpeqq, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pcmpeqq_u128, iemAImpl_pcmpeqq_u128_fallback));
+}
+
+
+/**
+ * @opcode      0x2a
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse4.1
+ * @opgroup     og_sse41_cachect
+ * @opxcpttype  1
+ * @optest      op1=-1 op2=2  -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movntdqa_Vdq_Mdq)
+{
+    IEMOP_MNEMONIC2(RM_MEM, MOVNTDQA, movntdqa, Vdq_WO, Mdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        /* Register, memory. */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660f382areg
+     * @opcode      0x2a
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x2b. */
+FNIEMOP_DEF(iemOp_packusdw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PACKUSDW, packusdw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse41Opt_FullFull_To_Full, iemAImpl_packusdw_u128);
+}
+
+
+/*  Opcode 0x66 0x0f 0x38 0x2c - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x2d - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x2e - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x2f - invalid (vex only). */
+
+/** Opcode 0x66 0x0f 0x38 0x30. */
+FNIEMOP_DEF(iemOp_pmovzxbw_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVZXBW, pmovzxbw, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovzxbw, 64);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x31. */
+FNIEMOP_DEF(iemOp_pmovzxbd_Vx_UxMd)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVZXBD, pmovzxbd, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovzxbd, 32);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x32. */
+FNIEMOP_DEF(iemOp_pmovzxbq_Vx_UxMw)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVZXBQ, pmovzxbq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovzxbq, 16);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x33. */
+FNIEMOP_DEF(iemOp_pmovzxwd_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVZXWD, pmovzxwd, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovzxwd, 64);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x34. */
+FNIEMOP_DEF(iemOp_pmovzxwq_Vx_UxMd)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVZXWQ, pmovzxwq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovzxwq, 32);
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x35. */
+FNIEMOP_DEF(iemOp_pmovzxdq_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(RM, PMOVZXDQ, pmovzxdq, Vx, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    IEMOP_BODY_PMOV_S_Z(pmovzxdq, 64);
+}
+
+
+/*  Opcode 0x66 0x0f 0x38 0x36 - invalid (vex only). */
+
+
+/** Opcode 0x66 0x0f 0x38 0x37. */
+FNIEMOP_DEF(iemOp_pcmpgtq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PCMPGTQ, pcmpgtq, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse42_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_pcmpgtq_u128, iemAImpl_pcmpgtq_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x38. */
+FNIEMOP_DEF(iemOp_pminsb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMINSB, pminsb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pminsb_u128, iemAImpl_pminsb_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x39. */
+FNIEMOP_DEF(iemOp_pminsd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMINSD, pminsd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pminsd_u128, iemAImpl_pminsd_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x3a. */
+FNIEMOP_DEF(iemOp_pminuw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMINUW, pminuw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pminuw_u128, iemAImpl_pminuw_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x3b. */
+FNIEMOP_DEF(iemOp_pminud_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMINUD, pminud, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pminud_u128, iemAImpl_pminud_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x3c. */
+FNIEMOP_DEF(iemOp_pmaxsb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMAXSB, pmaxsb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmaxsb_u128, iemAImpl_pmaxsb_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x3d. */
+FNIEMOP_DEF(iemOp_pmaxsd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMAXSD, pmaxsd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmaxsd_u128, iemAImpl_pmaxsd_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x3e. */
+FNIEMOP_DEF(iemOp_pmaxuw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMAXUW, pmaxuw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmaxuw_u128, iemAImpl_pmaxuw_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x3f. */
+FNIEMOP_DEF(iemOp_pmaxud_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMAXUD, pmaxud, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmaxud_u128, iemAImpl_pmaxud_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x40. */
+FNIEMOP_DEF(iemOp_pmulld_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMULLD, pmulld, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pmulld_u128, iemAImpl_pmulld_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0x41. */
+FNIEMOP_DEF(iemOp_phminposuw_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, PHMINPOSUW, phminposuw, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse41Opt_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_phminposuw_u128, iemAImpl_phminposuw_u128_fallback));
+}
+
+
+/*  Opcode 0x66 0x0f 0x38 0x42 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x43 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x44 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x45 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x46 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x47 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x48 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x49 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4a - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4b - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4c - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4e - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x4f - invalid. */
+
+/*  Opcode 0x66 0x0f 0x38 0x50 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x51 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x52 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x53 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x54 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x55 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x56 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x57 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x58 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x59 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x5a - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x5b - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x5c - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x5d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x5e - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x5f - invalid. */
+
+/*  Opcode 0x66 0x0f 0x38 0x60 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x61 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x62 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x63 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x64 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x65 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x66 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x67 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x68 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x69 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6a - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6b - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6c - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6e - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x6f - invalid. */
+
+/*  Opcode 0x66 0x0f 0x38 0x70 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x71 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x72 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x73 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x74 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x75 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x76 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x77 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x78 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x79 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x7a - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x7b - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x7c - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x7d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x7e - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x7f - invalid. */
+
+/** Opcode 0x66 0x0f 0x38 0x80. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+FNIEMOP_DEF(iemOp_invept_Gy_Mdq)
+{
+    IEMOP_MNEMONIC(invept, "invept Gy,Mdq");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_IN_VMX_OPERATION("invept", kVmxVDiag_Invept);
+    IEMOP_HLP_VMX_INSTR("invept", kVmxVDiag_Invept);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        /* Register, memory. */
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,  iEffSeg,         0);
+            IEM_MC_ARG(RTGCPTR,  GCPtrInveptDesc, 1);
+            IEM_MC_ARG(uint64_t, uInveptType,     2);
+            IEM_MC_FETCH_GREG_U64(uInveptType, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrInveptDesc, bRm, 0);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_invept, iEffSeg, GCPtrInveptDesc, uInveptType);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,  iEffSeg,         0);
+            IEM_MC_ARG(RTGCPTR,  GCPtrInveptDesc, 1);
+            IEM_MC_ARG(uint32_t, uInveptType,     2);
+            IEM_MC_FETCH_GREG_U32(uInveptType, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrInveptDesc, bRm, 0);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_invept, iEffSeg, GCPtrInveptDesc, uInveptType);
+            IEM_MC_END();
+        }
+    }
+    Log(("iemOp_invept_Gy_Mdq: invalid encoding -> #UD\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+#else
+FNIEMOP_STUB(iemOp_invept_Gy_Mdq);
+#endif
+
+/** Opcode 0x66 0x0f 0x38 0x81. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_invvpid_Gy_Mdq)
+{
+    IEMOP_MNEMONIC(invvpid, "invvpid Gy,Mdq");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEMOP_HLP_IN_VMX_OPERATION("invvpid", kVmxVDiag_Invvpid);
+    IEMOP_HLP_VMX_INSTR("invvpid", kVmxVDiag_Invvpid);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        /* Register, memory. */
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,  iEffSeg,          0);
+            IEM_MC_ARG(RTGCPTR,  GCPtrInvvpidDesc, 1);
+            IEM_MC_ARG(uint64_t, uInvvpidType,     2);
+            IEM_MC_FETCH_GREG_U64(uInvvpidType, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvvpidDesc, bRm, 0);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_invvpid, iEffSeg, GCPtrInvvpidDesc, uInvvpidType);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,  iEffSeg,          0);
+            IEM_MC_ARG(RTGCPTR,  GCPtrInvvpidDesc, 1);
+            IEM_MC_ARG(uint32_t, uInvvpidType,     2);
+            IEM_MC_FETCH_GREG_U32(uInvvpidType, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvvpidDesc, bRm, 0);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_invvpid, iEffSeg, GCPtrInvvpidDesc, uInvvpidType);
+            IEM_MC_END();
+        }
+    }
+    Log(("iemOp_invvpid_Gy_Mdq: invalid encoding -> #UD\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+#else
+FNIEMOP_STUB(iemOp_invvpid_Gy_Mdq);
+#endif
+
+/** Opcode 0x66 0x0f 0x38 0x82. */
+FNIEMOP_DEF(iemOp_invpcid_Gy_Mdq)
+{
+    IEMOP_MNEMONIC(invpcid, "invpcid Gy,Mdq");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        /* Register, memory. */
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,  iEffSeg,          0);
+            IEM_MC_ARG(RTGCPTR,  GCPtrInvpcidDesc, 1);
+            IEM_MC_ARG(uint64_t, uInvpcidType,     2);
+            IEM_MC_FETCH_GREG_U64(uInvpcidType, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvpcidDesc, bRm, 0);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_invpcid, iEffSeg, GCPtrInvpcidDesc, uInvpcidType);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,  iEffSeg,          0);
+            IEM_MC_ARG(RTGCPTR,  GCPtrInvpcidDesc, 1);
+            IEM_MC_ARG(uint32_t, uInvpcidType,     2);
+            IEM_MC_FETCH_GREG_U32(uInvpcidType, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrInvpcidDesc, bRm, 0);
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_invpcid, iEffSeg, GCPtrInvpcidDesc, uInvpcidType);
+            IEM_MC_END();
+        }
+    }
+    Log(("iemOp_invpcid_Gy_Mdq: invalid encoding -> #UD\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/*  Opcode 0x66 0x0f 0x38 0x83 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x84 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x85 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x86 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x87 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x88 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x89 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x8a - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x8b - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x8c - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x8d - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x8e - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x8f - invalid. */
+
+/*  Opcode 0x66 0x0f 0x38 0x90 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x91 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x92 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x93 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x94 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x95 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0x96 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x97 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x98 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x99 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9a - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9b - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9c - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9d - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9e - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0x9f - invalid (vex only). */
+
+/*  Opcode 0x66 0x0f 0x38 0xa0 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa1 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa2 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa3 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa5 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xa6 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xa7 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xa8 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xa9 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xaa - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xab - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xac - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xad - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xae - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xaf - invalid (vex only). */
+
+/*  Opcode 0x66 0x0f 0x38 0xb0 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb1 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb2 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb3 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb5 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xb6 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xb7 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xb8 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xb9 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xba - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xbb - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xbc - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xbd - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xbe - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xbf - invalid (vex only). */
+
+/*  Opcode      0x0f 0x38 0xc0 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc0 - invalid. */
+/*  Opcode      0x0f 0x38 0xc1 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc1 - invalid. */
+/*  Opcode      0x0f 0x38 0xc2 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc2 - invalid. */
+/*  Opcode      0x0f 0x38 0xc3 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc3 - invalid. */
+/*  Opcode      0x0f 0x38 0xc4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc4 - invalid. */
+/*  Opcode      0x0f 0x38 0xc5 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc5 - invalid. */
+/*  Opcode      0x0f 0x38 0xc6 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc6 - invalid. */
+/*  Opcode      0x0f 0x38 0xc7 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xc7 - invalid. */
+/** Opcode      0x0f 0x38 0xc8. */
+FNIEMOP_STUB(iemOp_sha1nexte_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xc8 - invalid. */
+/** Opcode      0x0f 0x38 0xc9. */
+FNIEMOP_STUB(iemOp_sha1msg1_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xc9 - invalid. */
+/** Opcode      0x0f 0x38 0xca. */
+FNIEMOP_STUB(iemOp_sha1msg2_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xca - invalid. */
+/** Opcode      0x0f 0x38 0xcb. */
+FNIEMOP_STUB(iemOp_sha256rnds2_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xcb - invalid. */
+/** Opcode      0x0f 0x38 0xcc. */
+FNIEMOP_STUB(iemOp_sha256msg1_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xcc - invalid. */
+/** Opcode      0x0f 0x38 0xcd. */
+FNIEMOP_STUB(iemOp_sha256msg2_Vdq_Wdq);
+/*  Opcode 0x66 0x0f 0x38 0xcd - invalid. */
+/*  Opcode      0x0f 0x38 0xce - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xce - invalid. */
+/*  Opcode      0x0f 0x38 0xcf - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xcf - invalid. */
+
+/*  Opcode 0x66 0x0f 0x38 0xd0 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd1 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd2 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd3 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd5 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd6 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd7 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd8 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xd9 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xda - invalid. */
+
+
+/** Opcode 0x66 0x0f 0x38 0xdb. */
+FNIEMOP_DEF(iemOp_aesimc_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, AESIMC, aesimc, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aesimc_u128, iemAImpl_aesimc_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0xdc. */
+FNIEMOP_DEF(iemOp_aesenc_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, AESENC, aesenc, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aesenc_u128, iemAImpl_aesenc_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0xdd. */
+FNIEMOP_DEF(iemOp_aesenclast_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, AESENCLAST, aesenclast, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aesenclast_u128, iemAImpl_aesenclast_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0xde. */
+FNIEMOP_DEF(iemOp_aesdec_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, AESDEC, aesdec, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aesdec_u128, iemAImpl_aesdec_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x38 0xdf. */
+FNIEMOP_DEF(iemOp_aesdeclast_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, AESDECLAST, aesdeclast, Vdq, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFull_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aesdeclast_u128, iemAImpl_aesdeclast_u128_fallback));
+}
+
+
+/*  Opcode 0x66 0x0f 0x38 0xe0 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe1 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe2 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe3 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe5 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe6 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe7 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe8 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xe9 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xea - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xeb - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xec - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xed - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xee - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xef - invalid. */
+
+
+/** Opcode      [0x66] 0x0f 0x38 0xf0. */
+FNIEMOP_DEF(iemOp_movbe_Gv_Mv)
+{
+    IEMOP_MNEMONIC2(RM, MOVBE, movbe, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovBe)
+        return iemOp_InvalidNeedRM(pVCpu);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (!IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t,  uSrc);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+                IEM_MC_BSWAP_LOCAL_U16(uSrc);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t,  uSrc);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+                IEM_MC_BSWAP_LOCAL_U32(uSrc);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t,  uSrc);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+                IEM_MC_BSWAP_LOCAL_U64(uSrc);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* Reg/reg not supported. */
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+}
+
+
+/*  Opcode 0xf3 0x0f 0x38 0xf0 - invalid. */
+
+
+/** Opcode 0xf2 0x0f 0x38 0xf0. */
+FNIEMOP_DEF(iemOp_crc32_Gd_Eb)
+{
+    IEMOP_MNEMONIC2(RM, CRC32, crc32, Gd, Eb, DISOPTYPE_HARMLESS, 0);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse42)
+        return iemOp_InvalidNeedRM(pVCpu);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint32_t *,          puDst, 0);
+        IEM_MC_ARG(uint8_t,             uSrc,  1);
+        IEM_MC_REF_GREG_U32(puDst,  IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_GREG_U8(uSrc,  IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u8, iemAImpl_crc32_u8_fallback), puDst, uSrc);
+        IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 1);
+        IEM_MC_ARG(uint32_t *,          puDst, 0);
+        IEM_MC_ARG(uint8_t,             uSrc,  1);
+        IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U8(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u8, iemAImpl_crc32_u8_fallback), puDst, uSrc);
+        IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode      [0x66] 0x0f 0x38 0xf1. */
+FNIEMOP_DEF(iemOp_movbe_Mv_Gv)
+{
+    IEMOP_MNEMONIC2(MR, MOVBE, movbe, Ev, Gv, DISOPTYPE_HARMLESS, 0);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovBe)
+        return iemOp_InvalidNeedRM(pVCpu);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (!IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Memory, register.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U16(u16Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_BSWAP_LOCAL_U16(u16Value);
+                IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u16Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_BSWAP_LOCAL_U32(u32Value);
+                IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_BSWAP_LOCAL_U64(u64Value);
+                IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* Reg/reg not supported. */
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+}
+
+
+/*  Opcode 0xf3 0x0f 0x38 0xf1 - invalid. */
+
+
+/** Opcode 0xf2 0x0f 0x38 0xf1. */
+FNIEMOP_DEF(iemOp_crc32_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, CRC32, crc32, Gd, Ev, DISOPTYPE_HARMLESS, 0);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse42)
+        return iemOp_InvalidNeedRM(pVCpu);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(2, 0);
+                IEM_MC_ARG(uint32_t *,          puDst, 0);
+                IEM_MC_ARG(uint16_t,            uSrc,  1);
+                IEM_MC_REF_GREG_U32(puDst,  IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U16(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u16, iemAImpl_crc32_u16_fallback),
+                                         puDst, uSrc);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(2, 0);
+                IEM_MC_ARG(uint32_t *,          puDst, 0);
+                IEM_MC_ARG(uint32_t,            uSrc,  1);
+                IEM_MC_REF_GREG_U32(puDst,  IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U32(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u32, iemAImpl_crc32_u32_fallback),
+                                         puDst, uSrc);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(2, 0);
+                IEM_MC_ARG(uint32_t *,          puDst, 0);
+                IEM_MC_ARG(uint64_t,            uSrc,  1);
+                IEM_MC_REF_GREG_U32(puDst,  IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u64, iemAImpl_crc32_u64_fallback),
+                                         puDst, uSrc);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(2, 1);
+                IEM_MC_ARG(uint32_t *,          puDst, 0);
+                IEM_MC_ARG(uint16_t,            uSrc,  1);
+                IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+                IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u16, iemAImpl_crc32_u16_fallback),
+                                         puDst, uSrc);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(2, 1);
+                IEM_MC_ARG(uint32_t *,          puDst, 0);
+                IEM_MC_ARG(uint32_t,            uSrc,  1);
+                IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+                IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u32, iemAImpl_crc32_u32_fallback),
+                                         puDst, uSrc);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(2, 1);
+                IEM_MC_ARG(uint32_t *,          puDst, 0);
+                IEM_MC_ARG(uint64_t,            uSrc,  1);
+                IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+                IEM_MC_REF_GREG_U32(puDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fSse42, iemAImpl_crc32_u64, iemAImpl_crc32_u64_fallback),
+                                         puDst, uSrc);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(puDst);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/*  Opcode      0x0f 0x38 0xf2 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xf2 - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xf2 - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xf2 - invalid. */
+
+/*  Opcode      0x0f 0x38 0xf3 - invalid (vex only - group 17). */
+/*  Opcode 0x66 0x0f 0x38 0xf3 - invalid (vex only - group 17). */
+/*  Opcode 0xf3 0x0f 0x38 0xf3 - invalid (vex only - group 17). */
+/*  Opcode 0xf2 0x0f 0x38 0xf3 - invalid (vex only - group 17). */
+
+/*  Opcode      0x0f 0x38 0xf4 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xf4 - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xf4 - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xf4 - invalid. */
+
+/*  Opcode      0x0f 0x38 0xf5 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xf5 - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xf5 - invalid (vex only). */
+/*  Opcode 0xf2 0x0f 0x38 0xf5 - invalid (vex only). */
+
+/*  Opcode      0x0f 0x38 0xf6 - invalid. */
+/** Opcode 0x66 0x0f 0x38 0xf6. */
+FNIEMOP_STUB(iemOp_adcx_Gy_Ey);
+/** Opcode 0xf3 0x0f 0x38 0xf6. */
+FNIEMOP_STUB(iemOp_adox_Gy_Ey);
+/*  Opcode 0xf2 0x0f 0x38 0xf6 - invalid (vex only). */
+
+/*  Opcode      0x0f 0x38 0xf7 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x38 0xf7 - invalid (vex only). */
+/*  Opcode 0xf3 0x0f 0x38 0xf7 - invalid (vex only). */
+/*  Opcode 0xf2 0x0f 0x38 0xf7 - invalid (vex only). */
+
+/*  Opcode      0x0f 0x38 0xf8 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xf8 - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xf8 - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xf8 - invalid. */
+
+/*  Opcode      0x0f 0x38 0xf9 - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xf9 - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xf9 - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xf9 - invalid. */
+
+/*  Opcode      0x0f 0x38 0xfa - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xfa - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xfa - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xfa - invalid. */
+
+/*  Opcode      0x0f 0x38 0xfb - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xfb - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xfb - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xfb - invalid. */
+
+/*  Opcode      0x0f 0x38 0xfc - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xfc - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xfc - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xfc - invalid. */
+
+/*  Opcode      0x0f 0x38 0xfd - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xfd - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xfd - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xfd - invalid. */
+
+/*  Opcode      0x0f 0x38 0xfe - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xfe - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xfe - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xfe - invalid. */
+
+/*  Opcode      0x0f 0x38 0xff - invalid. */
+/*  Opcode 0x66 0x0f 0x38 0xff - invalid. */
+/*  Opcode 0xf3 0x0f 0x38 0xff - invalid. */
+/*  Opcode 0xf2 0x0f 0x38 0xff - invalid. */
+
+
+/**
+ * Three byte opcode map, first two bytes are 0x0f 0x38.
+ * @sa      g_apfnVexMap2
+ */
+IEM_STATIC const PFNIEMOP g_apfnThreeByte0f38[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  iemOp_pshufb_Pq_Qq,         iemOp_pshufb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x01 */  iemOp_phaddw_Pq_Qq,         iemOp_phaddw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x02 */  iemOp_phaddd_Pq_Qq,         iemOp_phaddd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x03 */  iemOp_phaddsw_Pq_Qq,        iemOp_phaddsw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x04 */  iemOp_pmaddubsw_Pq_Qq,      iemOp_pmaddubsw_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x05 */  iemOp_phsubw_Pq_Qq,         iemOp_phsubw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x06 */  iemOp_phsubd_Pq_Qq,         iemOp_phsubd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x07 */  iemOp_phsubsw_Pq_Qq,        iemOp_phsubsw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x08 */  iemOp_psignb_Pq_Qq,         iemOp_psignb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x09 */  iemOp_psignw_Pq_Qq,         iemOp_psignw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0a */  iemOp_psignd_Pq_Qq,         iemOp_psignd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0b */  iemOp_pmulhrsw_Pq_Qq,       iemOp_pmulhrsw_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x10 */  iemOp_InvalidNeedRM,        iemOp_pblendvb_Vdq_Wdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x11 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x12 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x13 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x14 */  iemOp_InvalidNeedRM,        iemOp_blendvps_Vdq_Wdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x15 */  iemOp_InvalidNeedRM,        iemOp_blendvpd_Vdq_Wdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x16 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x17 */  iemOp_InvalidNeedRM,        iemOp_ptest_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x18 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x19 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1c */  iemOp_pabsb_Pq_Qq,          iemOp_pabsb_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1d */  iemOp_pabsw_Pq_Qq,          iemOp_pabsw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1e */  iemOp_pabsd_Pq_Qq,          iemOp_pabsd_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x20 */  iemOp_InvalidNeedRM,        iemOp_pmovsxbw_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x21 */  iemOp_InvalidNeedRM,        iemOp_pmovsxbd_Vx_UxMd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x22 */  iemOp_InvalidNeedRM,        iemOp_pmovsxbq_Vx_UxMw,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x23 */  iemOp_InvalidNeedRM,        iemOp_pmovsxwd_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x24 */  iemOp_InvalidNeedRM,        iemOp_pmovsxwq_Vx_UxMd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x25 */  iemOp_InvalidNeedRM,        iemOp_pmovsxdq_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x26 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x27 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x28 */  iemOp_InvalidNeedRM,        iemOp_pmuldq_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x29 */  iemOp_InvalidNeedRM,        iemOp_pcmpeqq_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2a */  iemOp_InvalidNeedRM,        iemOp_movntdqa_Vdq_Mdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2b */  iemOp_InvalidNeedRM,        iemOp_packusdw_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x2d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x2e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x2f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x30 */  iemOp_InvalidNeedRM,        iemOp_pmovzxbw_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x31 */  iemOp_InvalidNeedRM,        iemOp_pmovzxbd_Vx_UxMd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x32 */  iemOp_InvalidNeedRM,        iemOp_pmovzxbq_Vx_UxMw,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x33 */  iemOp_InvalidNeedRM,        iemOp_pmovzxwd_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x34 */  iemOp_InvalidNeedRM,        iemOp_pmovzxwq_Vx_UxMd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x35 */  iemOp_InvalidNeedRM,        iemOp_pmovzxdq_Vx_UxMq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x36 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x37 */  iemOp_InvalidNeedRM,        iemOp_pcmpgtq_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x38 */  iemOp_InvalidNeedRM,        iemOp_pminsb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x39 */  iemOp_InvalidNeedRM,        iemOp_pminsd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3a */  iemOp_InvalidNeedRM,        iemOp_pminuw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3b */  iemOp_InvalidNeedRM,        iemOp_pminud_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3c */  iemOp_InvalidNeedRM,        iemOp_pmaxsb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3d */  iemOp_InvalidNeedRM,        iemOp_pmaxsd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3e */  iemOp_InvalidNeedRM,        iemOp_pmaxuw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3f */  iemOp_InvalidNeedRM,        iemOp_pmaxud_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x40 */  iemOp_InvalidNeedRM,        iemOp_pmulld_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x41 */  iemOp_InvalidNeedRM,        iemOp_phminposuw_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x42 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x43 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x44 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x45 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x46 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x47 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x48 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x49 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x50 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x51 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x52 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x53 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x54 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x55 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x56 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x57 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x58 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x59 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x60 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x61 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x62 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x63 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x64 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x65 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x66 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x67 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x68 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x69 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x70 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x71 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x72 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x73 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x74 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x75 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x76 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x77 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x78 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x79 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x80 */  iemOp_InvalidNeedRM,        iemOp_invept_Gy_Mdq,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x81 */  iemOp_InvalidNeedRM,        iemOp_invvpid_Gy_Mdq,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x82 */  iemOp_InvalidNeedRM,        iemOp_invpcid_Gy_Mdq,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x83 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x84 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x85 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x86 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x87 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x88 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x89 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x90 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x91 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x92 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x93 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x94 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x95 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x96 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x97 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x98 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x99 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xa0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xaa */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xab */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xac */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xad */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xae */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xaf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xb0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xba */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbc */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbd */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbe */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xc0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc8 */  iemOp_sha1nexte_Vdq_Wdq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xc9 */  iemOp_sha1msg1_Vdq_Wdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xca */  iemOp_sha1msg2_Vdq_Wdq,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcb */  iemOp_sha256rnds2_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcc */  iemOp_sha256msg1_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcd */  iemOp_sha256msg2_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xce */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xd0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xda */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xdb */  iemOp_InvalidNeedRM,        iemOp_aesimc_Vdq_Wdq,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdc */  iemOp_InvalidNeedRM,        iemOp_aesenc_Vdq_Wdq,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdd */  iemOp_InvalidNeedRM,        iemOp_aesenclast_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xde */  iemOp_InvalidNeedRM,        iemOp_aesdec_Vdq_Wdq,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdf */  iemOp_InvalidNeedRM,        iemOp_aesdeclast_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xe0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xea */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xeb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xec */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xed */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xee */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xef */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xf0 */  iemOp_movbe_Gv_Mv,          iemOp_movbe_Gv_Mv,          iemOp_InvalidNeedRM,        iemOp_crc32_Gd_Eb,
+    /* 0xf1 */  iemOp_movbe_Mv_Gv,          iemOp_movbe_Mv_Gv,          iemOp_InvalidNeedRM,        iemOp_crc32_Gv_Ev,
+    /* 0xf2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf6 */  iemOp_InvalidNeedRM,        iemOp_adcx_Gy_Ey,           iemOp_adox_Gy_Ey,           iemOp_InvalidNeedRM,
+    /* 0xf7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfa */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfc */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfd */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfe */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xff */  IEMOP_X4(iemOp_InvalidNeedRM),
+};
+AssertCompile(RT_ELEMENTS(g_apfnThreeByte0f38) == 1024);
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h
new file mode 100644
index 00000000..9809bbc8
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsThree0f3a.cpp.h
@@ -0,0 +1,1549 @@
+/* $Id: IEMAllInstructionsThree0f3a.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation, 0x0f 0x3a map.
+ *
+ * @remarks IEMAllInstructionsVexMap3.cpp.h is a VEX mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/** @name Three byte opcodes with first two bytes 0x0f 0x3a
+ * @{
+ */
+
+/**
+ * Common worker for SSSE3 instructions on the forms:
+ *      pxxx    xmm1, xmm2/mem128, imm8
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. SSSE3 cpuid checks.
+ *
+ * @sa  iemOpCommonSse41_FullFullImm8_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSsse3_FullFullImm8_To_Full, PFNIEMAIMPLMEDIAOPTF2U128IMM8, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,               0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc,               1);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEM_MC_MAYBE_RAISE_SSSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,       IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,               0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc,         1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSSE3_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE 4.1 instructions on the forms:
+ *      pxxx    xmm1, xmm2/mem128, imm8
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * No SIMD exceptions. SSE 4.1 cpuid checks.
+ *
+ * @sa  iemOpCommonSsse3_FullFullImm8_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse41_FullFullImm8_To_Full, PFNIEMAIMPLMEDIAOPTF2U128IMM8, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM, imm8
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,               0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc,               1);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,       IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128], imm8.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,               0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc,         1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE 4.1 instructions of the form:
+ *      xxx     xmm1, xmm2/mem128, imm8
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * MXCSR is used as input and output.
+ * Exceptions type 4. SSE 4.1 cpuid checks.
+ *
+ * @sa  iemOpCommonSse41_FullFullImm8_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse41Fp_FullFullImm8_To_Full, FNIEMAIMPLMXCSRF2XMMIMM8, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM, imm8.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 2);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(pfnU128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128], imm8.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_LOCAL(RTGCPTR,                       GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(Src.uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(pfnU128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE-style AES-NI instructions of the form:
+ *      aesxxx  xmm1, xmm2/mem128, imm8
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. AES-NI cpuid checks.
+ *
+ * @sa  iemOpCommonSsse3_FullFullImm8_To_Full
+ * @sa  iemOpCommonSse41_FullFullImm8_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonAesNi_FullFullImm8_To_Full, PFNIEMAIMPLMEDIAOPTF2U128IMM8, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,               0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc,               1);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEM_MC_MAYBE_RAISE_AESNI_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,       IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,               0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc,         1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_AESNI_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x00 - invalid (vex only). */
+/** Opcode 0x66 0x0f 0x01 - invalid (vex only). */
+/** Opcode 0x66 0x0f 0x02 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x03 - invalid */
+/** Opcode 0x66 0x0f 0x04 - invalid (vex only). */
+/** Opcode 0x66 0x0f 0x05 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x06 - invalid (vex only) */
+/*  Opcode 0x66 0x0f 0x07 - invalid */
+/** Opcode 0x66 0x0f 0x08. */
+FNIEMOP_DEF(iemOp_roundps_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, ROUNDPS, roundps, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse41Fp_FullFullImm8_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_roundps_u128, iemAImpl_roundps_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x09. */
+FNIEMOP_DEF(iemOp_roundpd_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, ROUNDPD, roundpd, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse41Fp_FullFullImm8_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_roundpd_u128, iemAImpl_roundpd_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x0a. */
+FNIEMOP_DEF(iemOp_roundss_Vss_Wss_Ib)
+{
+    /* The instruction form is very similar to CMPSS. */
+    IEMOP_MNEMONIC3(RMI, ROUNDSS, roundss, Vss, Wss, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM32, XMM32.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 2);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_roundss_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_STORE_XREG_XMM_U32(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/, Dst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM32, [mem32].
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_LOCAL(RTGCPTR,                       GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U32(Src.uSrc2, 0 /*a_iDword */, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_roundss_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_STORE_XREG_XMM_U32(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/, Dst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/** Opcode 0x66 0x0f 0x0b. */
+FNIEMOP_DEF(iemOp_roundsd_Vsd_Wsd_Ib)
+{
+    /* The instruction form is very similar to CMPSD. */
+    IEMOP_MNEMONIC3(RMI, ROUNDSD, roundsd, Vsd, Wsd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM64, XMM64, imm8.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 2);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_roundsd_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_STORE_XREG_XMM_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iQword*/, Dst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM64, [mem64], imm8.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_LOCAL(RTGCPTR,                       GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U64(Src.uSrc2, 0 /*a_iQword */, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_roundsd_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_STORE_XREG_XMM_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iQword*/, Dst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x0c. */
+FNIEMOP_DEF(iemOp_blendps_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, BLENDPS, blendps, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFullImm8_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_blendps_u128, iemAImpl_blendps_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x0d. */
+FNIEMOP_DEF(iemOp_blendpd_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, BLENDPD, blendpd, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFullImm8_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_blendpd_u128, iemAImpl_blendpd_u128_fallback));
+}
+
+
+/** Opcode 0x66 0x0f 0x0e. */
+FNIEMOP_DEF(iemOp_pblendw_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PBLENDW, pblendw, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse41_FullFullImm8_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSse41, iemAImpl_pblendw_u128, iemAImpl_pblendw_u128_fallback));
+}
+
+
+/** Opcode      0x0f 0x0f. */
+FNIEMOP_DEF(iemOp_palignr_Pq_Qq_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PALIGNR, palignr, Pq, Qq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t,            uSrc, 1);
+        IEM_MC_ARG_CONST(uint8_t,       bImmArg, /*=*/ bImm, 2);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_EX(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_FETCH_MREG_U64(uSrc, IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_palignr_u64, iemAImpl_palignr_u64_fallback),
+                                 pDst, uSrc, bImmArg);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint64_t *,                  pDst,                0);
+        IEM_MC_ARG(uint64_t,                    uSrc,                1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_EX(IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSsse3);
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_palignr_u64, iemAImpl_palignr_u64_fallback),
+                                 pDst, uSrc, bImmArg);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x0f. */
+FNIEMOP_DEF(iemOp_palignr_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PALIGNR, palignr, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSsse3_FullFullImm8_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fSsse3, iemAImpl_palignr_u128, iemAImpl_palignr_u128_fallback));
+}
+
+
+/*  Opcode 0x66 0x0f 0x10 - invalid */
+/*  Opcode 0x66 0x0f 0x11 - invalid */
+/*  Opcode 0x66 0x0f 0x12 - invalid */
+/*  Opcode 0x66 0x0f 0x13 - invalid */
+
+
+/** Opcode 0x66 0x0f 0x14. */
+FNIEMOP_DEF(iemOp_pextrb_RdMb_Vdq_Ib)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_MNEMONIC3(MRI, PEXTRB, pextrb, Ev, Vq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * greg32, XMM.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint8_t,   uValue);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_AND_LOCAL_U8(bImm, 15);
+        IEM_MC_FETCH_XREG_U8(uValue, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iByte*/);
+        IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), uValue);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * [mem8], XMM.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint8_t,   uValue);
+        IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_AND_LOCAL_U8(bImm, 15);
+        IEM_MC_FETCH_XREG_U8(uValue, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iByte*/);
+        IEM_MC_STORE_MEM_U8(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uValue);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x15. */
+FNIEMOP_DEF(iemOp_pextrw_RdMw_Vdq_Ib)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_MNEMONIC3(MRI, PEXTRW, pextrw, Ev, Vq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * greg32, XMM.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint16_t,  uValue);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_AND_LOCAL_U8(bImm, 7);
+        IEM_MC_FETCH_XREG_U16(uValue, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iWord*/);
+        IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), uValue);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * [mem16], XMM.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint16_t,  uValue);
+        IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_AND_LOCAL_U8(bImm, 7);
+        IEM_MC_FETCH_XREG_U16(uValue, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iWord*/);
+        IEM_MC_STORE_MEM_U16(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uValue);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+FNIEMOP_DEF(iemOp_pextrd_q_RdMw_Vdq_Ib)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x16
+         * @opcodesub   rex.w=1
+         * @oppfx       0x66
+         * @opcpuid     sse
+         */
+        IEMOP_MNEMONIC3(MRI, PEXTRQ, pextrq, Ev, Vq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /*
+             * greg64, XMM.
+             */
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t,  uSrc);
+            IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE();
+            IEM_MC_AND_LOCAL_U8(bImm, 1);
+            IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iQword*/);
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), uSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /*
+             * [mem64], XMM.
+             */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t,  uSrc);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE();
+
+            IEM_MC_AND_LOCAL_U8(bImm, 1);
+            IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iQword*/);
+            IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x16
+         * @opcodesub   rex.w=0
+         * @oppfx       0x66
+         * @opcpuid     sse
+         */
+        IEMOP_MNEMONIC3(MRI, PEXTRD, pextrd, Ey, Vd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /*
+             * greg32, XMM.
+             */
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t,  uSrc);
+            IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE();
+            IEM_MC_AND_LOCAL_U8(bImm, 3);
+            IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/);
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), uSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /*
+             * [mem32], XMM.
+             */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint32_t,  uSrc);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE();
+            IEM_MC_AND_LOCAL_U8(bImm, 3);
+            IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/);
+            IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x17. */
+FNIEMOP_DEF(iemOp_extractps_Ed_Vdq_Ib)
+{
+    IEMOP_MNEMONIC3(MRI, EXTRACTPS, extractps, Ed, Vdq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * greg32, XMM.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint32_t,  uSrc);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_AND_LOCAL_U8(bImm, 3);
+        IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/);
+        IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), uSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * [mem32], XMM.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint32_t,  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_AND_LOCAL_U8(bImm, 3);
+        IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/);
+        IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode 0x66 0x0f 0x18 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x19 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x1a - invalid */
+/*  Opcode 0x66 0x0f 0x1b - invalid */
+/*  Opcode 0x66 0x0f 0x1c - invalid */
+/*  Opcode 0x66 0x0f 0x1d - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x1e - invalid */
+/*  Opcode 0x66 0x0f 0x1f - invalid */
+
+
+/** Opcode 0x66 0x0f 0x20. */
+FNIEMOP_DEF(iemOp_pinsrb_Vdq_RyMb_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PINSRB, pinsrb, Vd, Ey, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, greg32.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint8_t,   uSrc);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_FETCH_GREG_U8(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_AND_LOCAL_U8(bImm, 15);
+        IEM_MC_STORE_XREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iByte*/, uSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem8].
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint8_t,   uSrc);
+        IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_MEM_U8(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_AND_LOCAL_U8(bImm, 15);
+        IEM_MC_STORE_XREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iByte*/, uSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/** Opcode 0x66 0x0f 0x21, */
+FNIEMOP_DEF(iemOp_insertps_Vdq_UdqMd_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, INSERTPS, insertps, Vdq, Wdq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);  /// @todo
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 3);
+        IEM_MC_LOCAL(uint32_t,  uSrc);
+        IEM_MC_LOCAL(uint8_t,   uSrcSel);
+        IEM_MC_LOCAL(uint8_t,   uDstSel);
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_ASSIGN(uSrcSel, bImm);
+        IEM_MC_SHR_LOCAL_U8(uSrcSel, 6);
+        IEM_MC_AND_LOCAL_U8(uSrcSel, 3);
+        IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), uSrcSel);
+        IEM_MC_ASSIGN(uDstSel, bImm);
+        IEM_MC_SHR_LOCAL_U8(uDstSel, 4);
+        IEM_MC_AND_LOCAL_U8(uDstSel, 3);
+        IEM_MC_CLEAR_XREG_U32_MASK(IEM_GET_MODRM_REG(pVCpu, bRm), bImm);
+        IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), uDstSel, uSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem32].
+         */
+        IEM_MC_BEGIN(0, 3);
+        IEM_MC_LOCAL(uint32_t,  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+        IEM_MC_LOCAL(uint8_t,   uDstSel);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_ASSIGN(uDstSel, bImm);
+        IEM_MC_SHR_LOCAL_U8(uDstSel, 4);
+        IEM_MC_AND_LOCAL_U8(uDstSel, 3);
+        IEM_MC_CLEAR_XREG_U32_MASK(IEM_GET_MODRM_REG(pVCpu, bRm), bImm);
+        IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), uDstSel, uSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+FNIEMOP_DEF(iemOp_pinsrd_q_Vdq_Ey_Ib)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x22
+         * @opcodesub   rex.w=1
+         * @oppfx       0x66
+         * @opcpuid     sse
+         */
+        IEMOP_MNEMONIC3(RMI, PINSRQ, pinsrq, Vq, Ey, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /*
+             * XMM, greg64.
+             */
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t,  uSrc);
+            IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE();
+            IEM_MC_FETCH_GREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_AND_LOCAL_U8(bImm, 1);
+            IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iQword*/, uSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /*
+             * XMM, [mem64].
+             */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t,  uSrc);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE();
+
+            IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_AND_LOCAL_U8(bImm, 1);
+            IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iQword*/, uSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x22
+         * @opcodesub   rex.w=0
+         * @oppfx       0x66
+         * @opcpuid     sse
+         */
+        IEMOP_MNEMONIC3(RMI, PINSRD, pinsrd, Vd, Ey, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /*
+             * XMM, greg32.
+             */
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t,  uSrc);
+            IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE();
+            IEM_MC_FETCH_GREG_U32(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_AND_LOCAL_U8(bImm, 3);
+            IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/, uSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /*
+             * XMM, [mem32].
+             */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint32_t,  uSrc);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE41_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE();
+
+            IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_AND_LOCAL_U8(bImm, 3);
+            IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), bImm /*a_iDword*/, uSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/*  Opcode 0x66 0x0f 0x23 - invalid */
+/*  Opcode 0x66 0x0f 0x24 - invalid */
+/*  Opcode 0x66 0x0f 0x25 - invalid */
+/*  Opcode 0x66 0x0f 0x26 - invalid */
+/*  Opcode 0x66 0x0f 0x27 - invalid */
+/*  Opcode 0x66 0x0f 0x28 - invalid */
+/*  Opcode 0x66 0x0f 0x29 - invalid */
+/*  Opcode 0x66 0x0f 0x2a - invalid */
+/*  Opcode 0x66 0x0f 0x2b - invalid */
+/*  Opcode 0x66 0x0f 0x2c - invalid */
+/*  Opcode 0x66 0x0f 0x2d - invalid */
+/*  Opcode 0x66 0x0f 0x2e - invalid */
+/*  Opcode 0x66 0x0f 0x2f - invalid */
+
+
+/*  Opcode 0x66 0x0f 0x30 - invalid */
+/*  Opcode 0x66 0x0f 0x31 - invalid */
+/*  Opcode 0x66 0x0f 0x32 - invalid */
+/*  Opcode 0x66 0x0f 0x33 - invalid */
+/*  Opcode 0x66 0x0f 0x34 - invalid */
+/*  Opcode 0x66 0x0f 0x35 - invalid */
+/*  Opcode 0x66 0x0f 0x36 - invalid */
+/*  Opcode 0x66 0x0f 0x37 - invalid */
+/*  Opcode 0x66 0x0f 0x38 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x39 - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x3a - invalid */
+/*  Opcode 0x66 0x0f 0x3b - invalid */
+/*  Opcode 0x66 0x0f 0x3c - invalid */
+/*  Opcode 0x66 0x0f 0x3d - invalid */
+/*  Opcode 0x66 0x0f 0x3e - invalid */
+/*  Opcode 0x66 0x0f 0x3f - invalid */
+
+
+/** Opcode 0x66 0x0f 0x40. */
+FNIEMOP_STUB(iemOp_dpps_Vx_Wx_Ib);
+/** Opcode 0x66 0x0f 0x41, */
+FNIEMOP_STUB(iemOp_dppd_Vdq_Wdq_Ib);
+/** Opcode 0x66 0x0f 0x42. */
+FNIEMOP_STUB(iemOp_mpsadbw_Vx_Wx_Ib);
+/*  Opcode 0x66 0x0f 0x43 - invalid */
+
+
+/** Opcode 0x66 0x0f 0x44. */
+FNIEMOP_DEF(iemOp_pclmulqdq_Vdq_Wdq_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PCLMULQDQ, pclmulqdq, Vdq, Wdq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,               0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc,               1);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEM_MC_MAYBE_RAISE_PCLMUL_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,       IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fPclMul,
+                                                             iemAImpl_pclmulqdq_u128,
+                                                             iemAImpl_pclmulqdq_u128_fallback),
+                                 puDst, puSrc, bImmArg);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,               0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc,         1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_PCLMUL_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,             IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fPclMul,
+                                                             iemAImpl_pclmulqdq_u128,
+                                                             iemAImpl_pclmulqdq_u128_fallback),
+                                 puDst, puSrc, bImmArg);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode 0x66 0x0f 0x45 - invalid */
+/*  Opcode 0x66 0x0f 0x46 - invalid (vex only)  */
+/*  Opcode 0x66 0x0f 0x47 - invalid */
+/*  Opcode 0x66 0x0f 0x48 - invalid */
+/*  Opcode 0x66 0x0f 0x49 - invalid */
+/*  Opcode 0x66 0x0f 0x4a - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x4b - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x4c - invalid (vex only). */
+/*  Opcode 0x66 0x0f 0x4d - invalid */
+/*  Opcode 0x66 0x0f 0x4e - invalid */
+/*  Opcode 0x66 0x0f 0x4f - invalid */
+
+
+/*  Opcode 0x66 0x0f 0x50 - invalid */
+/*  Opcode 0x66 0x0f 0x51 - invalid */
+/*  Opcode 0x66 0x0f 0x52 - invalid */
+/*  Opcode 0x66 0x0f 0x53 - invalid */
+/*  Opcode 0x66 0x0f 0x54 - invalid */
+/*  Opcode 0x66 0x0f 0x55 - invalid */
+/*  Opcode 0x66 0x0f 0x56 - invalid */
+/*  Opcode 0x66 0x0f 0x57 - invalid */
+/*  Opcode 0x66 0x0f 0x58 - invalid */
+/*  Opcode 0x66 0x0f 0x59 - invalid */
+/*  Opcode 0x66 0x0f 0x5a - invalid */
+/*  Opcode 0x66 0x0f 0x5b - invalid */
+/*  Opcode 0x66 0x0f 0x5c - invalid */
+/*  Opcode 0x66 0x0f 0x5d - invalid */
+/*  Opcode 0x66 0x0f 0x5e - invalid */
+/*  Opcode 0x66 0x0f 0x5f - invalid */
+
+
+/** Opcode 0x66 0x0f 0x60. */
+FNIEMOP_STUB(iemOp_pcmpestrm_Vdq_Wdq_Ib);
+/** Opcode 0x66 0x0f 0x61, */
+FNIEMOP_STUB(iemOp_pcmpestri_Vdq_Wdq_Ib);
+/** Opcode 0x66 0x0f 0x62. */
+FNIEMOP_STUB(iemOp_pcmpistrm_Vdq_Wdq_Ib);
+
+
+/** Opcode 0x66 0x0f 0x63*/
+FNIEMOP_DEF(iemOp_pcmpistri_Vdq_Wdq_Ib)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 1);
+        IEM_MC_ARG(uint32_t *,                 pu32Ecx,             0);
+        IEM_MC_ARG(uint32_t *,                 pEFlags,             1);
+        IEM_MC_LOCAL(IEMPCMPISTRISRC,          Src);
+        IEM_MC_ARG_LOCAL_REF(PIEMPCMPISTRISRC, pSrc,           Src, 2);
+        IEM_MC_ARG_CONST(uint8_t,              bImmArg, /*=*/ bImm, 3);
+        IEM_MC_MAYBE_RAISE_SSE42_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_GREG_U32(pu32Ecx, X86_GREG_xCX);
+        IEM_MC_FETCH_XREG_U128(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_XREG_U128(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fSse42,
+                                                             iemAImpl_pcmpistri_u128,
+                                                             iemAImpl_pcmpistri_u128_fallback),
+                                 pu32Ecx, pEFlags, pSrc, bImmArg);
+        /** @todo testcase: High dword of RCX cleared? */
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_ARG(uint32_t *,                  pu32Ecx,             0);
+        IEM_MC_ARG(uint32_t *,                  pEFlags,             1);
+        IEM_MC_LOCAL(IEMPCMPISTRISRC,           Src);
+        IEM_MC_ARG_LOCAL_REF(PIEMPCMPISTRISRC,  pSrc,           Src, 2);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 3);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE42_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128(Src.uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_GREG_U32(pu32Ecx, X86_GREG_xCX);
+        IEM_MC_FETCH_XREG_U128(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fSse42,
+                                                             iemAImpl_pcmpistri_u128,
+                                                             iemAImpl_pcmpistri_u128_fallback),
+                                 pu32Ecx, pEFlags, pSrc, bImmArg);
+        /** @todo testcase: High dword of RCX cleared? */
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode 0x66 0x0f 0x64 - invalid */
+/*  Opcode 0x66 0x0f 0x65 - invalid */
+/*  Opcode 0x66 0x0f 0x66 - invalid */
+/*  Opcode 0x66 0x0f 0x67 - invalid */
+/*  Opcode 0x66 0x0f 0x68 - invalid */
+/*  Opcode 0x66 0x0f 0x69 - invalid */
+/*  Opcode 0x66 0x0f 0x6a - invalid */
+/*  Opcode 0x66 0x0f 0x6b - invalid */
+/*  Opcode 0x66 0x0f 0x6c - invalid */
+/*  Opcode 0x66 0x0f 0x6d - invalid */
+/*  Opcode 0x66 0x0f 0x6e - invalid */
+/*  Opcode 0x66 0x0f 0x6f - invalid */
+
+/*  Opcodes 0x0f 0x70 thru 0x0f 0xb0 are unused.  */
+
+
+/*  Opcode      0x0f 0xc0 - invalid */
+/*  Opcode      0x0f 0xc1 - invalid */
+/*  Opcode      0x0f 0xc2 - invalid */
+/*  Opcode      0x0f 0xc3 - invalid */
+/*  Opcode      0x0f 0xc4 - invalid */
+/*  Opcode      0x0f 0xc5 - invalid */
+/*  Opcode      0x0f 0xc6 - invalid */
+/*  Opcode      0x0f 0xc7 - invalid */
+/*  Opcode      0x0f 0xc8 - invalid */
+/*  Opcode      0x0f 0xc9 - invalid */
+/*  Opcode      0x0f 0xca - invalid */
+/*  Opcode      0x0f 0xcb - invalid */
+/*  Opcode      0x0f 0xcc */
+FNIEMOP_STUB(iemOp_sha1rnds4_Vdq_Wdq_Ib);
+/*  Opcode      0x0f 0xcd - invalid */
+/*  Opcode      0x0f 0xce - invalid */
+/*  Opcode      0x0f 0xcf - invalid */
+
+
+/*  Opcode 0x66 0x0f 0xd0 - invalid */
+/*  Opcode 0x66 0x0f 0xd1 - invalid */
+/*  Opcode 0x66 0x0f 0xd2 - invalid */
+/*  Opcode 0x66 0x0f 0xd3 - invalid */
+/*  Opcode 0x66 0x0f 0xd4 - invalid */
+/*  Opcode 0x66 0x0f 0xd5 - invalid */
+/*  Opcode 0x66 0x0f 0xd6 - invalid */
+/*  Opcode 0x66 0x0f 0xd7 - invalid */
+/*  Opcode 0x66 0x0f 0xd8 - invalid */
+/*  Opcode 0x66 0x0f 0xd9 - invalid */
+/*  Opcode 0x66 0x0f 0xda - invalid */
+/*  Opcode 0x66 0x0f 0xdb - invalid */
+/*  Opcode 0x66 0x0f 0xdc - invalid */
+/*  Opcode 0x66 0x0f 0xdd - invalid */
+/*  Opcode 0x66 0x0f 0xde - invalid */
+
+
+/*  Opcode 0x66 0x0f 0xdf - (aeskeygenassist). */
+FNIEMOP_DEF(iemOp_aeskeygen_Vdq_Wdq_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, AESKEYGEN, aeskeygen, Vdq, Wdq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAesNi_FullFullImm8_To_Full,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAesNi, iemAImpl_aeskeygenassist_u128, iemAImpl_aeskeygenassist_u128_fallback));
+}
+
+
+/*  Opcode 0xf2 0x0f 0xf0 - invalid (vex only) */
+
+
+/**
+ * Three byte opcode map, first two bytes are 0x0f 0x3a.
+ * @sa      g_apfnVexMap2
+ */
+IEM_STATIC const PFNIEMOP g_apfnThreeByte0f3a[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x01 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x02 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x03 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x04 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x05 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x06 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x07 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x08 */  iemOp_InvalidNeedRMImm8,    iemOp_roundps_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x09 */  iemOp_InvalidNeedRMImm8,    iemOp_roundpd_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0a */  iemOp_InvalidNeedRMImm8,    iemOp_roundss_Vss_Wss_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0b */  iemOp_InvalidNeedRMImm8,    iemOp_roundsd_Vsd_Wsd_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0c */  iemOp_InvalidNeedRMImm8,    iemOp_blendps_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0d */  iemOp_InvalidNeedRMImm8,    iemOp_blendpd_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0e */  iemOp_InvalidNeedRMImm8,    iemOp_pblendw_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0f */  iemOp_palignr_Pq_Qq_Ib,     iemOp_palignr_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+
+    /* 0x10 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x11 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x12 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x13 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x14 */  iemOp_InvalidNeedRMImm8,    iemOp_pextrb_RdMb_Vdq_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x15 */  iemOp_InvalidNeedRMImm8,    iemOp_pextrw_RdMw_Vdq_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x16 */  iemOp_InvalidNeedRMImm8,    iemOp_pextrd_q_RdMw_Vdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x17 */  iemOp_InvalidNeedRMImm8,    iemOp_extractps_Ed_Vdq_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x18 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x19 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x20 */  iemOp_InvalidNeedRMImm8,    iemOp_pinsrb_Vdq_RyMb_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x21 */  iemOp_InvalidNeedRMImm8,    iemOp_insertps_Vdq_UdqMd_Ib,iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x22 */  iemOp_InvalidNeedRMImm8,    iemOp_pinsrd_q_Vdq_Ey_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x23 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x24 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x25 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x26 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x27 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x28 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x29 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x30 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x31 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x32 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x33 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x34 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x35 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x36 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x37 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x38 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x39 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x40 */  iemOp_InvalidNeedRMImm8,    iemOp_dpps_Vx_Wx_Ib,        iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x41 */  iemOp_InvalidNeedRMImm8,    iemOp_dppd_Vdq_Wdq_Ib,      iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x42 */  iemOp_InvalidNeedRMImm8,    iemOp_mpsadbw_Vx_Wx_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x43 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x44 */  iemOp_InvalidNeedRMImm8,    iemOp_pclmulqdq_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x45 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x46 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x47 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x48 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x49 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x50 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x51 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x52 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x53 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x54 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x55 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x56 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x57 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x58 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x59 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x60 */  iemOp_InvalidNeedRMImm8,    iemOp_pcmpestrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x61 */  iemOp_InvalidNeedRMImm8,    iemOp_pcmpestri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x62 */  iemOp_InvalidNeedRMImm8,    iemOp_pcmpistrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x63 */  iemOp_InvalidNeedRMImm8,    iemOp_pcmpistri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x64 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x65 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x66 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x67 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x68 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x69 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x6f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x70 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x71 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x72 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x73 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x74 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x75 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x76 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x77 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x78 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x79 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x7f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x80 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x81 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x82 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x83 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x84 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x85 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x86 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x87 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x88 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x89 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x90 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x91 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x92 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x93 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x94 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x95 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x96 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x97 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x98 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x99 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xa0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xaa */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xab */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xac */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xad */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xae */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xaf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xb0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xba */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbe */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xc0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xca */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcc */  iemOp_sha1rnds4_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xcd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xce */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xd0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xda */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xde */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdf */  iemOp_InvalidNeedRMImm8, iemOp_aeskeygen_Vdq_Wdq_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+
+    /* 0xe0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xea */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xeb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xec */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xed */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xee */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xef */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xf0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfa */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfe */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xff */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+};
+AssertCompile(RT_ELEMENTS(g_apfnThreeByte0f3a) == 1024);
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h
new file mode 100644
index 00000000..25f2cc9f
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsTwoByte0f.cpp.h
@@ -0,0 +1,13975 @@
+/* $Id: IEMAllInstructionsTwoByte0f.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ *
+ * @remarks IEMAllInstructionsVexMap1.cpp.h is a VEX mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/** @name Two byte opcodes (first byte 0x0f).
+ *
+ * @{
+ */
+
+
+/**
+ * Common worker for MMX instructions on the form:
+ *      pxxx    mm1, mm2/mem64
+ */
+FNIEMOP_DEF_1(iemOpCommonMmx_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U64, pfnU64)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * MMX, MMX.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,    pSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * MMX, [mem64].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for MMX instructions on the form:
+ *      pxxx    mm1, mm2/mem64
+ *
+ * Unlike iemOpCommonMmx_FullFull_To_Full, the @a pfnU64 worker function takes
+ * no FXSAVE state, just the operands.
+ */
+FNIEMOP_DEF_1(iemOpCommonMmxOpt_FullFull_To_Full, PFNIEMAIMPLMEDIAOPTF2U64, pfnU64)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * MMX, MMX.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,    pSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU64, pDst, pSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * MMX, [mem64].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU64, pDst, pSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for MMX instructions on the form:
+ *      pxxx    mm1, mm2/mem64
+ * for instructions introduced with SSE.
+ */
+FNIEMOP_DEF_1(iemOpCommonMmxSse_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U64, pfnU64)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * MMX, MMX.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,    pSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * MMX, [mem64].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for MMX instructions on the form:
+ *      pxxx    mm1, mm2/mem64
+ * for instructions introduced with SSE.
+ *
+ * Unlike iemOpCommonMmxSse_FullFull_To_Full, the @a pfnU64 worker function takes
+ * no FXSAVE state, just the operands.
+ */
+FNIEMOP_DEF_1(iemOpCommonMmxSseOpt_FullFull_To_Full, PFNIEMAIMPLMEDIAOPTF2U64, pfnU64)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * MMX, MMX.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,    pSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU64, pDst, pSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * MMX, [mem64].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU64, pDst, pSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for MMX instructions on the form:
+ *      pxxx    mm1, mm2/mem64
+ * that was introduced with SSE2.
+ */
+FNIEMOP_DEF_2(iemOpCommonMmx_FullFull_To_Full_Ex, PFNIEMAIMPLMEDIAF2U64, pfnU64, bool, fSupported)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * MMX, MMX.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,    pSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_EX(fSupported);
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * MMX, [mem64].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_EX(fSupported);
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_CALL_MMX_AIMPL_2(pfnU64, pDst, pSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE instructions of the form:
+ *      pxxx    xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * SSE cpuid checks. No SIMD FP exceptions.
+ *
+ * @sa iemOpCommonSse2_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_2(pfnU128, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_2(pfnU128, pDst, pSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE2 instructions on the forms:
+ *      pxxx    xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. SSE2 cpuid checks.
+ *
+ * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse2_FullFull_To_Full, PFNIEMAIMPLMEDIAF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_2(pfnU128, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_2(pfnU128, pDst, pSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE2 instructions on the forms:
+ *      pxxx    xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. SSE2 cpuid checks.
+ *
+ * Unlike iemOpCommonSse2_FullFull_To_Full, the @a pfnU128 worker function takes
+ * no FXSAVE state, just the operands.
+ *
+ * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse2Opt_FullFull_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, pDst, pSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, pDst, pSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for MMX instructions on the forms:
+ *      pxxxx mm1, mm2/mem32
+ *
+ * The 2nd operand is the first half of a register, which in the memory case
+ * means a 32-bit memory access.
+ */
+FNIEMOP_DEF_1(iemOpCommonMmx_LowLow_To_Full, FNIEMAIMPLMEDIAOPTF2U64, pfnU64)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * MMX, MMX.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,              puDst, 0);
+        IEM_MC_ARG(uint64_t const *,        puSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(puDst,          IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_REF_MREG_U64_CONST(puSrc,    IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU64, puDst, puSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(puDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * MMX, [mem32].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint64_t *,                  puDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U32_ZX_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(puDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU64, puDst, puSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(puDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE instructions on the forms:
+ *      pxxxx xmm1, xmm2/mem128
+ *
+ * The 2nd operand is the first half of a register, which in the memory case
+ * 128-bit aligned 64-bit or 128-bit memory accessed for SSE.
+ *
+ * Exceptions type 4.
+ */
+FNIEMOP_DEF_1(iemOpCommonSse_LowLow_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,             puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,            puSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,   IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,             puDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,            uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,  puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        /** @todo Most CPUs probably only read the low qword. We read everything to
+         *        make sure we apply segmentation and alignment checks correctly.
+         *        When we have time, it would be interesting to explore what real
+         *        CPUs actually does and whether it will do a TLB load for the high
+         *        part or skip any associated \#PF. Ditto for segmentation \#GPs. */
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE2 instructions on the forms:
+ *      pxxxx xmm1, xmm2/mem128
+ *
+ * The 2nd operand is the first half of a register, which in the memory case
+ * 128-bit aligned 64-bit or 128-bit memory accessed for SSE.
+ *
+ * Exceptions type 4.
+ */
+FNIEMOP_DEF_1(iemOpCommonSse2_LowLow_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,             puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,            puSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,   IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,             puDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,            uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,  puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        /** @todo Most CPUs probably only read the low qword. We read everything to
+         *        make sure we apply segmentation and alignment checks correctly.
+         *        When we have time, it would be interesting to explore what real
+         *        CPUs actually does and whether it will do a TLB load for the high
+         *        part or skip any associated \#PF. Ditto for segmentation \#GPs. */
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for MMX instructions on the form:
+ *      pxxxx mm1, mm2/mem64
+ *
+ * The 2nd operand is the second half of a register, which in the memory case
+ * means a 64-bit memory access for MMX.
+ */
+FNIEMOP_DEF_1(iemOpCommonMmx_HighHigh_To_Full, PFNIEMAIMPLMEDIAOPTF2U64, pfnU64)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * MMX, MMX.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,              puDst, 0);
+        IEM_MC_ARG(uint64_t const *,        puSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(puDst,          IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_REF_MREG_U64_CONST(puSrc,    IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU64, puDst, puSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(puDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * MMX, [mem64].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(uint64_t *,                  puDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); /* intel docs this to be full 64-bit read */
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(puDst,              IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU64, puDst, puSrc);
+        IEM_MC_MODIFIED_MREG_BY_REF(puDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE instructions on the form:
+ *      pxxxx xmm1, xmm2/mem128
+ *
+ * The 2nd operand is the second half of a register, which for SSE a 128-bit
+ * aligned access where it may read the full 128 bits or only the upper 64 bits.
+ *
+ * Exceptions type 4.
+ */
+FNIEMOP_DEF_1(iemOpCommonSse_HighHigh_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,             puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,            puSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,   IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        /** @todo Most CPUs probably only read the high qword. We read everything to
+         *        make sure we apply segmentation and alignment checks correctly.
+         *        When we have time, it would be interesting to explore what real
+         *        CPUs actually does and whether it will do a TLB load for the lower
+         *        part or skip any associated \#PF. */
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE instructions on the forms:
+ *      pxxs       xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 2. SSE cpuid checks.
+ *
+ * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSseFp_FullFull_To_Full, PFNIEMAIMPLFPSSEF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM128, XMM128.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_LOCAL(IEMSSERESULT,          SseRes);
+        IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes,        SseRes,     0);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc1,                      1);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc2,                      2);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(pSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2);
+        IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM128, [mem128].
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_LOCAL(IEMSSERESULT,          SseRes);
+        IEM_MC_LOCAL(X86XMMREG,             uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes,        SseRes,     0);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc1,                      1);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,   pSrc2, uSrc2,               2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2);
+        IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE instructions on the forms:
+ *      pxxs       xmm1, xmm2/mem32
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 2. SSE cpuid checks.
+ *
+ * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSseFp_FullR32_To_Full, PFNIEMAIMPLFPSSEF2U128R32, pfnU128_R32)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM128, XMM32.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_LOCAL(IEMSSERESULT,          SseRes);
+        IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes,        SseRes,     0);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc1,                      1);
+        IEM_MC_ARG(PCRTFLOAT32U,            pSrc2,                      2);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_R32_CONST(pSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_3(pfnU128_R32, pSseRes, pSrc1, pSrc2);
+        IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM128, [mem32].
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_LOCAL(IEMSSERESULT,          SseRes);
+        IEM_MC_LOCAL(RTFLOAT32U,            r32Src2);
+        IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes,        SseRes,     0);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc1,                      1);
+        IEM_MC_ARG_LOCAL_REF(PCRTFLOAT32U,  pr32Src2,       r32Src2,    2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_R32(r32Src2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_3(pfnU128_R32, pSseRes, pSrc1, pr32Src2);
+        IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE2 instructions on the forms:
+ *      pxxd       xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 2. SSE cpuid checks.
+ *
+ * @sa iemOpCommonSseFp_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse2Fp_FullFull_To_Full, PFNIEMAIMPLFPSSEF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM128, XMM128.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_LOCAL(IEMSSERESULT,          SseRes);
+        IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes,        SseRes,     0);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc1,                      1);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc2,                      2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(pSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2);
+        IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM128, [mem128].
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_LOCAL(IEMSSERESULT,          SseRes);
+        IEM_MC_LOCAL(X86XMMREG,             uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes,        SseRes,     0);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc1,                      1);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,   pSrc2, uSrc2,               2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2);
+        IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE2 instructions on the forms:
+ *      pxxs       xmm1, xmm2/mem64
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 2. SSE2 cpuid checks.
+ *
+ * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse2Fp_FullR64_To_Full, PFNIEMAIMPLFPSSEF2U128R64, pfnU128_R64)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_LOCAL(IEMSSERESULT,          SseRes);
+        IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes,        SseRes,     0);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc1,                      1);
+        IEM_MC_ARG(PCRTFLOAT64U,            pSrc2,                      2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_R64_CONST(pSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_3(pfnU128_R64, pSseRes, pSrc1, pSrc2);
+        IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem64].
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_LOCAL(IEMSSERESULT,          SseRes);
+        IEM_MC_LOCAL(RTFLOAT64U,            r64Src2);
+        IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes,        SseRes,     0);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc1,                      1);
+        IEM_MC_ARG_LOCAL_REF(PCRTFLOAT64U,  pr64Src2,       r64Src2,    2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_R64(r64Src2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_3(pfnU128_R64, pSseRes, pSrc1, pr64Src2);
+        IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE2 instructions on the form:
+ *      pxxxx xmm1, xmm2/mem128
+ *
+ * The 2nd operand is the second half of a register, which for SSE a 128-bit
+ * aligned access where it may read the full 128 bits or only the upper 64 bits.
+ *
+ * Exceptions type 4.
+ */
+FNIEMOP_DEF_1(iemOpCommonSse2_HighHigh_To_Full, PFNIEMAIMPLMEDIAOPTF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,             puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,            puSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,   IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        /** @todo Most CPUs probably only read the high qword. We read everything to
+         *        make sure we apply segmentation and alignment checks correctly.
+         *        When we have time, it would be interesting to explore what real
+         *        CPUs actually does and whether it will do a TLB load for the lower
+         *        part or skip any associated \#PF. */
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE3 instructions on the forms:
+ *      hxxx      xmm1, xmm2/mem128
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 2. SSE3 cpuid checks.
+ *
+ * @sa iemOpCommonSse41_FullFull_To_Full, iemOpCommonSse2_FullFull_To_Full
+ */
+FNIEMOP_DEF_1(iemOpCommonSse3Fp_FullFull_To_Full, PFNIEMAIMPLFPSSEF2U128, pfnU128)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_LOCAL(IEMSSERESULT,          SseRes);
+        IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes,        SseRes,     0);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc1,                      1);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc2,                      2);
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(pSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2);
+        IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_LOCAL(IEMSSERESULT,          SseRes);
+        IEM_MC_LOCAL(X86XMMREG,             uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PIEMSSERESULT, pSseRes,        SseRes,     0);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc1,                      1);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,   pSrc2, uSrc2,               2);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_XMM_CONST(pSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_SSE_AIMPL_3(pfnU128, pSseRes, pSrc1, pSrc2);
+        IEM_MC_STORE_SSE_RESULT(SseRes, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x00 /0. */
+FNIEMOPRM_DEF(iemOp_Grp6_sldt)
+{
+    IEMOP_MNEMONIC(sldt, "sldt Rv/Mw");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DECODED_NL_1(OP_SLDT, IEMOPFORM_M_REG, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+        return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_sldt_reg, IEM_GET_MODRM_RM(pVCpu, bRm), pVCpu->iem.s.enmEffOpSize);
+    }
+
+    /* Ignore operand size here, memory refs are always 16-bit. */
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint16_t, iEffSeg,               0);
+    IEM_MC_ARG(RTGCPTR,  GCPtrEffDst,           1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DECODED_NL_1(OP_SLDT, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_sldt_mem, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x00 /1. */
+FNIEMOPRM_DEF(iemOp_Grp6_str)
+{
+    IEMOP_MNEMONIC(str, "str Rv/Mw");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DECODED_NL_1(OP_STR, IEMOPFORM_M_REG, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+        return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_str_reg, IEM_GET_MODRM_RM(pVCpu, bRm), pVCpu->iem.s.enmEffOpSize);
+    }
+
+    /* Ignore operand size here, memory refs are always 16-bit. */
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint16_t, iEffSeg,               0);
+    IEM_MC_ARG(RTGCPTR,  GCPtrEffDst,           1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DECODED_NL_1(OP_STR, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_str_mem, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x00 /2. */
+FNIEMOPRM_DEF(iemOp_Grp6_lldt)
+{
+    IEMOP_MNEMONIC(lldt, "lldt Ew");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DECODED_NL_1(OP_LLDT, IEMOPFORM_M_REG, OP_PARM_Ew, DISOPTYPE_DANGEROUS);
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_ARG(uint16_t, u16Sel, 0);
+        IEM_MC_FETCH_GREG_U16(u16Sel, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_CIMPL_1(iemCImpl_lldt, u16Sel);
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 1);
+        IEM_MC_ARG(uint16_t, u16Sel, 0);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DECODED_NL_1(OP_LLDT, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS);
+        IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO(); /** @todo test order */
+        IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_CALL_CIMPL_1(iemCImpl_lldt, u16Sel);
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x00 /3. */
+FNIEMOPRM_DEF(iemOp_Grp6_ltr)
+{
+    IEMOP_MNEMONIC(ltr, "ltr Ew");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_ARG(uint16_t, u16Sel, 0);
+        IEM_MC_FETCH_GREG_U16(u16Sel, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_CIMPL_1(iemCImpl_ltr, u16Sel);
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 1);
+        IEM_MC_ARG(uint16_t, u16Sel, 0);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_RAISE_GP0_IF_CPL_NOT_ZERO(); /** @todo test order */
+        IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_CALL_CIMPL_1(iemCImpl_ltr, u16Sel);
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x00 /3. */
+FNIEMOP_DEF_2(iemOpCommonGrp6VerX, uint8_t, bRm, bool, fWrite)
+{
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DECODED_NL_1(fWrite ? OP_VERW : OP_VERR, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint16_t,    u16Sel,            0);
+        IEM_MC_ARG_CONST(bool,  fWriteArg, fWrite, 1);
+        IEM_MC_FETCH_GREG_U16(u16Sel, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_CIMPL_2(iemCImpl_VerX, u16Sel, fWriteArg);
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(2, 1);
+        IEM_MC_ARG(uint16_t,    u16Sel,            0);
+        IEM_MC_ARG_CONST(bool,  fWriteArg, fWrite, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DECODED_NL_1(fWrite ? OP_VERW : OP_VERR, IEMOPFORM_M_MEM, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+        IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_CALL_CIMPL_2(iemCImpl_VerX, u16Sel, fWriteArg);
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x00 /4. */
+FNIEMOPRM_DEF(iemOp_Grp6_verr)
+{
+    IEMOP_MNEMONIC(verr, "verr Ew");
+    IEMOP_HLP_MIN_286();
+    return FNIEMOP_CALL_2(iemOpCommonGrp6VerX, bRm, false);
+}
+
+
+/** Opcode 0x0f 0x00 /5. */
+FNIEMOPRM_DEF(iemOp_Grp6_verw)
+{
+    IEMOP_MNEMONIC(verw, "verw Ew");
+    IEMOP_HLP_MIN_286();
+    return FNIEMOP_CALL_2(iemOpCommonGrp6VerX, bRm, true);
+}
+
+
+/**
+ * Group 6 jump table.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup6[8] =
+{
+    iemOp_Grp6_sldt,
+    iemOp_Grp6_str,
+    iemOp_Grp6_lldt,
+    iemOp_Grp6_ltr,
+    iemOp_Grp6_verr,
+    iemOp_Grp6_verw,
+    iemOp_InvalidWithRM,
+    iemOp_InvalidWithRM
+};
+
+/** Opcode 0x0f 0x00. */
+FNIEMOP_DEF(iemOp_Grp6)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    return FNIEMOP_CALL_1(g_apfnGroup6[IEM_GET_MODRM_REG_8(bRm)], bRm);
+}
+
+
+/** Opcode 0x0f 0x01 /0. */
+FNIEMOP_DEF_1(iemOp_Grp7_sgdt, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(sgdt, "sgdt Ms");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_64BIT_OP_SIZE();
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                    0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEffSrc,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_sgdt, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 /0. */
+FNIEMOP_DEF(iemOp_Grp7_vmcall)
+{
+    IEMOP_MNEMONIC(vmcall, "vmcall");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the VMX instructions. ASSUMING no lock for now. */
+
+    /* Note! We do not check any CPUMFEATURES::fSvm here as we (GIM) generally
+             want all hypercalls regardless of instruction used, and if a
+             hypercall isn't handled by GIM or HMSvm will raise an #UD.
+             (NEM/win makes ASSUMPTIONS about this behavior.)  */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmcall);
+}
+
+
+/** Opcode 0x0f 0x01 /0. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_Grp7_vmlaunch)
+{
+    IEMOP_MNEMONIC(vmlaunch, "vmlaunch");
+    IEMOP_HLP_IN_VMX_OPERATION("vmlaunch", kVmxVDiag_Vmentry);
+    IEMOP_HLP_VMX_INSTR("vmlaunch", kVmxVDiag_Vmentry);
+    IEMOP_HLP_DONE_DECODING();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmlaunch);
+}
+#else
+FNIEMOP_DEF(iemOp_Grp7_vmlaunch)
+{
+    IEMOP_BITCH_ABOUT_STUB();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+#endif
+
+
+/** Opcode 0x0f 0x01 /0. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_Grp7_vmresume)
+{
+    IEMOP_MNEMONIC(vmresume, "vmresume");
+    IEMOP_HLP_IN_VMX_OPERATION("vmresume", kVmxVDiag_Vmentry);
+    IEMOP_HLP_VMX_INSTR("vmresume", kVmxVDiag_Vmentry);
+    IEMOP_HLP_DONE_DECODING();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmresume);
+}
+#else
+FNIEMOP_DEF(iemOp_Grp7_vmresume)
+{
+    IEMOP_BITCH_ABOUT_STUB();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+#endif
+
+
+/** Opcode 0x0f 0x01 /0. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_Grp7_vmxoff)
+{
+    IEMOP_MNEMONIC(vmxoff, "vmxoff");
+    IEMOP_HLP_IN_VMX_OPERATION("vmxoff", kVmxVDiag_Vmxoff);
+    IEMOP_HLP_VMX_INSTR("vmxoff", kVmxVDiag_Vmxoff);
+    IEMOP_HLP_DONE_DECODING();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmxoff);
+}
+#else
+FNIEMOP_DEF(iemOp_Grp7_vmxoff)
+{
+    IEMOP_BITCH_ABOUT_STUB();
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+#endif
+
+
+/** Opcode 0x0f 0x01 /1. */
+FNIEMOP_DEF_1(iemOp_Grp7_sidt, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(sidt, "sidt Ms");
+    IEMOP_HLP_MIN_286();
+    IEMOP_HLP_64BIT_OP_SIZE();
+    IEM_MC_BEGIN(2, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                    0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEffSrc,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_sidt, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 /1. */
+FNIEMOP_DEF(iemOp_Grp7_monitor)
+{
+    IEMOP_MNEMONIC(monitor, "monitor");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo Verify that monitor is allergic to lock prefixes. */
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_monitor, pVCpu->iem.s.iEffSeg);
+}
+
+
+/** Opcode 0x0f 0x01 /1. */
+FNIEMOP_DEF(iemOp_Grp7_mwait)
+{
+    IEMOP_MNEMONIC(mwait, "mwait"); /** @todo Verify that mwait is allergic to lock prefixes. */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_mwait);
+}
+
+
+/** Opcode 0x0f 0x01 /2. */
+FNIEMOP_DEF_1(iemOp_Grp7_lgdt, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(lgdt, "lgdt");
+    IEMOP_HLP_64BIT_OP_SIZE();
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                    0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEffSrc,                                1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSizeArg,/*=*/pVCpu->iem.s.enmEffOpSize, 2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_lgdt, iEffSeg, GCPtrEffSrc, enmEffOpSizeArg);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 0xd0. */
+FNIEMOP_DEF(iemOp_Grp7_xgetbv)
+{
+    IEMOP_MNEMONIC(xgetbv, "xgetbv");
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor)
+    {
+        /** @todo r=ramshankar: We should use
+         *        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX and
+         *        IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES here. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES();
+        return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_xgetbv);
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x0f 0x01 0xd1. */
+FNIEMOP_DEF(iemOp_Grp7_xsetbv)
+{
+    IEMOP_MNEMONIC(xsetbv, "xsetbv");
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor)
+    {
+        /** @todo r=ramshankar: We should use
+         *        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX and
+         *        IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES here. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_REPZ_OR_REPNZ_PREFIXES();
+        return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_xsetbv);
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x0f 0x01 /3. */
+FNIEMOP_DEF_1(iemOp_Grp7_lidt, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(lidt, "lidt");
+    IEMMODE enmEffOpSize = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT
+                         ? IEMMODE_64BIT
+                         : pVCpu->iem.s.enmEffOpSize;
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                            0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEffSrc,                        1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSizeArg,/*=*/enmEffOpSize,  2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_lidt, iEffSeg, GCPtrEffSrc, enmEffOpSizeArg);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 0xd8. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_vmrun)
+{
+    IEMOP_MNEMONIC(vmrun, "vmrun");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmrun);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_vmrun);
+#endif
+
+/** Opcode 0x0f 0x01 0xd9. */
+FNIEMOP_DEF(iemOp_Grp7_Amd_vmmcall)
+{
+    IEMOP_MNEMONIC(vmmcall, "vmmcall");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+
+    /* Note! We do not check any CPUMFEATURES::fSvm here as we (GIM) generally
+             want all hypercalls regardless of instruction used, and if a
+             hypercall isn't handled by GIM or HMSvm will raise an #UD.
+             (NEM/win makes ASSUMPTIONS about this behavior.) */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmmcall);
+}
+
+/** Opcode 0x0f 0x01 0xda. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_vmload)
+{
+    IEMOP_MNEMONIC(vmload, "vmload");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmload);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_vmload);
+#endif
+
+
+/** Opcode 0x0f 0x01 0xdb. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_vmsave)
+{
+    IEMOP_MNEMONIC(vmsave, "vmsave");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_vmsave);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_vmsave);
+#endif
+
+
+/** Opcode 0x0f 0x01 0xdc. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_stgi)
+{
+    IEMOP_MNEMONIC(stgi, "stgi");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_stgi);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_stgi);
+#endif
+
+
+/** Opcode 0x0f 0x01 0xdd. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_clgi)
+{
+    IEMOP_MNEMONIC(clgi, "clgi");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_clgi);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_clgi);
+#endif
+
+
+/** Opcode 0x0f 0x01 0xdf. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_invlpga)
+{
+    IEMOP_MNEMONIC(invlpga, "invlpga");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_invlpga);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_invlpga);
+#endif
+
+
+/** Opcode 0x0f 0x01 0xde. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
+FNIEMOP_DEF(iemOp_Grp7_Amd_skinit)
+{
+    IEMOP_MNEMONIC(skinit, "skinit");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX(); /** @todo check prefix effect on the SVM instructions. ASSUMING no lock for now. */
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_skinit);
+}
+#else
+FNIEMOP_UD_STUB(iemOp_Grp7_Amd_skinit);
+#endif
+
+
+/** Opcode 0x0f 0x01 /4. */
+FNIEMOP_DEF_1(iemOp_Grp7_smsw, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(smsw, "smsw");
+    IEMOP_HLP_MIN_286();
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_smsw_reg, IEM_GET_MODRM_RM(pVCpu, bRm), pVCpu->iem.s.enmEffOpSize);
+    }
+
+    /* Ignore operand size here, memory refs are always 16-bit. */
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint16_t, iEffSeg,               0);
+    IEM_MC_ARG(RTGCPTR,  GCPtrEffDst,           1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_smsw_mem, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 /6. */
+FNIEMOP_DEF_1(iemOp_Grp7_lmsw, uint8_t, bRm)
+{
+    /* The operand size is effectively ignored, all is 16-bit and only the
+       lower 3-bits are used. */
+    IEMOP_MNEMONIC(lmsw, "lmsw");
+    IEMOP_HLP_MIN_286();
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint16_t, u16Tmp,                         0);
+        IEM_MC_ARG_CONST(RTGCPTR,  GCPtrEffDst, NIL_RTGCPTR, 1);
+        IEM_MC_FETCH_GREG_U16(u16Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_CIMPL_2(iemCImpl_lmsw, u16Tmp, GCPtrEffDst);
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint16_t, u16Tmp,      0);
+        IEM_MC_ARG(RTGCPTR,  GCPtrEffDst, 1);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+        IEM_MC_CALL_CIMPL_2(iemCImpl_lmsw, u16Tmp, GCPtrEffDst);
+        IEM_MC_END();
+    }
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 /7. */
+FNIEMOP_DEF_1(iemOp_Grp7_invlpg, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(invlpg, "invlpg");
+    IEMOP_HLP_MIN_486();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_BEGIN(1, 1);
+    IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 0);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEM_MC_CALL_CIMPL_1(iemCImpl_invlpg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0x01 0xf8. */
+FNIEMOP_DEF(iemOp_Grp7_swapgs)
+{
+    IEMOP_MNEMONIC(swapgs, "swapgs");
+    IEMOP_HLP_ONLY_64BIT();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_swapgs);
+}
+
+
+/** Opcode 0x0f 0x01 0xf9. */
+FNIEMOP_DEF(iemOp_Grp7_rdtscp)
+{
+    IEMOP_MNEMONIC(rdtscp, "rdtscp");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdtscp);
+}
+
+
+/**
+ * Group 7 jump table, memory variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup7Mem[8] =
+{
+    iemOp_Grp7_sgdt,
+    iemOp_Grp7_sidt,
+    iemOp_Grp7_lgdt,
+    iemOp_Grp7_lidt,
+    iemOp_Grp7_smsw,
+    iemOp_InvalidWithRM,
+    iemOp_Grp7_lmsw,
+    iemOp_Grp7_invlpg
+};
+
+
+/** Opcode 0x0f 0x01. */
+FNIEMOP_DEF(iemOp_Grp7)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+        return FNIEMOP_CALL_1(g_apfnGroup7Mem[IEM_GET_MODRM_REG_8(bRm)], bRm);
+
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0:
+            switch (IEM_GET_MODRM_RM_8(bRm))
+            {
+                case 1: return FNIEMOP_CALL(iemOp_Grp7_vmcall);
+                case 2: return FNIEMOP_CALL(iemOp_Grp7_vmlaunch);
+                case 3: return FNIEMOP_CALL(iemOp_Grp7_vmresume);
+                case 4: return FNIEMOP_CALL(iemOp_Grp7_vmxoff);
+            }
+            return IEMOP_RAISE_INVALID_OPCODE();
+
+        case 1:
+            switch (IEM_GET_MODRM_RM_8(bRm))
+            {
+                case 0: return FNIEMOP_CALL(iemOp_Grp7_monitor);
+                case 1: return FNIEMOP_CALL(iemOp_Grp7_mwait);
+            }
+            return IEMOP_RAISE_INVALID_OPCODE();
+
+        case 2:
+            switch (IEM_GET_MODRM_RM_8(bRm))
+            {
+                case 0: return FNIEMOP_CALL(iemOp_Grp7_xgetbv);
+                case 1: return FNIEMOP_CALL(iemOp_Grp7_xsetbv);
+            }
+            return IEMOP_RAISE_INVALID_OPCODE();
+
+        case 3:
+            switch (IEM_GET_MODRM_RM_8(bRm))
+            {
+                case 0: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmrun);
+                case 1: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmmcall);
+                case 2: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmload);
+                case 3: return FNIEMOP_CALL(iemOp_Grp7_Amd_vmsave);
+                case 4: return FNIEMOP_CALL(iemOp_Grp7_Amd_stgi);
+                case 5: return FNIEMOP_CALL(iemOp_Grp7_Amd_clgi);
+                case 6: return FNIEMOP_CALL(iemOp_Grp7_Amd_skinit);
+                case 7: return FNIEMOP_CALL(iemOp_Grp7_Amd_invlpga);
+                IEM_NOT_REACHED_DEFAULT_CASE_RET();
+            }
+
+        case 4:
+            return FNIEMOP_CALL_1(iemOp_Grp7_smsw, bRm);
+
+        case 5:
+            return IEMOP_RAISE_INVALID_OPCODE();
+
+        case 6:
+            return FNIEMOP_CALL_1(iemOp_Grp7_lmsw, bRm);
+
+        case 7:
+            switch (IEM_GET_MODRM_RM_8(bRm))
+            {
+                case 0: return FNIEMOP_CALL(iemOp_Grp7_swapgs);
+                case 1: return FNIEMOP_CALL(iemOp_Grp7_rdtscp);
+            }
+            return IEMOP_RAISE_INVALID_OPCODE();
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+/** Opcode 0x0f 0x00 /3. */
+FNIEMOP_DEF_1(iemOpCommonLarLsl_Gv_Ew, bool, fIsLar)
+{
+    IEMOP_HLP_NO_REAL_OR_V86_MODE();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DECODED_NL_2(fIsLar ? OP_LAR : OP_LSL, IEMOPFORM_RM_REG, OP_PARM_Gv, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,  pu16Dst,           0);
+                IEM_MC_ARG(uint16_t,    u16Sel,            1);
+                IEM_MC_ARG_CONST(bool,  fIsLarArg, fIsLar, 2);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U16(u16Sel, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u16, pu16Dst, u16Sel, fIsLarArg);
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_32BIT:
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,  pu64Dst,           0);
+                IEM_MC_ARG(uint16_t,    u16Sel,            1);
+                IEM_MC_ARG_CONST(bool,  fIsLarArg, fIsLar, 2);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U16(u16Sel, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u64, pu64Dst, u16Sel, fIsLarArg);
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+            {
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *,  pu16Dst,           0);
+                IEM_MC_ARG(uint16_t,    u16Sel,            1);
+                IEM_MC_ARG_CONST(bool,  fIsLarArg, fIsLar, 2);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DECODED_NL_2(fIsLar ? OP_LAR : OP_LSL, IEMOPFORM_RM_MEM, OP_PARM_Gv, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+
+                IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u16, pu16Dst, u16Sel, fIsLarArg);
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            case IEMMODE_32BIT:
+            case IEMMODE_64BIT:
+            {
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *,  pu64Dst,           0);
+                IEM_MC_ARG(uint16_t,    u16Sel,            1);
+                IEM_MC_ARG_CONST(bool,  fIsLarArg, fIsLar, 2);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+                IEMOP_HLP_DECODED_NL_2(fIsLar ? OP_LAR : OP_LSL, IEMOPFORM_RM_MEM, OP_PARM_Gv, OP_PARM_Ew, DISOPTYPE_DANGEROUS | DISOPTYPE_PRIVILEGED_NOTRAP);
+/** @todo testcase: make sure it's a 16-bit read. */
+
+                IEM_MC_FETCH_MEM_U16(u16Sel, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_CALL_CIMPL_3(iemCImpl_LarLsl_u64, pu64Dst, u16Sel, fIsLarArg);
+
+                IEM_MC_END();
+                return VINF_SUCCESS;
+            }
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+
+/** Opcode 0x0f 0x02. */
+FNIEMOP_DEF(iemOp_lar_Gv_Ew)
+{
+    IEMOP_MNEMONIC(lar, "lar Gv,Ew");
+    return FNIEMOP_CALL_1(iemOpCommonLarLsl_Gv_Ew, true);
+}
+
+
+/** Opcode 0x0f 0x03. */
+FNIEMOP_DEF(iemOp_lsl_Gv_Ew)
+{
+    IEMOP_MNEMONIC(lsl, "lsl Gv,Ew");
+    return FNIEMOP_CALL_1(iemOpCommonLarLsl_Gv_Ew, false);
+}
+
+
+/** Opcode 0x0f 0x05. */
+FNIEMOP_DEF(iemOp_syscall)
+{
+    IEMOP_MNEMONIC(syscall, "syscall"); /** @todo 286 LOADALL   */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_syscall);
+}
+
+
+/** Opcode 0x0f 0x06. */
+FNIEMOP_DEF(iemOp_clts)
+{
+    IEMOP_MNEMONIC(clts, "clts");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_clts);
+}
+
+
+/** Opcode 0x0f 0x07. */
+FNIEMOP_DEF(iemOp_sysret)
+{
+    IEMOP_MNEMONIC(sysret, "sysret");  /** @todo 386 LOADALL   */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_sysret);
+}
+
+
+/** Opcode 0x0f 0x08. */
+FNIEMOP_DEF(iemOp_invd)
+{
+    IEMOP_MNEMONIC0(FIXED, INVD, invd, DISOPTYPE_PRIVILEGED, 0);
+    IEMOP_HLP_MIN_486();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_invd);
+}
+
+
+/** Opcode 0x0f 0x09. */
+FNIEMOP_DEF(iemOp_wbinvd)
+{
+    IEMOP_MNEMONIC0(FIXED, WBINVD, wbinvd, DISOPTYPE_PRIVILEGED, 0);
+    IEMOP_HLP_MIN_486();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_wbinvd);
+}
+
+
+/** Opcode 0x0f 0x0b. */
+FNIEMOP_DEF(iemOp_ud2)
+{
+    IEMOP_MNEMONIC(ud2, "ud2");
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/** Opcode 0x0f 0x0d. */
+FNIEMOP_DEF(iemOp_nop_Ev_GrpP)
+{
+    /* AMD prefetch group, Intel implements this as NOP Ev (and so do we). */
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fLongMode && !IEM_GET_GUEST_CPU_FEATURES(pVCpu)->f3DNowPrefetch)
+    {
+        IEMOP_MNEMONIC(GrpPNotSupported, "GrpP");
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_MNEMONIC(GrpPInvalid, "GrpP");
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 2: /* Aliased to /0 for the time being. */
+        case 4: /* Aliased to /0 for the time being. */
+        case 5: /* Aliased to /0 for the time being. */
+        case 6: /* Aliased to /0 for the time being. */
+        case 7: /* Aliased to /0 for the time being. */
+        case 0: IEMOP_MNEMONIC(prefetch, "prefetch"); break;
+        case 1: IEMOP_MNEMONIC(prefetchw_1, "prefetchw"); break;
+        case 3: IEMOP_MNEMONIC(prefetchw_3, "prefetchw"); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+
+    IEM_MC_BEGIN(0, 1);
+    IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    /* Currently a NOP. */
+    NOREF(GCPtrEffSrc);
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0x0f 0x0e. */
+FNIEMOP_DEF(iemOp_femms)
+{
+    IEMOP_MNEMONIC(femms, "femms");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_FROM_MMX_MODE();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0x0f 0x0f. */
+FNIEMOP_DEF(iemOp_3Dnow)
+{
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->f3DNow)
+    {
+        IEMOP_MNEMONIC(Inv3Dnow, "3Dnow");
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+
+#ifdef IEM_WITH_3DNOW
+    /* This is pretty sparse, use switch instead of table. */
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL_1(iemOp_3DNowDispatcher, b);
+#else
+    IEMOP_BITCH_ABOUT_STUB();
+    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_movups_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, MOVUPS, movups, Vps_WO, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM128, XMM128.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm),
+                              IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM128, [mem128].
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+}
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movupd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, MOVUPD, movupd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM128, XMM128.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm),
+                              IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM128, [mem128].
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       0xf3
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_movss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, MOVSS, movss, VssZx_WO, Wss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM32, XMM32.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /*a_iDword*/ );
+        IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM128, [mem32].
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U32_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       0xf2
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movsd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, MOVSD, movsd, VsdZx_WO, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM64, XMM64.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM128, [mem64].
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movups_Wps_Vps)
+{
+    IEMOP_MNEMONIC2(MR, MOVUPS, movups, Wps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM128, XMM128.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm),
+                              IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * [mem128], XMM128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movupd_Wpd_Vpd)
+{
+    IEMOP_MNEMONIC2(MR, MOVUPD, movupd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM128, XMM128.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm),
+                              IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * [mem128], XMM128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       0xf3
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_movss_Wss_Vss)
+{
+    IEMOP_MNEMONIC2(MR, MOVSS, movss, Wss_WO, Vss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM32, XMM32.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/);
+        IEM_MC_STORE_XREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), 0 /*a_iDword*/, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * [mem32], XMM32.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/);
+        IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       0xf2
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movsd_Wsd_Vsd)
+{
+    IEMOP_MNEMONIC2(MR, MOVSD, movsd, Wsd_WO, Vsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM64, XMM64.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * [mem64], XMM64.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+FNIEMOP_DEF(iemOp_movlps_Vq_Mq__movhlps)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /**
+         * @opcode      0x12
+         * @opcodesub   11 mr/reg
+         * @oppfx       none
+         * @opcpuid     sse
+         * @opgroup     og_sse_simdfp_datamove
+         * @opxcpttype  5
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(RM_REG, MOVHLPS, movhlps, Vq_WO, UqHi, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 1 /* a_iQword*/);
+        IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x12
+         * @opcodesub   !11 mr/reg
+         * @oppfx       none
+         * @opcpuid     sse
+         * @opgroup     og_sse_simdfp_datamove
+         * @opxcpttype  5
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         * @opfunction  iemOp_movlps_Vq_Mq__vmovhlps
+         */
+        IEMOP_MNEMONIC2(RM_MEM, MOVLPS, movlps, Vq, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x12
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movlpd_Vq_Mq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC2(RM_MEM, MOVLPD, movlpd, Vq_WO, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660f12m3
+     * @opcode      0x12
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x12
+ * @oppfx       0xf3
+ * @opcpuid     sse3
+ * @opgroup     og_sse3_pcksclr_datamove
+ * @opxcpttype  4
+ * @optest      op1=-1 op2=0xdddddddd00000002eeeeeeee00000001 ->
+ *              op1=0x00000002000000020000000100000001
+ */
+FNIEMOP_DEF(iemOp_movsldup_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, MOVSLDUP, movsldup, Vdq_WO, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 0);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 0);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 2);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 2);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 0);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 0);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 2);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 2);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x12
+ * @oppfx       0xf2
+ * @opcpuid     sse3
+ * @opgroup     og_sse3_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=-1 op2=0xddddddddeeeeeeee2222222211111111 ->
+ *              op1=0x22222222111111112222222211111111
+ */
+FNIEMOP_DEF(iemOp_movddup_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, MOVDDUP, movddup, Vdq_WO, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM128, XMM64.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_ARG(uint64_t,                    uSrc, 0);
+
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc);
+        IEM_MC_STORE_XREG_HI_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM128, [mem64].
+         */
+        IEM_MC_BEGIN(1, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+        IEM_MC_ARG(uint64_t,                    uSrc, 0);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc);
+        IEM_MC_STORE_XREG_HI_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x13
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movlps_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC2(MR_MEM, MOVLPS, movlps, Mq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud0f13m3
+     * @opcode      0x13
+     * @opcodesub   11 mr/reg
+     * @oppfx       none
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x13
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movlpd_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC2(MR_MEM, MOVLPD, movlpd, Mq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660f13m3
+     * @opcode      0x13
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opmnemonic  udf30f13
+ * @opcode      0x13
+ * @oppfx       0xf3
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udf20f13
+ * @opcode      0x13
+ * @oppfx       0xf2
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/** Opcode      0x0f 0x14 - unpcklps Vx, Wx*/
+FNIEMOP_DEF(iemOp_unpcklps_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, UNPCKLPS, unpcklps, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse_LowLow_To_Full, iemAImpl_unpcklps_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x14 - unpcklpd Vx, Wx   */
+FNIEMOP_DEF(iemOp_unpcklpd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, UNPCKLPD, unpcklpd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_LowLow_To_Full, iemAImpl_unpcklpd_u128);
+}
+
+
+/**
+ * @opdone
+ * @opmnemonic  udf30f14
+ * @opcode      0x14
+ * @oppfx       0xf3
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udf20f14
+ * @opcode      0x14
+ * @oppfx       0xf2
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/** Opcode      0x0f 0x15 - unpckhps Vx, Wx   */
+FNIEMOP_DEF(iemOp_unpckhps_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, UNPCKHPS, unpckhps, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse_HighHigh_To_Full, iemAImpl_unpckhps_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x15 - unpckhpd Vx, Wx   */
+FNIEMOP_DEF(iemOp_unpckhpd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, UNPCKHPD, unpckhpd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_HighHigh_To_Full, iemAImpl_unpckhpd_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x15 - invalid */
+/*  Opcode 0xf2 0x0f 0x15 - invalid */
+
+/**
+ * @opdone
+ * @opmnemonic  udf30f15
+ * @opcode      0x15
+ * @oppfx       0xf3
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udf20f15
+ * @opcode      0x15
+ * @oppfx       0xf2
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+FNIEMOP_DEF(iemOp_movhps_Vdq_Mq__movlhps_Vdq_Uq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /**
+         * @opcode      0x16
+         * @opcodesub   11 mr/reg
+         * @oppfx       none
+         * @opcpuid     sse
+         * @opgroup     og_sse_simdfp_datamove
+         * @opxcpttype  5
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(RM_REG, MOVLHPS, movlhps, VqHi_WO, Uq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_XREG_HI_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x16
+         * @opcodesub   !11 mr/reg
+         * @oppfx       none
+         * @opcpuid     sse
+         * @opgroup     og_sse_simdfp_datamove
+         * @opxcpttype  5
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         * @opfunction  iemOp_movhps_Vdq_Mq__movlhps_Vdq_Uq
+         */
+        IEMOP_MNEMONIC2(RM_MEM, MOVHPS, movhps, VqHi_WO, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_HI_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x16
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movhpd_Vdq_Mq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC2(RM_MEM, MOVHPD, movhpd, VqHi_WO, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_HI_U64(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660f16m3
+     * @opcode      0x16
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x16
+ * @oppfx       0xf3
+ * @opcpuid     sse3
+ * @opgroup     og_sse3_pcksclr_datamove
+ * @opxcpttype  4
+ * @optest      op1=-1 op2=0x00000002dddddddd00000001eeeeeeee ->
+ *              op1=0x00000002000000020000000100000001
+ */
+FNIEMOP_DEF(iemOp_movshdup_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, MOVSHDUP, movshdup, Vdq_WO, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM128, XMM128.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 1);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 1);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 3);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 3);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM128, [mem128].
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 1);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 1);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 3);
+        IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 3);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/**
+ * @opdone
+ * @opmnemonic  udf30f16
+ * @opcode      0x16
+ * @oppfx       0xf2
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+
+/**
+ * @opcode      0x17
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movhps_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC2(MR_MEM, MOVHPS, movhps, Mq_WO, VqHi, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 1 /* a_iQword*/);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud0f17m3
+     * @opcode      0x17
+     * @opcodesub   11 mr/reg
+     * @oppfx       none
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x17
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movhpd_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC2(MR_MEM, MOVHPD, movhpd, Mq_WO, VqHi, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 1 /* a_iQword*/);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660f17m3
+     * @opcode      0x17
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opdone
+ * @opmnemonic  udf30f17
+ * @opcode      0x17
+ * @oppfx       0xf3
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udf20f17
+ * @opcode      0x17
+ * @oppfx       0xf2
+ * @opunused    intel-modrm
+ * @opcpuid     sse
+ * @optest      ->
+ * @opdone
+ */
+
+
+/** Opcode 0x0f 0x18. */
+FNIEMOP_DEF(iemOp_prefetch_Grp16)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        switch (IEM_GET_MODRM_REG_8(bRm))
+        {
+            case 4: /* Aliased to /0 for the time being according to AMD. */
+            case 5: /* Aliased to /0 for the time being according to AMD. */
+            case 6: /* Aliased to /0 for the time being according to AMD. */
+            case 7: /* Aliased to /0 for the time being according to AMD. */
+            case 0: IEMOP_MNEMONIC(prefetchNTA, "prefetchNTA m8"); break;
+            case 1: IEMOP_MNEMONIC(prefetchT0, "prefetchT0  m8"); break;
+            case 2: IEMOP_MNEMONIC(prefetchT1, "prefetchT1  m8"); break;
+            case 3: IEMOP_MNEMONIC(prefetchT2, "prefetchT2  m8"); break;
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        /* Currently a NOP. */
+        NOREF(GCPtrEffSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x0f 0x19..0x1f. */
+FNIEMOP_DEF(iemOp_nop_Ev)
+{
+    IEMOP_MNEMONIC(nop_Ev, "nop Ev");
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        /* Currently a NOP. */
+        NOREF(GCPtrEffSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x20. */
+FNIEMOP_DEF(iemOp_mov_Rd_Cd)
+{
+    /* mod is ignored, as is operand size overrides. */
+    IEMOP_MNEMONIC(mov_Rd_Cd, "mov Rd,Cd");
+    IEMOP_HLP_MIN_386();
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+    else
+        pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    uint8_t iCrReg = IEM_GET_MODRM_REG(pVCpu, bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)
+    {
+        /* The lock prefix can be used to encode CR8 accesses on some CPUs. */
+        if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovCr8In32Bit)
+            return IEMOP_RAISE_INVALID_OPCODE(); /* #UD takes precedence over #GP(), see test. */
+        iCrReg |= 8;
+    }
+    switch (iCrReg)
+    {
+        case 0: case 2: case 3: case 4: case 8:
+            break;
+        default:
+            return IEMOP_RAISE_INVALID_OPCODE();
+    }
+    IEMOP_HLP_DONE_DECODING();
+
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Rd_Cd, IEM_GET_MODRM_RM(pVCpu, bRm), iCrReg);
+}
+
+
+/** Opcode 0x0f 0x21. */
+FNIEMOP_DEF(iemOp_mov_Rd_Dd)
+{
+    IEMOP_MNEMONIC(mov_Rd_Dd, "mov Rd,Dd");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX_R)
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Rd_Dd,
+                                   IEM_GET_MODRM_RM(pVCpu, bRm),
+                                   IEM_GET_MODRM_REG_8(bRm));
+}
+
+
+/** Opcode 0x0f 0x22. */
+FNIEMOP_DEF(iemOp_mov_Cd_Rd)
+{
+    /* mod is ignored, as is operand size overrides. */
+    IEMOP_MNEMONIC(mov_Cd_Rd, "mov Cd,Rd");
+    IEMOP_HLP_MIN_386();
+    if (pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT)
+        pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_64BIT;
+    else
+        pVCpu->iem.s.enmEffOpSize = pVCpu->iem.s.enmDefOpSize = IEMMODE_32BIT;
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    uint8_t iCrReg = IEM_GET_MODRM_REG(pVCpu, bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK)
+    {
+        /* The lock prefix can be used to encode CR8 accesses on some CPUs. */
+        if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovCr8In32Bit)
+            return IEMOP_RAISE_INVALID_OPCODE(); /* #UD takes precedence over #GP(), see test. */
+        iCrReg |= 8;
+    }
+    switch (iCrReg)
+    {
+        case 0: case 2: case 3: case 4: case 8:
+            break;
+        default:
+            return IEMOP_RAISE_INVALID_OPCODE();
+    }
+    IEMOP_HLP_DONE_DECODING();
+
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Cd_Rd, iCrReg, IEM_GET_MODRM_RM(pVCpu, bRm));
+}
+
+
+/** Opcode 0x0f 0x23. */
+FNIEMOP_DEF(iemOp_mov_Dd_Rd)
+{
+    IEMOP_MNEMONIC(mov_Dd_Rd, "mov Dd,Rd");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_REX_R)
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Dd_Rd,
+                                   IEM_GET_MODRM_REG_8(bRm),
+                                   IEM_GET_MODRM_RM(pVCpu, bRm));
+}
+
+
+/** Opcode 0x0f 0x24. */
+FNIEMOP_DEF(iemOp_mov_Rd_Td)
+{
+    IEMOP_MNEMONIC(mov_Rd_Td, "mov Rd,Td");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (RT_LIKELY(IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_PENTIUM))
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Rd_Td,
+                                   IEM_GET_MODRM_RM(pVCpu, bRm),
+                                   IEM_GET_MODRM_REG_8(bRm));
+}
+
+
+/** Opcode 0x0f 0x26. */
+FNIEMOP_DEF(iemOp_mov_Td_Rd)
+{
+    IEMOP_MNEMONIC(mov_Td_Rd, "mov Td,Rd");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (RT_LIKELY(IEM_GET_TARGET_CPU(pVCpu) >= IEMTARGETCPU_PENTIUM))
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_mov_Td_Rd,
+                                   IEM_GET_MODRM_REG_8(bRm),
+                                   IEM_GET_MODRM_RM(pVCpu, bRm));
+}
+
+
+/**
+ * @opcode      0x28
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movaps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, MOVAPS, movaps, Vps_WO, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm),
+                              IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/**
+ * @opcode      0x28
+ * @oppfx       66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movapd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, MOVAPD, movapd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm),
+                              IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/*  Opcode 0xf3 0x0f 0x28 - invalid */
+/*  Opcode 0xf2 0x0f 0x28 - invalid */
+
+/**
+ * @opcode      0x29
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_simdfp_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movaps_Wps_Vps)
+{
+    IEMOP_MNEMONIC2(MR, MOVAPS, movaps, Wps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm),
+                              IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/**
+ * @opcode      0x29
+ * @oppfx       66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movapd_Wpd_Vpd)
+{
+    IEMOP_MNEMONIC2(MR, MOVAPD, movapd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm),
+                              IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/*  Opcode 0xf3 0x0f 0x29 - invalid */
+/*  Opcode 0xf2 0x0f 0x29 - invalid */
+
+
+/** Opcode      0x0f 0x2a - cvtpi2ps Vps, Qpi */
+FNIEMOP_DEF(iemOp_cvtpi2ps_Vps_Qpi)
+{
+    IEMOP_MNEMONIC2(RM, CVTPI2PS, cvtpi2ps, Vps, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, MMX
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(X86XMMREG,             Dst);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,    pDst, Dst,          1);
+        IEM_MC_ARG(uint64_t,                u64Src,             2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_MAYBE_RAISE_FPU_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Dst, IEM_GET_MODRM_REG(pVCpu, bRm)); /* Need it because the high quadword remains unchanged. */
+        IEM_MC_FETCH_MREG_U64(u64Src, IEM_GET_MODRM_RM_8(bRm));
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpi2ps_u128, pfMxcsr, pDst, u64Src);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem64]
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(X86XMMREG,             Dst);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,    pDst, Dst,          1);
+        IEM_MC_ARG(uint64_t,                u64Src,             2);
+        IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_MAYBE_RAISE_FPU_XCPT();
+        IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpi2ps_u128, pfMxcsr, pDst, u64Src);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x2a - cvtpi2pd Vpd, Qpi */
+FNIEMOP_DEF(iemOp_cvtpi2pd_Vpd_Qpi)
+{
+    IEMOP_MNEMONIC2(RM, CVTPI2PD, cvtpi2pd, Vps, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, MMX
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(X86XMMREG,             Dst);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,    pDst, Dst,          1);
+        IEM_MC_ARG(uint64_t,                u64Src,             2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_MAYBE_RAISE_FPU_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_MREG_U64(u64Src, IEM_GET_MODRM_RM_8(bRm));
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpi2pd_u128, pfMxcsr, pDst, u64Src);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem64]
+         */
+        IEM_MC_BEGIN(3, 3);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(X86XMMREG,             Dst);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,    pDst, Dst,          1);
+        IEM_MC_ARG(uint64_t,                u64Src,             2);
+        IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_MAYBE_RAISE_FPU_XCPT();
+        IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        /* Doesn't cause a transition to MMX mode. */
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpi2pd_u128, pfMxcsr, pDst, u64Src);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0xf3 0x0f 0x2a - cvtsi2ss Vss, Ey */
+FNIEMOP_DEF(iemOp_cvtsi2ss_Vss_Ey)
+{
+    IEMOP_MNEMONIC2(RM, CVTSI2SS, cvtsi2ss, Vss, Ey, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* XMM, greg64 */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,    fMxcsr);
+            IEM_MC_LOCAL(RTFLOAT32U,  r32Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,    pfMxcsr, fMxcsr,  0);
+            IEM_MC_ARG_LOCAL_REF(PRTFLOAT32U,   pr32Dst, r32Dst,  1);
+            IEM_MC_ARG(const int64_t *,         pi64Src,          2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_GREG_I64_CONST(pi64Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2ss_r32_i64, pfMxcsr, pr32Dst, pi64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_XREG_R32(IEM_GET_MODRM_REG(pVCpu, bRm), r32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem64] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,   fMxcsr);
+            IEM_MC_LOCAL(RTFLOAT32U, r32Dst);
+            IEM_MC_LOCAL(int64_t,    i64Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,        pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(PRTFLOAT32U,       pr32Dst, r32Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const int64_t *,   pi64Src, i64Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_I64(i64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2ss_r32_i64, pfMxcsr, pr32Dst, pi64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_XREG_R32(IEM_GET_MODRM_REG(pVCpu, bRm), r32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg, XMM */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,   fMxcsr);
+            IEM_MC_LOCAL(RTFLOAT32U, r32Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,  pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(PRTFLOAT32U, pr32Dst, r32Dst,     1);
+            IEM_MC_ARG(const int32_t *,       pi32Src,             2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_GREG_I32_CONST(pi32Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2ss_r32_i32, pfMxcsr, pr32Dst, pi32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_XREG_R32(IEM_GET_MODRM_REG(pVCpu, bRm), r32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* greg, [mem32] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,   fMxcsr);
+            IEM_MC_LOCAL(RTFLOAT32U, r32Dst);
+            IEM_MC_LOCAL(int32_t,    i32Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,       pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(PRTFLOAT32U,      pr32Dst, r32Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const int32_t *,  pi32Src, i32Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_I32(i32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2ss_r32_i32, pfMxcsr, pr32Dst, pi32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_XREG_R32(IEM_GET_MODRM_REG(pVCpu, bRm), r32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode 0xf2 0x0f 0x2a - cvtsi2sd Vsd, Ey */
+FNIEMOP_DEF(iemOp_cvtsi2sd_Vsd_Ey)
+{
+    IEMOP_MNEMONIC2(RM, CVTSI2SD, cvtsi2sd, Vsd, Ey, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* XMM, greg64 */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,    fMxcsr);
+            IEM_MC_LOCAL(RTFLOAT64U,  r64Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,    pfMxcsr, fMxcsr,  0);
+            IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U,   pr64Dst, r64Dst,  1);
+            IEM_MC_ARG(const int64_t *,         pi64Src,          2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_GREG_I64_CONST(pi64Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2sd_r64_i64, pfMxcsr, pr64Dst, pi64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_XREG_R64(IEM_GET_MODRM_REG(pVCpu, bRm), r64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem64] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,   fMxcsr);
+            IEM_MC_LOCAL(RTFLOAT64U, r64Dst);
+            IEM_MC_LOCAL(int64_t,    i64Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,        pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U,       pr64Dst, r64Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const int64_t *,   pi64Src, i64Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_I64(i64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2sd_r64_i64, pfMxcsr, pr64Dst, pi64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_XREG_R64(IEM_GET_MODRM_REG(pVCpu, bRm), r64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* XMM, greg32 */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,   fMxcsr);
+            IEM_MC_LOCAL(RTFLOAT64U, r64Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,  pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U, pr64Dst, r64Dst,     1);
+            IEM_MC_ARG(const int32_t *,       pi32Src,             2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_GREG_I32_CONST(pi32Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2sd_r64_i32, pfMxcsr, pr64Dst, pi32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_XREG_R64(IEM_GET_MODRM_REG(pVCpu, bRm), r64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem32] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,   fMxcsr);
+            IEM_MC_LOCAL(RTFLOAT64U, r64Dst);
+            IEM_MC_LOCAL(int32_t,    i32Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,       pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(PRTFLOAT64U,      pr64Dst, r64Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const int32_t *,  pi32Src, i32Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_I32(i32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsi2sd_r64_i32, pfMxcsr, pr64Dst, pi32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_XREG_R64(IEM_GET_MODRM_REG(pVCpu, bRm), r64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x2b
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse1_cachect
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movntps_Mps_Vps)
+{
+    IEMOP_MNEMONIC2(MR_MEM, MOVNTPS, movntps, Mps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        /*
+         * memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    /* The register, register encoding is invalid. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/**
+ * @opcode      0x2b
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_cachect
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movntpd_Mpd_Vpd)
+{
+    IEMOP_MNEMONIC2(MR_MEM, MOVNTPD, movntpd, Mpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        /*
+         * memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc); /** @todo optimize this one day... */
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    /* The register, register encoding is invalid. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+/*  Opcode 0xf3 0x0f 0x2b - invalid */
+/*  Opcode 0xf2 0x0f 0x2b - invalid */
+
+
+/** Opcode      0x0f 0x2c - cvttps2pi Ppi, Wps */
+FNIEMOP_DEF(iemOp_cvttps2pi_Ppi_Wps)
+{
+    IEMOP_MNEMONIC2(RM, CVTTPS2PI, cvttps2pi, Pq, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(uint64_t,              u64Dst);
+        IEM_MC_ARG_LOCAL_REF(uint64_t *,    pu64Dst, u64Dst,    1);
+        IEM_MC_ARG(uint64_t,                u64Src,             2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_U64(u64Src, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/);
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvttps2pi_u128, pfMxcsr, pu64Dst, u64Src);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(uint64_t,              u64Dst);
+        IEM_MC_ARG_LOCAL_REF(uint64_t *,    pu64Dst, u64Dst,    1);
+        IEM_MC_ARG(uint64_t,                u64Src,             2);
+        IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvttps2pi_u128, pfMxcsr, pu64Dst, u64Src);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x2c - cvttpd2pi Ppi, Wpd */
+FNIEMOP_DEF(iemOp_cvttpd2pi_Ppi_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, CVTTPD2PI, cvttpd2pi, Pq, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(uint64_t,              u64Dst);
+        IEM_MC_ARG_LOCAL_REF(uint64_t *,    pu64Dst, u64Dst,    1);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc,               2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvttpd2pi_u128, pfMxcsr, pu64Dst, pSrc);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 3);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(uint64_t,              u64Dst);
+        IEM_MC_ARG_LOCAL_REF(uint64_t *,    pu64Dst,    u64Dst, 1);
+        IEM_MC_LOCAL(X86XMMREG,             uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,   pSrc,       uSrc,   2);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MXCSR(pfMxcsr);
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvttpd2pi_u128, pfMxcsr, pu64Dst, pSrc);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0xf3 0x0f 0x2c - cvttss2si Gy, Wss */
+FNIEMOP_DEF(iemOp_cvttss2si_Gy_Wss)
+{
+    IEMOP_MNEMONIC2(RM, CVTTSS2SI, cvttss2si, Gy, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg64, XMM */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int64_t,   i64Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int64_t *,  pi64Dst, i64Dst,     1);
+            IEM_MC_ARG(const uint32_t *,     pu32Src,             2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_XREG_U32_CONST(pu32Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttss2si_i64_r32, pfMxcsr, pi64Dst, pu32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* greg64, [mem64] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int64_t,   i64Dst);
+            IEM_MC_LOCAL(uint32_t,  u32Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,        pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int64_t *,         pi64Dst, i64Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const uint32_t *,  pu32Src, u32Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttss2si_i64_r32, pfMxcsr, pi64Dst, pu32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg, XMM */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int32_t,   i32Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int32_t *,  pi32Dst, i32Dst,     1);
+            IEM_MC_ARG(const uint32_t *,     pu32Src,             2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_XREG_U32_CONST(pu32Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttss2si_i32_r32, pfMxcsr, pi32Dst, pu32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* greg, [mem] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int32_t,   i32Dst);
+            IEM_MC_LOCAL(uint32_t,  u32Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,        pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int32_t *,         pi32Dst, i32Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const uint32_t *,  pu32Src, u32Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttss2si_i32_r32, pfMxcsr, pi32Dst, pu32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode 0xf2 0x0f 0x2c - cvttsd2si Gy, Wsd */
+FNIEMOP_DEF(iemOp_cvttsd2si_Gy_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, CVTTSD2SI, cvttsd2si, Gy, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg64, XMM */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int64_t,   i64Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int64_t *,  pi64Dst, i64Dst,     1);
+            IEM_MC_ARG(const uint64_t *,     pu64Src,             2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_XREG_U64_CONST(pu64Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttsd2si_i64_r64, pfMxcsr, pi64Dst, pu64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* greg64, [mem64] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int64_t,   i64Dst);
+            IEM_MC_LOCAL(uint64_t,  u64Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,        pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int64_t *,         pi64Dst, i64Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const uint64_t *,  pu64Src, u64Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttsd2si_i64_r64, pfMxcsr, pi64Dst, pu64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg, XMM */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int32_t,   i32Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int32_t *,  pi32Dst, i32Dst,     1);
+            IEM_MC_ARG(const uint64_t *,     pu64Src,             2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_XREG_U64_CONST(pu64Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttsd2si_i32_r64, pfMxcsr, pi32Dst, pu64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* greg32, [mem32] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int32_t,   i32Dst);
+            IEM_MC_LOCAL(uint64_t,  u64Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,        pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int32_t *,         pi32Dst, i32Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const uint64_t *,  pu64Src, u64Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvttsd2si_i32_r64, pfMxcsr, pi32Dst, pu64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode      0x0f 0x2d - cvtps2pi Ppi, Wps */
+FNIEMOP_DEF(iemOp_cvtps2pi_Ppi_Wps)
+{
+    IEMOP_MNEMONIC2(RM, CVTPS2PI, cvtps2pi, Pq, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(uint64_t,              u64Dst);
+        IEM_MC_ARG_LOCAL_REF(uint64_t *,    pu64Dst, u64Dst,    1);
+        IEM_MC_ARG(uint64_t,                u64Src,             2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_U64(u64Src, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/);
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtps2pi_u128, pfMxcsr, pu64Dst, u64Src);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(uint64_t,              u64Dst);
+        IEM_MC_ARG_LOCAL_REF(uint64_t *,    pu64Dst, u64Dst,    1);
+        IEM_MC_ARG(uint64_t,                u64Src,             2);
+        IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtps2pi_u128, pfMxcsr, pu64Dst, u64Src);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x2d - cvtpd2pi Qpi, Wpd */
+FNIEMOP_DEF(iemOp_cvtpd2pi_Qpi_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, CVTPD2PI, cvtpd2pi, Pq, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /// @todo
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(uint64_t,              u64Dst);
+        IEM_MC_ARG_LOCAL_REF(uint64_t *,    pu64Dst, u64Dst,    1);
+        IEM_MC_ARG(PCX86XMMREG,             pSrc,               2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpd2pi_u128, pfMxcsr, pu64Dst, pSrc);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 3);
+        IEM_MC_ARG(uint32_t *,              pfMxcsr,            0);
+        IEM_MC_LOCAL(uint64_t,              u64Dst);
+        IEM_MC_ARG_LOCAL_REF(uint64_t *,    pu64Dst,    u64Dst, 1);
+        IEM_MC_LOCAL(X86XMMREG,             uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,   pSrc,       uSrc,   2);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MXCSR(pfMxcsr);
+
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_cvtpd2pi_u128, pfMxcsr, pu64Dst, pSrc);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0xf3 0x0f 0x2d - cvtss2si Gy, Wss */
+FNIEMOP_DEF(iemOp_cvtss2si_Gy_Wss)
+{
+    IEMOP_MNEMONIC2(RM, CVTSS2SI, cvtss2si, Gy, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg64, XMM */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int64_t,   i64Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int64_t *,  pi64Dst, i64Dst,     1);
+            IEM_MC_ARG(const uint32_t *,     pu32Src,             2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_XREG_U32_CONST(pu32Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtss2si_i64_r32, pfMxcsr, pi64Dst, pu32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* greg64, [mem64] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int64_t,   i64Dst);
+            IEM_MC_LOCAL(uint32_t,  u32Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,        pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int64_t *,         pi64Dst, i64Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const uint32_t *,  pu32Src, u32Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtss2si_i64_r32, pfMxcsr, pi64Dst, pu32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg, XMM */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int32_t,   i32Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int32_t *,  pi32Dst, i32Dst,     1);
+            IEM_MC_ARG(const uint32_t *,     pu32Src,             2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_XREG_U32_CONST(pu32Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtss2si_i32_r32, pfMxcsr, pi32Dst, pu32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* greg, [mem] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int32_t,   i32Dst);
+            IEM_MC_LOCAL(uint32_t,  u32Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,        pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int32_t *,         pi32Dst, i32Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const uint32_t *,  pu32Src, u32Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtss2si_i32_r32, pfMxcsr, pi32Dst, pu32Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode 0xf2 0x0f 0x2d - cvtsd2si Gy, Wsd */
+FNIEMOP_DEF(iemOp_cvtsd2si_Gy_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, CVTSD2SI, cvtsd2si, Gy, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg64, XMM */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int64_t,   i64Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int64_t *,  pi64Dst, i64Dst,     1);
+            IEM_MC_ARG(const uint64_t *,     pu64Src,             2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_XREG_U64_CONST(pu64Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsd2si_i64_r64, pfMxcsr, pi64Dst, pu64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* greg64, [mem64] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int64_t,   i64Dst);
+            IEM_MC_LOCAL(uint64_t,  u64Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,        pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int64_t *,         pi64Dst, i64Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const uint64_t *,  pu64Src, u64Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsd2si_i64_r64, pfMxcsr, pi64Dst, pu64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_I64(IEM_GET_MODRM_REG(pVCpu, bRm), i64Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg32, XMM */
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int32_t,   i32Dst);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *, pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int32_t *,  pi32Dst, i32Dst,     1);
+            IEM_MC_ARG(const uint64_t *,     pu64Src,             2);
+
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_REF_XREG_U64_CONST(pu64Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsd2si_i32_r64, pfMxcsr, pi32Dst, pu64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* greg32, [mem64] */
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTGCPTR,   GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t,  fMxcsr);
+            IEM_MC_LOCAL(int32_t,   i32Dst);
+            IEM_MC_LOCAL(uint64_t,  u64Src);
+            IEM_MC_ARG_LOCAL_REF(uint32_t *,        pfMxcsr, fMxcsr,     0);
+            IEM_MC_ARG_LOCAL_REF(int32_t *,         pi32Dst, i32Dst,     1);
+            IEM_MC_ARG_LOCAL_REF(const uint64_t *,  pu64Src, u64Src,     2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_PREPARE_SSE_USAGE(); /** @todo: This is superfluous because IEM_MC_CALL_SSE_AIMPL_3() is calling this but the tstIEMCheckMc testcase depends on it. */
+
+            IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_SSE_AIMPL_3(iemAImpl_cvtsd2si_i32_r64, pfMxcsr, pi32Dst, pu64Src);
+            IEM_MC_SSE_UPDATE_MXCSR(fMxcsr);
+            IEM_MC_IF_MXCSR_XCPT_PENDING()
+                IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+            IEM_MC_ELSE()
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), i32Dst);
+            IEM_MC_ENDIF();
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode      0x0f 0x2e - ucomiss Vss, Wss */
+FNIEMOP_DEF(iemOp_ucomiss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, UCOMISS, ucomiss, Vss, Wss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 1);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc2,             3);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(puSrc2,      IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_ucomiss_u128, pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_LOCAL(X86XMMREG,                 uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,       puSrc2, uSrc2,      3);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,       IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_ucomiss_u128, pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x2e - ucomisd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_ucomisd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, UCOMISD, ucomisd, Vsd, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 1);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc2,             3);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(puSrc2,      IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_ucomisd_u128, pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_LOCAL(X86XMMREG,                 uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,       puSrc2, uSrc2,      3);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U64(uSrc2, 0 /*a_QWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,       IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_ucomisd_u128, pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode 0xf3 0x0f 0x2e - invalid */
+/*  Opcode 0xf2 0x0f 0x2e - invalid */
+
+
+/** Opcode      0x0f 0x2f - comiss Vss, Wss */
+FNIEMOP_DEF(iemOp_comiss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, COMISS, comiss, Vss, Wss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 1);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc2,             3);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(puSrc2,      IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_comiss_u128, pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_LOCAL(X86XMMREG,                 uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,       puSrc2, uSrc2,      3);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,       IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_comiss_u128, pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x2f - comisd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_comisd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, COMISD, comisd, Vsd, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 1);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc2,             3);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(puSrc2,      IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_comisd_u128, pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_LOCAL(X86XMMREG,                 uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,       puSrc2, uSrc2,      3);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U64(uSrc2, 0 /*a_QWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,       IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_comisd_u128, pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode 0xf3 0x0f 0x2f - invalid */
+/*  Opcode 0xf2 0x0f 0x2f - invalid */
+
+/** Opcode 0x0f 0x30. */
+FNIEMOP_DEF(iemOp_wrmsr)
+{
+    IEMOP_MNEMONIC(wrmsr, "wrmsr");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_wrmsr);
+}
+
+
+/** Opcode 0x0f 0x31. */
+FNIEMOP_DEF(iemOp_rdtsc)
+{
+    IEMOP_MNEMONIC(rdtsc, "rdtsc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdtsc);
+}
+
+
+/** Opcode 0x0f 0x33. */
+FNIEMOP_DEF(iemOp_rdmsr)
+{
+    IEMOP_MNEMONIC(rdmsr, "rdmsr");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdmsr);
+}
+
+
+/** Opcode 0x0f 0x34. */
+FNIEMOP_DEF(iemOp_rdpmc)
+{
+    IEMOP_MNEMONIC(rdpmc, "rdpmc");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rdpmc);
+}
+
+
+/** Opcode 0x0f 0x34. */
+FNIEMOP_DEF(iemOp_sysenter)
+{
+    IEMOP_MNEMONIC0(FIXED, SYSENTER, sysenter, DISOPTYPE_CONTROLFLOW | DISOPTYPE_UNCOND_CONTROLFLOW, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_sysenter);
+}
+
+/** Opcode 0x0f 0x35. */
+FNIEMOP_DEF(iemOp_sysexit)
+{
+    IEMOP_MNEMONIC0(FIXED, SYSEXIT, sysexit, DISOPTYPE_CONTROLFLOW | DISOPTYPE_UNCOND_CONTROLFLOW, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_1(iemCImpl_sysexit, pVCpu->iem.s.enmEffOpSize);
+}
+
+/** Opcode 0x0f 0x37. */
+FNIEMOP_STUB(iemOp_getsec);
+
+
+/** Opcode 0x0f 0x38. */
+FNIEMOP_DEF(iemOp_3byte_Esc_0f_38)
+{
+#ifdef IEM_WITH_THREE_0F_38
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnThreeByte0f38[(uintptr_t)b * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+    IEMOP_BITCH_ABOUT_STUB();
+    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/** Opcode 0x0f 0x3a. */
+FNIEMOP_DEF(iemOp_3byte_Esc_0f_3a)
+{
+#ifdef IEM_WITH_THREE_0F_3A
+    uint8_t b; IEM_OPCODE_GET_NEXT_U8(&b);
+    return FNIEMOP_CALL(g_apfnThreeByte0f3a[(uintptr_t)b * 4 + pVCpu->iem.s.idxPrefix]);
+#else
+    IEMOP_BITCH_ABOUT_STUB();
+    return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#endif
+}
+
+
+/**
+ * Implements a conditional move.
+ *
+ * Wish there was an obvious way to do this where we could share and reduce
+ * code bloat.
+ *
+ * @param   a_Cnd       The conditional "microcode" operation.
+ */
+#define CMOV_X(a_Cnd) \
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \
+    if (IEM_IS_MODRM_REG_MODE(bRm)) \
+    { \
+        switch (pVCpu->iem.s.enmEffOpSize) \
+        { \
+            case IEMMODE_16BIT: \
+                IEM_MC_BEGIN(0, 1); \
+                IEM_MC_LOCAL(uint16_t, u16Tmp); \
+                a_Cnd { \
+                    IEM_MC_FETCH_GREG_U16(u16Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+                    IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+                IEM_MC_END(); \
+                break; \
+    \
+            case IEMMODE_32BIT: \
+                IEM_MC_BEGIN(0, 1); \
+                IEM_MC_LOCAL(uint32_t, u32Tmp); \
+                a_Cnd { \
+                    IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+                    IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); \
+                } IEM_MC_ELSE() { \
+                    IEM_MC_CLEAR_HIGH_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm)); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+                IEM_MC_END(); \
+                break; \
+    \
+            case IEMMODE_64BIT: \
+                IEM_MC_BEGIN(0, 1); \
+                IEM_MC_LOCAL(uint64_t, u64Tmp); \
+                a_Cnd { \
+                    IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+                    IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+                IEM_MC_END(); \
+                break; \
+    \
+            IEM_NOT_REACHED_DEFAULT_CASE_RET(); \
+        } \
+    } \
+    else \
+    { \
+        switch (pVCpu->iem.s.enmEffOpSize) \
+        { \
+            case IEMMODE_16BIT: \
+                IEM_MC_BEGIN(0, 2); \
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc); \
+                IEM_MC_LOCAL(uint16_t, u16Tmp); \
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+                IEM_MC_FETCH_MEM_U16(u16Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+                a_Cnd { \
+                    IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Tmp); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+                IEM_MC_END(); \
+                break; \
+    \
+            case IEMMODE_32BIT: \
+                IEM_MC_BEGIN(0, 2); \
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc); \
+                IEM_MC_LOCAL(uint32_t, u32Tmp); \
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+                IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+                a_Cnd { \
+                    IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp); \
+                } IEM_MC_ELSE() { \
+                    IEM_MC_CLEAR_HIGH_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm)); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+                IEM_MC_END(); \
+                break; \
+    \
+            case IEMMODE_64BIT: \
+                IEM_MC_BEGIN(0, 2); \
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc); \
+                IEM_MC_LOCAL(uint64_t, u64Tmp); \
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+                IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+                a_Cnd { \
+                    IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp); \
+                } IEM_MC_ENDIF(); \
+                IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+                IEM_MC_END(); \
+                break; \
+    \
+            IEM_NOT_REACHED_DEFAULT_CASE_RET(); \
+        } \
+    } do {} while (0)
+
+
+
+/** Opcode 0x0f 0x40. */
+FNIEMOP_DEF(iemOp_cmovo_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovo_Gv_Ev, "cmovo Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF));
+}
+
+
+/** Opcode 0x0f 0x41. */
+FNIEMOP_DEF(iemOp_cmovno_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovno_Gv_Ev, "cmovno Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_OF));
+}
+
+
+/** Opcode 0x0f 0x42. */
+FNIEMOP_DEF(iemOp_cmovc_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovc_Gv_Ev, "cmovc Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF));
+}
+
+
+/** Opcode 0x0f 0x43. */
+FNIEMOP_DEF(iemOp_cmovnc_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovnc_Gv_Ev, "cmovnc Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_CF));
+}
+
+
+/** Opcode 0x0f 0x44. */
+FNIEMOP_DEF(iemOp_cmove_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmove_Gv_Ev, "cmove Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF));
+}
+
+
+/** Opcode 0x0f 0x45. */
+FNIEMOP_DEF(iemOp_cmovne_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovne_Gv_Ev, "cmovne Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF));
+}
+
+
+/** Opcode 0x0f 0x46. */
+FNIEMOP_DEF(iemOp_cmovbe_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovbe_Gv_Ev, "cmovbe Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF));
+}
+
+
+/** Opcode 0x0f 0x47. */
+FNIEMOP_DEF(iemOp_cmovnbe_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovnbe_Gv_Ev, "cmovnbe Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_NO_BITS_SET(X86_EFL_CF | X86_EFL_ZF));
+}
+
+
+/** Opcode 0x0f 0x48. */
+FNIEMOP_DEF(iemOp_cmovs_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovs_Gv_Ev, "cmovs Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF));
+}
+
+
+/** Opcode 0x0f 0x49. */
+FNIEMOP_DEF(iemOp_cmovns_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovns_Gv_Ev, "cmovns Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_SF));
+}
+
+
+/** Opcode 0x0f 0x4a. */
+FNIEMOP_DEF(iemOp_cmovp_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovp_Gv_Ev, "cmovp Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF));
+}
+
+
+/** Opcode 0x0f 0x4b. */
+FNIEMOP_DEF(iemOp_cmovnp_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovnp_Gv_Ev, "cmovnp Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_PF));
+}
+
+
+/** Opcode 0x0f 0x4c. */
+FNIEMOP_DEF(iemOp_cmovl_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovl_Gv_Ev, "cmovl Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF));
+}
+
+
+/** Opcode 0x0f 0x4d. */
+FNIEMOP_DEF(iemOp_cmovnl_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovnl_Gv_Ev, "cmovnl Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BITS_EQ(X86_EFL_SF, X86_EFL_OF));
+}
+
+
+/** Opcode 0x0f 0x4e. */
+FNIEMOP_DEF(iemOp_cmovle_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovle_Gv_Ev, "cmovle Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF));
+}
+
+
+/** Opcode 0x0f 0x4f. */
+FNIEMOP_DEF(iemOp_cmovnle_Gv_Ev)
+{
+    IEMOP_MNEMONIC(cmovnle_Gv_Ev, "cmovnle Gv,Ev");
+    CMOV_X(IEM_MC_IF_EFL_BIT_NOT_SET_AND_BITS_EQ(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF));
+}
+
+#undef CMOV_X
+
+/** Opcode      0x0f 0x50 - movmskps Gy, Ups */
+FNIEMOP_DEF(iemOp_movmskps_Gy_Ups)
+{
+    IEMOP_MNEMONIC2(RM_REG, MOVMSKPS, movmskps, Gy, Ux, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /** @todo */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 1);
+        IEM_MC_LOCAL(uint8_t,           u8Dst);
+        IEM_MC_ARG_LOCAL_REF(uint8_t *, pu8Dst,  u8Dst, 0);
+        IEM_MC_ARG(PCRTUINT128U,        puSrc,          1);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_movmskps_u128, pu8Dst, puSrc);
+        IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u8Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    /* No memory operand. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x66 0x0f 0x50 - movmskpd Gy, Upd */
+FNIEMOP_DEF(iemOp_movmskpd_Gy_Upd)
+{
+    IEMOP_MNEMONIC2(RM_REG, MOVMSKPD, movmskpd, Gy, Ux, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0); /** @todo */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 1);
+        IEM_MC_LOCAL(uint8_t,           u8Dst);
+        IEM_MC_ARG_LOCAL_REF(uint8_t *, pu8Dst,  u8Dst, 0);
+        IEM_MC_ARG(PCRTUINT128U,        puSrc,          1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_movmskpd_u128, pu8Dst, puSrc);
+        IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG_8(bRm), u8Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    /* No memory operand. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+}
+
+
+/*  Opcode 0xf3 0x0f 0x50 - invalid */
+/*  Opcode 0xf2 0x0f 0x50 - invalid */
+
+
+/** Opcode      0x0f 0x51 - sqrtps Vps, Wps */
+FNIEMOP_DEF(iemOp_sqrtps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, SQRTPS, sqrtps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_sqrtps_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x51 - sqrtpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_sqrtpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, SQRTPD, sqrtpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_sqrtpd_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0x51 - sqrtss Vss, Wss */
+FNIEMOP_DEF(iemOp_sqrtss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, SQRTSS, sqrtss, Vss, Wss, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_sqrtss_u128_r32);
+}
+
+
+/** Opcode 0xf2 0x0f 0x51 - sqrtsd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_sqrtsd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, SQRTSD, sqrtsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_sqrtsd_u128_r64);
+}
+
+
+/** Opcode      0x0f 0x52 - rsqrtps Vps, Wps */
+FNIEMOP_DEF(iemOp_rsqrtps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, RSQRTPS, rsqrtps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_rsqrtps_u128);
+}
+
+
+/*  Opcode 0x66 0x0f 0x52 - invalid */
+
+
+/** Opcode 0xf3 0x0f 0x52 - rsqrtss Vss, Wss */
+FNIEMOP_DEF(iemOp_rsqrtss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, RSQRTSS, rsqrtss, Vss, Wss, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_rsqrtss_u128_r32);
+}
+
+
+/*  Opcode 0xf2 0x0f 0x52 - invalid */
+
+/** Opcode      0x0f 0x53 - rcpps Vps, Wps */
+FNIEMOP_STUB(iemOp_rcpps_Vps_Wps);
+/*  Opcode 0x66 0x0f 0x53 - invalid */
+/** Opcode 0xf3 0x0f 0x53 - rcpss Vss, Wss */
+FNIEMOP_STUB(iemOp_rcpss_Vss_Wss);
+/*  Opcode 0xf2 0x0f 0x53 - invalid */
+
+
+/** Opcode      0x0f 0x54 - andps Vps, Wps */
+FNIEMOP_DEF(iemOp_andps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, ANDPS, andps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse_FullFull_To_Full, iemAImpl_pand_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x54 - andpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_andpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, ANDPD, andpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pand_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x54 - invalid */
+/*  Opcode 0xf2 0x0f 0x54 - invalid */
+
+
+/** Opcode      0x0f 0x55 - andnps Vps, Wps */
+FNIEMOP_DEF(iemOp_andnps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, ANDNPS, andnps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse_FullFull_To_Full, iemAImpl_pandn_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x55 - andnpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_andnpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, ANDNPD, andnpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pandn_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x55 - invalid */
+/*  Opcode 0xf2 0x0f 0x55 - invalid */
+
+
+/** Opcode      0x0f 0x56 - orps Vps, Wps */
+FNIEMOP_DEF(iemOp_orps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, ORPS, orps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse_FullFull_To_Full, iemAImpl_por_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x56 - orpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_orpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, ORPD, orpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_por_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x56 - invalid */
+/*  Opcode 0xf2 0x0f 0x56 - invalid */
+
+
+/** Opcode      0x0f 0x57 - xorps Vps, Wps */
+FNIEMOP_DEF(iemOp_xorps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, XORPS, xorps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse_FullFull_To_Full, iemAImpl_pxor_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x57 - xorpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_xorpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, XORPD, xorpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pxor_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x57 - invalid */
+/*  Opcode 0xf2 0x0f 0x57 - invalid */
+
+/** Opcode      0x0f 0x58 - addps Vps, Wps */
+FNIEMOP_DEF(iemOp_addps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, ADDPS, addps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_addps_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x58 - addpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_addpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, ADDPD, addpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_addpd_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0x58 - addss Vss, Wss */
+FNIEMOP_DEF(iemOp_addss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, ADDSS, addss, Vss, Wss, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_addss_u128_r32);
+}
+
+
+/** Opcode 0xf2 0x0f 0x58 - addsd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_addsd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, ADDSD, addsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_addsd_u128_r64);
+}
+
+
+/** Opcode      0x0f 0x59 - mulps Vps, Wps */
+FNIEMOP_DEF(iemOp_mulps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, MULPS, mulps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_mulps_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x59 - mulpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_mulpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, MULPD, mulpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_mulpd_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0x59 - mulss Vss, Wss */
+FNIEMOP_DEF(iemOp_mulss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, MULSS, mulss, Vss, Wss, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_mulss_u128_r32);
+}
+
+
+/** Opcode 0xf2 0x0f 0x59 - mulsd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_mulsd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, MULSD, mulsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_mulsd_u128_r64);
+}
+
+
+/** Opcode      0x0f 0x5a - cvtps2pd Vpd, Wps */
+FNIEMOP_DEF(iemOp_cvtps2pd_Vpd_Wps)
+{
+    IEMOP_MNEMONIC2(RM, CVTPS2PD, cvtps2pd, Vpd, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtps2pd_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x5a - cvtpd2ps Vps, Wpd */
+FNIEMOP_DEF(iemOp_cvtpd2ps_Vps_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, CVTPD2PS, cvtpd2ps, Vps, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtpd2ps_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0x5a - cvtss2sd Vsd, Wss */
+FNIEMOP_DEF(iemOp_cvtss2sd_Vsd_Wss)
+{
+    IEMOP_MNEMONIC2(RM, CVTSS2SD, cvtss2sd, Vsd, Wss, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_cvtss2sd_u128_r32);
+}
+
+
+/** Opcode 0xf2 0x0f 0x5a - cvtsd2ss Vss, Wsd */
+FNIEMOP_DEF(iemOp_cvtsd2ss_Vss_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, CVTSD2SS, cvtsd2ss, Vss, Wsd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_cvtsd2ss_u128_r64);
+}
+
+
+/** Opcode      0x0f 0x5b - cvtdq2ps Vps, Wdq */
+FNIEMOP_DEF(iemOp_cvtdq2ps_Vps_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, CVTDQ2PS, cvtdq2ps, Vps, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtdq2ps_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x5b - cvtps2dq Vdq, Wps */
+FNIEMOP_DEF(iemOp_cvtps2dq_Vdq_Wps)
+{
+    IEMOP_MNEMONIC2(RM, CVTPS2DQ, cvtps2dq, Vdq, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtps2dq_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0x5b - cvttps2dq Vdq, Wps */
+FNIEMOP_DEF(iemOp_cvttps2dq_Vdq_Wps)
+{
+    IEMOP_MNEMONIC2(RM, CVTTPS2DQ, cvttps2dq, Vdq, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvttps2dq_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0x5b - invalid */
+
+
+/** Opcode      0x0f 0x5c - subps Vps, Wps */
+FNIEMOP_DEF(iemOp_subps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, SUBPS, subps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_subps_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x5c - subpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_subpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, SUBPD, subpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_subpd_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0x5c - subss Vss, Wss */
+FNIEMOP_DEF(iemOp_subss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, SUBSS, subss, Vss, Wss, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_subss_u128_r32);
+}
+
+
+/** Opcode 0xf2 0x0f 0x5c - subsd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_subsd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, SUBSD, subsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_subsd_u128_r64);
+}
+
+
+/** Opcode      0x0f 0x5d - minps Vps, Wps */
+FNIEMOP_DEF(iemOp_minps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, MINPS, minps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_minps_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x5d - minpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_minpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, MINPD, minpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_minpd_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0x5d - minss Vss, Wss */
+FNIEMOP_DEF(iemOp_minss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, MINSS, minss, Vss, Wss, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_minss_u128_r32);
+}
+
+
+/** Opcode 0xf2 0x0f 0x5d - minsd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_minsd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, MINSD, minsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_minsd_u128_r64);
+}
+
+
+/** Opcode      0x0f 0x5e - divps Vps, Wps */
+FNIEMOP_DEF(iemOp_divps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, DIVPS, divps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_divps_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x5e - divpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_divpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, DIVPD, divpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_divpd_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0x5e - divss Vss, Wss */
+FNIEMOP_DEF(iemOp_divss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, DIVSS, divss, Vss, Wss, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_divss_u128_r32);
+}
+
+
+/** Opcode 0xf2 0x0f 0x5e - divsd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_divsd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, DIVSD, divsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_divsd_u128_r64);
+}
+
+
+/** Opcode      0x0f 0x5f - maxps Vps, Wps */
+FNIEMOP_DEF(iemOp_maxps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, MAXPS, maxps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullFull_To_Full, iemAImpl_maxps_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x5f - maxpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_maxpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, MAXPD, maxpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_maxpd_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0x5f - maxss Vss, Wss */
+FNIEMOP_DEF(iemOp_maxss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, MAXSS, maxss, Vss, Wss, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSseFp_FullR32_To_Full, iemAImpl_maxss_u128_r32);
+}
+
+
+/** Opcode 0xf2 0x0f 0x5f - maxsd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_maxsd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, MAXSD, maxsd, Vsd, Wsd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullR64_To_Full, iemAImpl_maxsd_u128_r64);
+}
+
+
+/** Opcode      0x0f 0x60 - punpcklbw Pq, Qd */
+FNIEMOP_DEF(iemOp_punpcklbw_Pq_Qd)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKLBW, punpcklbw, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_LowLow_To_Full, iemAImpl_punpcklbw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x60 - punpcklbw Vx, W */
+FNIEMOP_DEF(iemOp_punpcklbw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKLBW, punpcklbw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_LowLow_To_Full, iemAImpl_punpcklbw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x60 - invalid */
+
+
+/** Opcode      0x0f 0x61 - punpcklwd Pq, Qd */
+FNIEMOP_DEF(iemOp_punpcklwd_Pq_Qd)
+{
+    /** @todo AMD mark the MMX version as 3DNow!. Intel says MMX CPUID req. */
+    IEMOP_MNEMONIC2(RM, PUNPCKLWD, punpcklwd, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_LowLow_To_Full, iemAImpl_punpcklwd_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x61 - punpcklwd Vx, Wx */
+FNIEMOP_DEF(iemOp_punpcklwd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKLWD, punpcklwd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_LowLow_To_Full, iemAImpl_punpcklwd_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x61 - invalid */
+
+
+/** Opcode      0x0f 0x62 - punpckldq Pq, Qd */
+FNIEMOP_DEF(iemOp_punpckldq_Pq_Qd)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKLDQ, punpckldq, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_LowLow_To_Full, iemAImpl_punpckldq_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x62 - punpckldq Vx, Wx */
+FNIEMOP_DEF(iemOp_punpckldq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKLDQ, punpckldq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_LowLow_To_Full, iemAImpl_punpckldq_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x62 - invalid */
+
+
+
+/** Opcode      0x0f 0x63 - packsswb Pq, Qq */
+FNIEMOP_DEF(iemOp_packsswb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PACKSSWB, packsswb, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_packsswb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x63 - packsswb Vx, Wx */
+FNIEMOP_DEF(iemOp_packsswb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PACKSSWB, packsswb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_packsswb_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x63 - invalid */
+
+
+/** Opcode      0x0f 0x64 - pcmpgtb Pq, Qq */
+FNIEMOP_DEF(iemOp_pcmpgtb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PCMPGTB, pcmpgtb, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpgtb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x64 - pcmpgtb Vx, Wx */
+FNIEMOP_DEF(iemOp_pcmpgtb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PCMPGTB, pcmpgtb, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpgtb_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x64 - invalid */
+
+
+/** Opcode      0x0f 0x65 - pcmpgtw Pq, Qq */
+FNIEMOP_DEF(iemOp_pcmpgtw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PCMPGTW, pcmpgtw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpgtw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x65 - pcmpgtw Vx, Wx */
+FNIEMOP_DEF(iemOp_pcmpgtw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PCMPGTW, pcmpgtw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpgtw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x65 - invalid */
+
+
+/** Opcode      0x0f 0x66 - pcmpgtd Pq, Qq */
+FNIEMOP_DEF(iemOp_pcmpgtd_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PCMPGTD, pcmpgtd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpgtd_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x66 - pcmpgtd Vx, Wx */
+FNIEMOP_DEF(iemOp_pcmpgtd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PCMPGTD, pcmpgtd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpgtd_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x66 - invalid */
+
+
+/** Opcode      0x0f 0x67 - packuswb Pq, Qq */
+FNIEMOP_DEF(iemOp_packuswb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PACKUSWB, packuswb, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_packuswb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x67 - packuswb Vx, Wx */
+FNIEMOP_DEF(iemOp_packuswb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PACKUSWB, packuswb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_packuswb_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x67 - invalid */
+
+
+/** Opcode      0x0f 0x68 - punpckhbw Pq, Qq
+ * @note Intel and AMD both uses Qd for the second parameter, however they
+ *       both list it as a mmX/mem64 operand and intel describes it as being
+ *       loaded as a qword, so it should be Qq, shouldn't it? */
+FNIEMOP_DEF(iemOp_punpckhbw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKHBW, punpckhbw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_HighHigh_To_Full, iemAImpl_punpckhbw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x68 - punpckhbw Vx, Wx */
+FNIEMOP_DEF(iemOp_punpckhbw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKHBW, punpckhbw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_HighHigh_To_Full, iemAImpl_punpckhbw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x68 - invalid */
+
+
+/** Opcode      0x0f 0x69 - punpckhwd Pq, Qq
+ * @note Intel and AMD both uses Qd for the second parameter, however they
+ *       both list it as a mmX/mem64 operand and intel describes it as being
+ *       loaded as a qword, so it should be Qq, shouldn't it? */
+FNIEMOP_DEF(iemOp_punpckhwd_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKHWD, punpckhwd, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_HighHigh_To_Full, iemAImpl_punpckhwd_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x69 - punpckhwd Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_punpckhwd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKHWD, punpckhwd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_HighHigh_To_Full, iemAImpl_punpckhwd_u128);
+
+}
+
+
+/*  Opcode 0xf3 0x0f 0x69 - invalid */
+
+
+/** Opcode      0x0f 0x6a - punpckhdq Pq, Qq
+ * @note Intel and AMD both uses Qd for the second parameter, however they
+ *       both list it as a mmX/mem64 operand and intel describes it as being
+ *       loaded as a qword, so it should be Qq, shouldn't it? */
+FNIEMOP_DEF(iemOp_punpckhdq_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKHDQ, punpckhdq, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_HighHigh_To_Full, iemAImpl_punpckhdq_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x6a - punpckhdq Vx, Wx */
+FNIEMOP_DEF(iemOp_punpckhdq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKHDQ, punpckhdq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_HighHigh_To_Full, iemAImpl_punpckhdq_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x6a - invalid */
+
+
+/** Opcode      0x0f 0x6b - packssdw Pq, Qd */
+FNIEMOP_DEF(iemOp_packssdw_Pq_Qd)
+{
+    IEMOP_MNEMONIC2(RM, PACKSSDW, packssdw, Pq, Qd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_packssdw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x6b - packssdw Vx, Wx */
+FNIEMOP_DEF(iemOp_packssdw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PACKSSDW, packssdw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_packssdw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x6b - invalid */
+
+
+/*  Opcode      0x0f 0x6c - invalid */
+
+
+/** Opcode 0x66 0x0f 0x6c - punpcklqdq Vx, Wx */
+FNIEMOP_DEF(iemOp_punpcklqdq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKLQDQ, punpcklqdq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_LowLow_To_Full, iemAImpl_punpcklqdq_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x6c - invalid */
+/*  Opcode 0xf2 0x0f 0x6c - invalid */
+
+
+/*  Opcode      0x0f 0x6d - invalid */
+
+
+/** Opcode 0x66 0x0f 0x6d - punpckhqdq Vx, Wx */
+FNIEMOP_DEF(iemOp_punpckhqdq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PUNPCKHQDQ, punpckhqdq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_HighHigh_To_Full, iemAImpl_punpckhqdq_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x6d - invalid */
+
+
+FNIEMOP_DEF(iemOp_movd_q_Pd_Ey)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x6e
+         * @opcodesub   rex.w=1
+         * @oppfx       none
+         * @opcpuid     mmx
+         * @opgroup     og_mmx_datamove
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2   ftw=0xff
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42 ftw=0xff
+         */
+        IEMOP_MNEMONIC2(RM, MOVQ, movq, Pq_WO, Eq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* MMX, greg64 */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* MMX, [mem64] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x6e
+         * @opcodesub   rex.w=0
+         * @oppfx       none
+         * @opcpuid     mmx
+         * @opgroup     og_mmx_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_movd_q_Pd_Ey
+         * @optest      op1=1 op2=2   -> op1=2   ftw=0xff
+         * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+         */
+        IEMOP_MNEMONIC2(RM, MOVD, movd, PdZx_WO, Ed, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* MMX, greg32 */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_MREG_U32_ZX_U64(IEM_GET_MODRM_REG_8(bRm), u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* MMX, [mem32] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_MREG_U32_ZX_U64(IEM_GET_MODRM_REG_8(bRm), u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+FNIEMOP_DEF(iemOp_movd_q_Vy_Ey)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x6e
+         * @opcodesub   rex.w=1
+         * @oppfx       0x66
+         * @opcpuid     sse2
+         * @opgroup     og_sse2_simdint_datamove
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(RM, MOVQ, movq, VqZx_WO, Eq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* XMM, greg64 */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem64] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x6e
+         * @opcodesub   rex.w=0
+         * @oppfx       0x66
+         * @opcpuid     sse2
+         * @opgroup     og_sse2_simdint_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_movd_q_Vy_Ey
+         * @optest      op1=1 op2=2   -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(RM, MOVD, movd, VdZx_WO, Ed, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* XMM, greg32 */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_XREG_U32_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem32] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_XREG_U32_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+/*  Opcode 0xf3 0x0f 0x6e - invalid */
+
+
+/**
+ * @opcode      0x6f
+ * @oppfx       none
+ * @opcpuid     mmx
+ * @opgroup     og_mmx_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2   -> op1=2   ftw=0xff
+ * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+ */
+FNIEMOP_DEF(iemOp_movq_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, MOVD, movd, Pq_WO, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_FETCH_MREG_U64(u64Tmp, IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t, u64Tmp);
+        IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), u64Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/**
+ * @opcode      0x6f
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_simdint_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movdqa_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, MOVDQA, movdqa, Vdq_WO, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm),
+                              IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/**
+ * @opcode      0x6f
+ * @oppfx       0xf3
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_simdint_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movdqu_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(RM, MOVDQU, movdqu, Vdq_WO, Wdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm),
+                              IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_MEM_U128(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode      0x0f 0x70 - pshufw Pq, Qq, Ib */
+FNIEMOP_DEF(iemOp_pshufw_Pq_Qq_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PSHUFW, pshufw, Pq, Qq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG(uint64_t const *,    pSrc, 1);
+        IEM_MC_ARG_CONST(uint8_t,       bImmArg, /*=*/ bImm, 2);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst,       IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_REF_MREG_U64_CONST(pSrc, IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pshufw_u64, pDst, pSrc, bImmArg);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(uint64_t *,                  pDst,       0);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_ARG_LOCAL_REF(uint64_t const *,  pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pshufw_u64, pDst, pSrc, bImmArg);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common worker for SSE2 instructions on the forms:
+ *      pshufd      xmm1, xmm2/mem128, imm8
+ *      pshufhw     xmm1, xmm2/mem128, imm8
+ *      pshuflw     xmm1, xmm2/mem128, imm8
+ *
+ * Proper alignment of the 128-bit operand is enforced.
+ * Exceptions type 4. SSE2 cpuid checks.
+ */
+FNIEMOP_DEF_1(iemOpCommonSse2_pshufXX_Vx_Wx_Ib, PFNIEMAIMPLMEDIAPSHUFU128, pfnWorker)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,         puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,        puSrc, 1);
+        IEM_MC_ARG_CONST(uint8_t,       bImmArg, /*=*/ bImm, 2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,       IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(pfnWorker, puDst, puSrc, bImmArg);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(pfnWorker, puDst, puSrc, bImmArg);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0x70 - pshufd Vx, Wx, Ib */
+FNIEMOP_DEF(iemOp_pshufd_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PSHUFD, pshufd, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_pshufXX_Vx_Wx_Ib, iemAImpl_pshufd_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0x70 - pshufhw Vx, Wx, Ib */
+FNIEMOP_DEF(iemOp_pshufhw_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PSHUFHW, pshufhw, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_pshufXX_Vx_Wx_Ib, iemAImpl_pshufhw_u128);
+}
+
+
+/** Opcode 0xf2 0x0f 0x70 - pshuflw Vx, Wx, Ib */
+FNIEMOP_DEF(iemOp_pshuflw_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PSHUFLW, pshuflw, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_pshufXX_Vx_Wx_Ib, iemAImpl_pshuflw_u128);
+}
+
+
+/**
+ * Common worker for MMX instructions of the form:
+ *      psrlw       mm, imm8
+ *      psraw       mm, imm8
+ *      psllw       mm, imm8
+ *      psrld       mm, imm8
+ *      psrad       mm, imm8
+ *      pslld       mm, imm8
+ *      psrlq       mm, imm8
+ *      psllq       mm, imm8
+ *
+ */
+FNIEMOP_DEF_2(iemOpCommonMmx_Shift_Imm, uint8_t, bRm, FNIEMAIMPLMEDIAPSHIFTU64, pfnU64)
+{
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, immediate.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,          pDst, 0);
+        IEM_MC_ARG_CONST(uint8_t,       bShiftArg, /*=*/ bImm, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_MREG_U64(pDst,       IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU64, pDst, bShiftArg);
+        IEM_MC_MODIFIED_MREG_BY_REF(pDst);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory not supported.
+         */
+        /// @todo Caller already enforced register mode?!
+        AssertFailedReturn(VINF_SUCCESS);
+    }
+}
+
+
+/**
+ * Common worker for SSE2 instructions of the form:
+ *      psrlw       xmm, imm8
+ *      psraw       xmm, imm8
+ *      psllw       xmm, imm8
+ *      psrld       xmm, imm8
+ *      psrad       xmm, imm8
+ *      pslld       xmm, imm8
+ *      psrlq       xmm, imm8
+ *      psllq       xmm, imm8
+ *
+ */
+FNIEMOP_DEF_2(iemOpCommonSse2_Shift_Imm, uint8_t, bRm, FNIEMAIMPLMEDIAPSHIFTU128, pfnU128)
+{
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, immediate.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,         pDst, 0);
+        IEM_MC_ARG_CONST(uint8_t,       bShiftArg, /*=*/ bImm, 1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(pfnU128, pDst, bShiftArg);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        /// @todo Caller already enforced register mode?!
+        AssertFailedReturn(VINF_SUCCESS);
+    }
+}
+
+
+/** Opcode 0x0f 0x71 11/2 - psrlw Nq, Ib */
+FNIEMOPRM_DEF(iemOp_Grp12_psrlw_Nq_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRLW, psrlw, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psrlw_imm_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x71 11/2. */
+FNIEMOPRM_DEF(iemOp_Grp12_psrlw_Ux_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRLW, psrlw, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psrlw_imm_u128);
+}
+
+
+/** Opcode 0x0f 0x71 11/4. */
+FNIEMOPRM_DEF(iemOp_Grp12_psraw_Nq_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRAW, psraw, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psraw_imm_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x71 11/4. */
+FNIEMOPRM_DEF(iemOp_Grp12_psraw_Ux_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRAW, psraw, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psraw_imm_u128);
+}
+
+
+/** Opcode 0x0f 0x71 11/6. */
+FNIEMOPRM_DEF(iemOp_Grp12_psllw_Nq_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSLLW, psllw, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psllw_imm_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x71 11/6. */
+FNIEMOPRM_DEF(iemOp_Grp12_psllw_Ux_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSLLW, psllw, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psllw_imm_u128);
+}
+
+
+/**
+ * Group 12 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup12RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_Grp12_psrlw_Nq_Ib,   iemOp_Grp12_psrlw_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /4 */ iemOp_Grp12_psraw_Nq_Ib,   iemOp_Grp12_psraw_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_Grp12_psllw_Nq_Ib,   iemOp_Grp12_psllw_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8)
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup12RegReg) == 8*4);
+
+
+/** Opcode 0x0f 0x71. */
+FNIEMOP_DEF(iemOp_Grp12)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnGroup12RegReg[  IEM_GET_MODRM_REG_8(bRm) * 4
+                                                  + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/** Opcode 0x0f 0x72 11/2. */
+FNIEMOPRM_DEF(iemOp_Grp13_psrld_Nq_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRLD, psrld, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psrld_imm_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x72 11/2. */
+FNIEMOPRM_DEF(iemOp_Grp13_psrld_Ux_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRLD, psrld, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psrld_imm_u128);
+}
+
+
+/** Opcode 0x0f 0x72 11/4. */
+FNIEMOPRM_DEF(iemOp_Grp13_psrad_Nq_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRAD, psrad, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psrad_imm_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x72 11/4. */
+FNIEMOPRM_DEF(iemOp_Grp13_psrad_Ux_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRAD, psrad, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psrad_imm_u128);
+}
+
+
+/** Opcode 0x0f 0x72 11/6. */
+FNIEMOPRM_DEF(iemOp_Grp13_pslld_Nq_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSLLD, pslld, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_pslld_imm_u64);
+}
+
+/** Opcode 0x66 0x0f 0x72 11/6. */
+FNIEMOPRM_DEF(iemOp_Grp13_pslld_Ux_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSLLD, pslld, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_pslld_imm_u128);
+}
+
+
+/**
+ * Group 13 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup13RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_Grp13_psrld_Nq_Ib,   iemOp_Grp13_psrld_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /4 */ iemOp_Grp13_psrad_Nq_Ib,   iemOp_Grp13_psrad_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_Grp13_pslld_Nq_Ib,   iemOp_Grp13_pslld_Ux_Ib,    iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8)
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup13RegReg) == 8*4);
+
+/** Opcode 0x0f 0x72. */
+FNIEMOP_DEF(iemOp_Grp13)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnGroup13RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4
+                                                  + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/** Opcode 0x0f 0x73 11/2. */
+FNIEMOPRM_DEF(iemOp_Grp14_psrlq_Nq_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRLQ, psrlq, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psrlq_imm_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x73 11/2. */
+FNIEMOPRM_DEF(iemOp_Grp14_psrlq_Ux_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRLQ, psrlq, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psrlq_imm_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x73 11/3. */
+FNIEMOPRM_DEF(iemOp_Grp14_psrldq_Ux_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSRLDQ, psrldq, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psrldq_imm_u128);
+}
+
+
+/** Opcode 0x0f 0x73 11/6. */
+FNIEMOPRM_DEF(iemOp_Grp14_psllq_Nq_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSLLQ, psllq, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_Shift_Imm, bRm, iemAImpl_psllq_imm_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x73 11/6. */
+FNIEMOPRM_DEF(iemOp_Grp14_psllq_Ux_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSLLQ, psllq, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_psllq_imm_u128);
+}
+
+
+/** Opcode 0x66 0x0f 0x73 11/7. */
+FNIEMOPRM_DEF(iemOp_Grp14_pslldq_Ux_Ib)
+{
+//    IEMOP_MNEMONIC2(RI, PSLLDQ, pslldq, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_2(iemOpCommonSse2_Shift_Imm, bRm, iemAImpl_pslldq_imm_u128);
+}
+
+/**
+ * Group 14 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup14RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_Grp14_psrlq_Nq_Ib,     iemOp_Grp14_psrlq_Ux_Ib,  iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ iemOp_InvalidWithRMNeedImm8, iemOp_Grp14_psrldq_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /4 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_Grp14_psllq_Nq_Ib,     iemOp_Grp14_psllq_Ux_Ib,  iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ iemOp_InvalidWithRMNeedImm8, iemOp_Grp14_pslldq_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup14RegReg) == 8*4);
+
+
+/** Opcode 0x0f 0x73. */
+FNIEMOP_DEF(iemOp_Grp14)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnGroup14RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4
+                                                  + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/** Opcode      0x0f 0x74 - pcmpeqb Pq, Qq */
+FNIEMOP_DEF(iemOp_pcmpeqb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PCMPEQB, pcmpeqb, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpeqb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x74 - pcmpeqb Vx, Wx */
+FNIEMOP_DEF(iemOp_pcmpeqb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PCMPEQB, pcmpeqb, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpeqb_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x74 - invalid */
+/*  Opcode 0xf2 0x0f 0x74 - invalid */
+
+
+/** Opcode      0x0f 0x75 - pcmpeqw Pq, Qq */
+FNIEMOP_DEF(iemOp_pcmpeqw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PCMPEQW, pcmpeqw, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpeqw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x75 - pcmpeqw Vx, Wx */
+FNIEMOP_DEF(iemOp_pcmpeqw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PCMPEQW, pcmpeqw, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpeqw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x75 - invalid */
+/*  Opcode 0xf2 0x0f 0x75 - invalid */
+
+
+/** Opcode      0x0f 0x76 - pcmpeqd Pq, Qq */
+FNIEMOP_DEF(iemOp_pcmpeqd_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PCMPEQD, pcmpeqd, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pcmpeqd_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0x76 - pcmpeqd Vx, Wx */
+FNIEMOP_DEF(iemOp_pcmpeqd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PCMPEQD, pcmpeqd, Vx, Wx, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pcmpeqd_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x76 - invalid */
+/*  Opcode 0xf2 0x0f 0x76 - invalid */
+
+
+/** Opcode      0x0f 0x77 - emms (vex has vzeroall and vzeroupper here) */
+FNIEMOP_DEF(iemOp_emms)
+{
+    IEMOP_MNEMONIC(emms, "emms");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+    IEM_MC_BEGIN(0,0);
+    IEM_MC_MAYBE_RAISE_DEVICE_NOT_AVAILABLE();
+    IEM_MC_MAYBE_RAISE_FPU_XCPT();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_FPU_FROM_MMX_MODE();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+/*  Opcode 0x66 0x0f 0x77 - invalid */
+/*  Opcode 0xf3 0x0f 0x77 - invalid */
+/*  Opcode 0xf2 0x0f 0x77 - invalid */
+
+/** Opcode      0x0f 0x78 - VMREAD Ey, Gy */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_vmread_Ey_Gy)
+{
+    IEMOP_MNEMONIC(vmread, "vmread Ey,Gy");
+    IEMOP_HLP_IN_VMX_OPERATION("vmread", kVmxVDiag_Vmread);
+    IEMOP_HLP_VMX_INSTR("vmread", kVmxVDiag_Vmread);
+    IEMMODE const enmEffOpSize = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? IEMMODE_64BIT : IEMMODE_32BIT;
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+        if (enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint64_t *, pu64Dst, 0);
+            IEM_MC_ARG(uint64_t,   u64Enc,  1);
+            IEM_MC_FETCH_GREG_U64(u64Enc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_CIMPL_2(iemCImpl_vmread_reg64, pu64Dst, u64Enc);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint32_t *, pu32Dst, 0);
+            IEM_MC_ARG(uint32_t,   u32Enc,  1);
+            IEM_MC_FETCH_GREG_U32(u32Enc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_CIMPL_2(iemCImpl_vmread_reg32, pu32Dst, u32Enc);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        if (enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,       iEffSeg,                                          0);
+            IEM_MC_ARG(RTGCPTR,       GCPtrVal,                                         1);
+            IEM_MC_ARG(uint64_t,      u64Enc,                                           2);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+            IEM_MC_FETCH_GREG_U64(u64Enc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_vmread_mem_reg64, iEffSeg, GCPtrVal, u64Enc);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,       iEffSeg,                                          0);
+            IEM_MC_ARG(RTGCPTR,       GCPtrVal,                                         1);
+            IEM_MC_ARG(uint32_t,      u32Enc,                                           2);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+            IEM_MC_FETCH_GREG_U32(u32Enc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_vmread_mem_reg32, iEffSeg, GCPtrVal, u32Enc);
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_STUB(iemOp_vmread_Ey_Gy);
+#endif
+
+/*  Opcode 0x66 0x0f 0x78 - AMD Group 17 */
+FNIEMOP_STUB(iemOp_AmdGrp17);
+/*  Opcode 0xf3 0x0f 0x78 - invalid */
+/*  Opcode 0xf2 0x0f 0x78 - invalid */
+
+/** Opcode      0x0f 0x79 - VMWRITE Gy, Ey */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF(iemOp_vmwrite_Gy_Ey)
+{
+    IEMOP_MNEMONIC(vmwrite, "vmwrite Gy,Ey");
+    IEMOP_HLP_IN_VMX_OPERATION("vmwrite", kVmxVDiag_Vmwrite);
+    IEMOP_HLP_VMX_INSTR("vmwrite", kVmxVDiag_Vmwrite);
+    IEMMODE const enmEffOpSize = pVCpu->iem.s.enmCpuMode == IEMMODE_64BIT ? IEMMODE_64BIT : IEMMODE_32BIT;
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+        if (enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint64_t, u64Val, 0);
+            IEM_MC_ARG(uint64_t, u64Enc, 1);
+            IEM_MC_FETCH_GREG_U64(u64Val, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_FETCH_GREG_U64(u64Enc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALL_CIMPL_2(iemCImpl_vmwrite_reg, u64Val, u64Enc);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint32_t, u32Val, 0);
+            IEM_MC_ARG(uint32_t, u32Enc, 1);
+            IEM_MC_FETCH_GREG_U32(u32Val, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_FETCH_GREG_U32(u32Enc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALL_CIMPL_2(iemCImpl_vmwrite_reg, u32Val, u32Enc);
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,       iEffSeg,                                          0);
+            IEM_MC_ARG(RTGCPTR,       GCPtrVal,                                         1);
+            IEM_MC_ARG(uint64_t,      u64Enc,                                           2);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+            IEM_MC_FETCH_GREG_U64(u64Enc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_vmwrite_mem, iEffSeg, GCPtrVal, u64Enc);
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint8_t,       iEffSeg,                                          0);
+            IEM_MC_ARG(RTGCPTR,       GCPtrVal,                                         1);
+            IEM_MC_ARG(uint32_t,      u32Enc,                                           2);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrVal, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+            IEM_MC_FETCH_GREG_U32(u32Enc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+            IEM_MC_CALL_CIMPL_3(iemCImpl_vmwrite_mem, iEffSeg, GCPtrVal, u32Enc);
+            IEM_MC_END();
+        }
+    }
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_STUB(iemOp_vmwrite_Gy_Ey);
+#endif
+/*  Opcode 0x66 0x0f 0x79 - invalid */
+/*  Opcode 0xf3 0x0f 0x79 - invalid */
+/*  Opcode 0xf2 0x0f 0x79 - invalid */
+
+/*  Opcode      0x0f 0x7a - invalid */
+/*  Opcode 0x66 0x0f 0x7a - invalid */
+/*  Opcode 0xf3 0x0f 0x7a - invalid */
+/*  Opcode 0xf2 0x0f 0x7a - invalid */
+
+/*  Opcode      0x0f 0x7b - invalid */
+/*  Opcode 0x66 0x0f 0x7b - invalid */
+/*  Opcode 0xf3 0x0f 0x7b - invalid */
+/*  Opcode 0xf2 0x0f 0x7b - invalid */
+
+/*  Opcode      0x0f 0x7c - invalid */
+
+
+/** Opcode 0x66 0x0f 0x7c - haddpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_haddpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, HADDPD, haddpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_haddpd_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x7c - invalid */
+
+
+/** Opcode 0xf2 0x0f 0x7c - haddps Vps, Wps */
+FNIEMOP_DEF(iemOp_haddps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, HADDPS, haddps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_haddps_u128);
+}
+
+
+/*  Opcode      0x0f 0x7d - invalid */
+
+
+/** Opcode 0x66 0x0f 0x7d - hsubpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_hsubpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, HSUBPD, hsubpd, Vpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_hsubpd_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0x7d - invalid */
+
+
+/** Opcode 0xf2 0x0f 0x7d - hsubps Vps, Wps */
+FNIEMOP_DEF(iemOp_hsubps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, HSUBPS, hsubps, Vps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_hsubps_u128);
+}
+
+
+/** Opcode      0x0f 0x7e - movd_q Ey, Pd */
+FNIEMOP_DEF(iemOp_movd_q_Ey_Pd)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x7e
+         * @opcodesub   rex.w=1
+         * @oppfx       none
+         * @opcpuid     mmx
+         * @opgroup     og_mmx_datamove
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2   ftw=0xff
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42 ftw=0xff
+         */
+        IEMOP_MNEMONIC2(MR, MOVQ, movq, Eq_WO, Pq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg64, MMX */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_FETCH_MREG_U64(u64Tmp, IEM_GET_MODRM_REG_8(bRm));
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem64], MMX */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_FETCH_MREG_U64(u64Tmp, IEM_GET_MODRM_REG_8(bRm));
+            IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x7e
+         * @opcodesub   rex.w=0
+         * @oppfx       none
+         * @opcpuid     mmx
+         * @opgroup     og_mmx_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_movd_q_Pd_Ey
+         * @optest      op1=1 op2=2   -> op1=2   ftw=0xff
+         * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+         */
+        IEMOP_MNEMONIC2(MR, MOVD, movd, Ed_WO, Pd, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg32, MMX */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_FETCH_MREG_U32(u32Tmp, IEM_GET_MODRM_REG_8(bRm));
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem32], MMX */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+            IEM_MC_FPU_TO_MMX_MODE();
+
+            IEM_MC_FETCH_MREG_U32(u32Tmp, IEM_GET_MODRM_REG_8(bRm));
+            IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+FNIEMOP_DEF(iemOp_movd_q_Ey_Vy)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x7e
+         * @opcodesub   rex.w=1
+         * @oppfx       0x66
+         * @opcpuid     sse2
+         * @opgroup     og_sse2_simdint_datamove
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(MR, MOVQ, movq, Eq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg64, XMM */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+            IEM_MC_FETCH_XREG_U64(u64Tmp, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/);
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem64], XMM */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+            IEM_MC_FETCH_XREG_U64(u64Tmp, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/);
+            IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x7e
+         * @opcodesub   rex.w=0
+         * @oppfx       0x66
+         * @opcpuid     sse2
+         * @opgroup     og_sse2_simdint_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_movd_q_Vy_Ey
+         * @optest      op1=1 op2=2   -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(MR, MOVD, movd, Ed_WO, Vd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OZ_PFX);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg32, XMM */
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+            IEM_MC_FETCH_XREG_U32(u32Tmp, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/);
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem32], XMM */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+            IEM_MC_FETCH_XREG_U32(u32Tmp, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/);
+            IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+/**
+ * @opcode      0x7e
+ * @oppfx       0xf3
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  none
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movq_Vq_Wq)
+{
+    IEMOP_MNEMONIC2(RM, MOVQ, movq, VqZx_WO, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM128, XMM64.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM128, [mem64].
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/*  Opcode 0xf2 0x0f 0x7e - invalid */
+
+
+/** Opcode      0x0f 0x7f - movq Qq, Pq */
+FNIEMOP_DEF(iemOp_movq_Qq_Pq)
+{
+    IEMOP_MNEMONIC2(MR, MOVQ, movq, Qq_WO, Pq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_IGNORES_REXW);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * MMX, MMX.
+         */
+        /** @todo testcase: REX.B / REX.R and MMX register indexing. Ignored? */
+        /** @todo testcase: REX.B / REX.R and segment register indexing. Ignored? */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t, u64Tmp);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_FETCH_MREG_U64(u64Tmp, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_RM_8(bRm), u64Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * [mem64], MMX.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t, u64Tmp);
+        IEM_MC_LOCAL(RTGCPTR,  GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_FETCH_MREG_U64(u64Tmp, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/** Opcode 0x66 0x0f 0x7f - movdqa Wx,Vx */
+FNIEMOP_DEF(iemOp_movdqa_Wx_Vx)
+{
+    IEMOP_MNEMONIC2(MR, MOVDQA, movdqa, Wx_WO, Vx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm),
+                              IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * [mem128], XMM.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(u128Tmp, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/** Opcode 0xf3 0x0f 0x7f - movdqu Wx,Vx */
+FNIEMOP_DEF(iemOp_movdqu_Wx_Vx)
+{
+    IEMOP_MNEMONIC2(MR, MOVDQU, movdqu, Wx_WO, Vx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_COPY_XREG_U128(IEM_GET_MODRM_RM(pVCpu, bRm),
+                              IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * [mem128], XMM.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(u128Tmp, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/*  Opcode 0xf2 0x0f 0x7f - invalid */
+
+
+
+/** Opcode 0x0f 0x80. */
+FNIEMOP_DEF(iemOp_jo_Jv)
+{
+    IEMOP_MNEMONIC(jo_Jv, "jo  Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x81. */
+FNIEMOP_DEF(iemOp_jno_Jv)
+{
+    IEMOP_MNEMONIC(jno_Jv, "jno Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x82. */
+FNIEMOP_DEF(iemOp_jc_Jv)
+{
+    IEMOP_MNEMONIC(jc_Jv, "jc/jb/jnae Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x83. */
+FNIEMOP_DEF(iemOp_jnc_Jv)
+{
+    IEMOP_MNEMONIC(jnc_Jv, "jnc/jnb/jae Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x84. */
+FNIEMOP_DEF(iemOp_je_Jv)
+{
+    IEMOP_MNEMONIC(je_Jv, "je/jz Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x85. */
+FNIEMOP_DEF(iemOp_jne_Jv)
+{
+    IEMOP_MNEMONIC(jne_Jv, "jne/jnz Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x86. */
+FNIEMOP_DEF(iemOp_jbe_Jv)
+{
+    IEMOP_MNEMONIC(jbe_Jv, "jbe/jna Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x87. */
+FNIEMOP_DEF(iemOp_jnbe_Jv)
+{
+    IEMOP_MNEMONIC(ja_Jv, "jnbe/ja Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x88. */
+FNIEMOP_DEF(iemOp_js_Jv)
+{
+    IEMOP_MNEMONIC(js_Jv, "js  Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x89. */
+FNIEMOP_DEF(iemOp_jns_Jv)
+{
+    IEMOP_MNEMONIC(jns_Jv, "jns Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x8a. */
+FNIEMOP_DEF(iemOp_jp_Jv)
+{
+    IEMOP_MNEMONIC(jp_Jv, "jp  Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x8b. */
+FNIEMOP_DEF(iemOp_jnp_Jv)
+{
+    IEMOP_MNEMONIC(jnp_Jv, "jnp Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x8c. */
+FNIEMOP_DEF(iemOp_jl_Jv)
+{
+    IEMOP_MNEMONIC(jl_Jv, "jl/jnge Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x8d. */
+FNIEMOP_DEF(iemOp_jnl_Jv)
+{
+    IEMOP_MNEMONIC(jge_Jv, "jnl/jge Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x8e. */
+FNIEMOP_DEF(iemOp_jle_Jv)
+{
+    IEMOP_MNEMONIC(jle_Jv, "jle/jng Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ELSE() {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x8f. */
+FNIEMOP_DEF(iemOp_jnle_Jv)
+{
+    IEMOP_MNEMONIC(jg_Jv, "jnle/jg Jv");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE_AND_INTEL_IGNORES_OP_SIZE_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_16BIT)
+    {
+        int16_t i16Imm; IEM_OPCODE_GET_NEXT_S16(&i16Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S16_AND_FINISH(i16Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+    else
+    {
+        int32_t i32Imm; IEM_OPCODE_GET_NEXT_S32(&i32Imm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+        } IEM_MC_ELSE() {
+            IEM_MC_REL_JMP_S32_AND_FINISH(i32Imm);
+        } IEM_MC_ENDIF();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x90. */
+FNIEMOP_DEF(iemOp_seto_Eb)
+{
+    IEMOP_MNEMONIC(seto_Eb, "seto Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x91. */
+FNIEMOP_DEF(iemOp_setno_Eb)
+{
+    IEMOP_MNEMONIC(setno_Eb, "setno Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x92. */
+FNIEMOP_DEF(iemOp_setc_Eb)
+{
+    IEMOP_MNEMONIC(setc_Eb, "setc Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x93. */
+FNIEMOP_DEF(iemOp_setnc_Eb)
+{
+    IEMOP_MNEMONIC(setnc_Eb, "setnc Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_CF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x94. */
+FNIEMOP_DEF(iemOp_sete_Eb)
+{
+    IEMOP_MNEMONIC(sete_Eb, "sete Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x95. */
+FNIEMOP_DEF(iemOp_setne_Eb)
+{
+    IEMOP_MNEMONIC(setne_Eb, "setne Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x96. */
+FNIEMOP_DEF(iemOp_setbe_Eb)
+{
+    IEMOP_MNEMONIC(setbe_Eb, "setbe Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x97. */
+FNIEMOP_DEF(iemOp_setnbe_Eb)
+{
+    IEMOP_MNEMONIC(setnbe_Eb, "setnbe Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_ANY_BITS_SET(X86_EFL_CF | X86_EFL_ZF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x98. */
+FNIEMOP_DEF(iemOp_sets_Eb)
+{
+    IEMOP_MNEMONIC(sets_Eb, "sets Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x99. */
+FNIEMOP_DEF(iemOp_setns_Eb)
+{
+    IEMOP_MNEMONIC(setns_Eb, "setns Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_SF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x9a. */
+FNIEMOP_DEF(iemOp_setp_Eb)
+{
+    IEMOP_MNEMONIC(setp_Eb, "setp Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x9b. */
+FNIEMOP_DEF(iemOp_setnp_Eb)
+{
+    IEMOP_MNEMONIC(setnp_Eb, "setnp Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET(X86_EFL_PF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x9c. */
+FNIEMOP_DEF(iemOp_setl_Eb)
+{
+    IEMOP_MNEMONIC(setl_Eb, "setl Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x9d. */
+FNIEMOP_DEF(iemOp_setnl_Eb)
+{
+    IEMOP_MNEMONIC(setnl_Eb, "setnl Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BITS_NE(X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x9e. */
+FNIEMOP_DEF(iemOp_setle_Eb)
+{
+    IEMOP_MNEMONIC(setle_Eb, "setle Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0x9f. */
+FNIEMOP_DEF(iemOp_setnle_Eb)
+{
+    IEMOP_MNEMONIC(setnle_Eb, "setnle Eb");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Encoding test: Check if the 'reg' field is ignored or decoded in
+     *        any way. AMD says it's "unused", whatever that means.  We're
+     *        ignoring for now. */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register target */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_GREG_U8_CONST(IEM_GET_MODRM_RM(pVCpu, bRm), 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /* memory target */
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_IF_EFL_BIT_SET_OR_BITS_NE(X86_EFL_ZF, X86_EFL_SF, X86_EFL_OF) {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        } IEM_MC_ELSE() {
+            IEM_MC_STORE_MEM_U8_CONST(pVCpu->iem.s.iEffSeg, GCPtrEffDst, 1);
+        } IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Common 'push segment-register' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonPushSReg, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    Assert(iReg < X86_SREG_FS || pVCpu->iem.s.enmCpuMode != IEMMODE_64BIT);
+    IEMOP_HLP_DEFAULT_64BIT_OP_SIZE();
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint16_t, u16Value);
+            IEM_MC_FETCH_SREG_U16(u16Value, iReg);
+            IEM_MC_PUSH_U16(u16Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_FETCH_SREG_ZX_U32(u32Value, iReg);
+            IEM_MC_PUSH_U32_SREG(u32Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_FETCH_SREG_ZX_U64(u64Value, iReg);
+            IEM_MC_PUSH_U64(u64Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcode 0x0f 0xa0. */
+FNIEMOP_DEF(iemOp_push_fs)
+{
+    IEMOP_MNEMONIC(push_fs, "push fs");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_FS);
+}
+
+
+/** Opcode 0x0f 0xa1. */
+FNIEMOP_DEF(iemOp_pop_fs)
+{
+    IEMOP_MNEMONIC(pop_fs, "pop fs");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_FS, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/** Opcode 0x0f 0xa2. */
+FNIEMOP_DEF(iemOp_cpuid)
+{
+    IEMOP_MNEMONIC(cpuid, "cpuid");
+    IEMOP_HLP_MIN_486(); /* not all 486es. */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_cpuid);
+}
+
+
+/**
+ * Common worker for iemOp_bt_Ev_Gv, iemOp_btc_Ev_Gv, iemOp_btr_Ev_Gv and
+ * iemOp_bts_Ev_Gv.
+ */
+FNIEMOP_DEF_1(iemOpCommonBit_Ev_Gv, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register destination. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,        u16Src,                 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_AND_LOCAL_U16(u16Src, 0xf);
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,        u32Src,                 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_AND_LOCAL_U32(u32Src, 0x1f);
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,        u64Src,                 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                2);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_AND_LOCAL_U64(u64Src, 0x3f);
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory destination. */
+
+        uint32_t fAccess;
+        if (pImpl->pfnLockedU16)
+            fAccess = IEM_ACCESS_DATA_RW;
+        else /* BT */
+            fAccess = IEM_ACCESS_DATA_R;
+
+        /** @todo test negative bit offsets! */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint16_t *,              pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,                u16Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+                IEM_MC_LOCAL(int16_t,               i16AddrAdj);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_ASSIGN(i16AddrAdj, u16Src);
+                IEM_MC_AND_ARG_U16(u16Src, 0x0f);
+                IEM_MC_SAR_LOCAL_S16(i16AddrAdj, 4);
+                IEM_MC_SHL_LOCAL_S16(i16AddrAdj, 1);
+                IEM_MC_ADD_LOCAL_S16_TO_EFF_ADDR(GCPtrEffDst, i16AddrAdj);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+
+                IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint32_t *,              pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,                u32Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+                IEM_MC_LOCAL(int32_t,               i32AddrAdj);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_ASSIGN(i32AddrAdj, u32Src);
+                IEM_MC_AND_ARG_U32(u32Src, 0x1f);
+                IEM_MC_SAR_LOCAL_S32(i32AddrAdj, 5);
+                IEM_MC_SHL_LOCAL_S32(i32AddrAdj, 2);
+                IEM_MC_ADD_LOCAL_S32_TO_EFF_ADDR(GCPtrEffDst, i32AddrAdj);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+
+                IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 2);
+                IEM_MC_ARG(uint64_t *,              pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,                u64Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+                IEM_MC_LOCAL(int64_t,               i64AddrAdj);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_ASSIGN(i64AddrAdj, u64Src);
+                IEM_MC_AND_ARG_U64(u64Src, 0x3f);
+                IEM_MC_SAR_LOCAL_S64(i64AddrAdj, 6);
+                IEM_MC_SHL_LOCAL_S64(i64AddrAdj, 3);
+                IEM_MC_ADD_LOCAL_S64_TO_EFF_ADDR(GCPtrEffDst, i64AddrAdj);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+
+                IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0x0f 0xa3. */
+FNIEMOP_DEF(iemOp_bt_Ev_Gv)
+{
+    IEMOP_MNEMONIC(bt_Ev_Gv, "bt  Ev,Gv");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_bt);
+}
+
+
+/**
+ * Common worker for iemOp_shrd_Ev_Gv_Ib and iemOp_shld_Ev_Gv_Ib.
+ */
+FNIEMOP_DEF_1(iemOpCommonShldShrd_Ib, PCIEMOPSHIFTDBLSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF | X86_EFL_OF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,        u16Src,                 1);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, /*=*/cShift, 2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,        u32Src,                 1);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, /*=*/cShift, 2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,        u64Src,                 1);
+                IEM_MC_ARG_CONST(uint8_t,   cShiftArg, /*=*/cShift, 2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint16_t *,              pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,                u16Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint32_t *,              pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,                u32Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint64_t *,              pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,                u64Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t cShift; IEM_OPCODE_GET_NEXT_U8(&cShift);
+                IEM_MC_ASSIGN(cShiftArg, cShift);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/**
+ * Common worker for iemOp_shrd_Ev_Gv_CL and iemOp_shld_Ev_Gv_CL.
+ */
+FNIEMOP_DEF_1(iemOpCommonShldShrd_CL, PCIEMOPSHIFTDBLSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF | X86_EFL_OF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,        u16Src,                 1);
+                IEM_MC_ARG(uint8_t,         cShiftArg,              2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,        u32Src,                 1);
+                IEM_MC_ARG(uint8_t,         cShiftArg,              2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,        u64Src,                 1);
+                IEM_MC_ARG(uint8_t,         cShiftArg,              2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint16_t *,              pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,                u16Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU16, pu16Dst, u16Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint32_t *,              pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,                u32Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU32, pu32Dst, u32Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 2);
+                IEM_MC_ARG(uint64_t *,              pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,                u64Src,                 1);
+                IEM_MC_ARG(uint8_t,                 cShiftArg,              2);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U8(cShiftArg, X86_GREG_xCX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnNormalU64, pu64Dst, u64Src, cShiftArg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+
+/** Opcode 0x0f 0xa4. */
+FNIEMOP_DEF(iemOp_shld_Ev_Gv_Ib)
+{
+    IEMOP_MNEMONIC(shld_Ev_Gv_Ib, "shld Ev,Gv,Ib");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonShldShrd_Ib, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shld_eflags));
+}
+
+
+/** Opcode 0x0f 0xa5. */
+FNIEMOP_DEF(iemOp_shld_Ev_Gv_CL)
+{
+    IEMOP_MNEMONIC(shld_Ev_Gv_CL, "shld Ev,Gv,CL");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonShldShrd_CL, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shld_eflags));
+}
+
+
+/** Opcode 0x0f 0xa8. */
+FNIEMOP_DEF(iemOp_push_gs)
+{
+    IEMOP_MNEMONIC(push_gs, "push gs");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return FNIEMOP_CALL_1(iemOpCommonPushSReg, X86_SREG_GS);
+}
+
+
+/** Opcode 0x0f 0xa9. */
+FNIEMOP_DEF(iemOp_pop_gs)
+{
+    IEMOP_MNEMONIC(pop_gs, "pop gs");
+    IEMOP_HLP_MIN_386();
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_2(iemCImpl_pop_Sreg, X86_SREG_GS, pVCpu->iem.s.enmEffOpSize);
+}
+
+
+/** Opcode 0x0f 0xaa. */
+FNIEMOP_DEF(iemOp_rsm)
+{
+    IEMOP_MNEMONIC0(FIXED, RSM, rsm, DISOPTYPE_HARMLESS, 0);
+    IEMOP_HLP_MIN_386(); /* 386SL and later. */
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    return IEM_MC_DEFER_TO_CIMPL_0(iemCImpl_rsm);
+}
+
+
+
+/** Opcode 0x0f 0xab. */
+FNIEMOP_DEF(iemOp_bts_Ev_Gv)
+{
+    IEMOP_MNEMONIC(bts_Ev_Gv, "bts Ev,Gv");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_bts);
+}
+
+
+/** Opcode 0x0f 0xac. */
+FNIEMOP_DEF(iemOp_shrd_Ev_Gv_Ib)
+{
+    IEMOP_MNEMONIC(shrd_Ev_Gv_Ib, "shrd Ev,Gv,Ib");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonShldShrd_Ib, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shrd_eflags));
+}
+
+
+/** Opcode 0x0f 0xad. */
+FNIEMOP_DEF(iemOp_shrd_Ev_Gv_CL)
+{
+    IEMOP_MNEMONIC(shrd_Ev_Gv_CL, "shrd Ev,Gv,CL");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonShldShrd_CL, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_shrd_eflags));
+}
+
+
+/** Opcode 0x0f 0xae mem/0. */
+FNIEMOP_DEF_1(iemOp_Grp15_fxsave,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fxsave, "fxsave m512");
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFxSaveRstor)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_fxsave, iEffSeg, GCPtrEff, enmEffOpSize);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0xae mem/1. */
+FNIEMOP_DEF_1(iemOp_Grp15_fxrstor,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(fxrstor, "fxrstor m512");
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fFxSaveRstor)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(3, 1);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_fxrstor, iEffSeg, GCPtrEff, enmEffOpSize);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/2
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_mxcsrsm
+ * @opxcpttype  5
+ * @optest      op1=0      -> mxcsr=0
+ * @optest      op1=0x2083 -> mxcsr=0x2083
+ * @optest      op1=0xfffffffe -> value.xcpt=0xd
+ * @optest      op1=0x2083 cr0|=ts -> value.xcpt=0x7
+ * @optest      op1=0x2083 cr0|=em -> value.xcpt=0x6
+ * @optest      op1=0x2083 cr0|=mp -> mxcsr=0x2083
+ * @optest      op1=0x2083 cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      op1=0x2083 cr0|=ts,em -> value.xcpt=0x6
+ * @optest      op1=0x2083 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      op1=0x2083 cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      op1=0x2083 cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_ldmxcsr, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, LDMXCSR, ldmxcsr, Md_RO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_ldmxcsr, iEffSeg, GCPtrEff);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/3
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse_mxcsrsm
+ * @opxcpttype  5
+ * @optest      mxcsr=0      -> op1=0
+ * @optest      mxcsr=0x2083 -> op1=0x2083
+ * @optest      mxcsr=0x2084 cr0|=ts -> value.xcpt=0x7
+ * @optest      mxcsr=0x2085 cr0|=em -> value.xcpt=0x6
+ * @optest      mxcsr=0x2086 cr0|=mp -> op1=0x2086
+ * @optest      mxcsr=0x2087 cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      mxcsr=0x2088 cr0|=ts,em -> value.xcpt=0x6
+ * @optest      mxcsr=0x2089 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      mxcsr=0x208a cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      mxcsr=0x208b cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_stmxcsr,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, STMXCSR, stmxcsr, Md_WO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_stmxcsr, iEffSeg, GCPtrEff);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/4
+ * @oppfx       none
+ * @opcpuid     xsave
+ * @opgroup     og_system
+ * @opxcpttype  none
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_xsave,    uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, XSAVE, xsave, M_RW, DISOPTYPE_HARMLESS, 0);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_xsave, iEffSeg, GCPtrEff, enmEffOpSize);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/5
+ * @oppfx       none
+ * @opcpuid     xsave
+ * @opgroup     og_system
+ * @opxcpttype  none
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_xrstor,   uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, XRSTOR, xrstor, M_RO, DISOPTYPE_HARMLESS, 0);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fXSaveRstor)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(3, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 2);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ACTUALIZE_FPU_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_3(iemCImpl_xrstor, iEffSeg, GCPtrEff, enmEffOpSize);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+/** Opcode 0x0f 0xae mem/6. */
+FNIEMOP_STUB_1(iemOp_Grp15_xsaveopt, uint8_t, bRm);
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/7
+ * @oppfx       none
+ * @opcpuid     clfsh
+ * @opgroup     og_cachectl
+ * @optest      op1=1 ->
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_clflush,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, CLFLUSH, clflush, Mb_RO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fClFlush)
+        return FNIEMOP_CALL_1(iemOp_InvalidWithRMAllNeeded, bRm);
+
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_clflush_clflushopt, iEffSeg, GCPtrEff);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+/**
+ * @opmaps      grp15
+ * @opcode      !11/7
+ * @oppfx       0x66
+ * @opcpuid     clflushopt
+ * @opgroup     og_cachectl
+ * @optest      op1=1 ->
+ */
+FNIEMOP_DEF_1(iemOp_Grp15_clflushopt,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(M_MEM, CLFLUSHOPT, clflushopt, Mb_RO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fClFlushOpt)
+        return FNIEMOP_CALL_1(iemOp_InvalidWithRMAllNeeded, bRm);
+
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_clflush_clflushopt, iEffSeg, GCPtrEff);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+
+/** Opcode 0x0f 0xae 11b/5. */
+FNIEMOP_DEF_1(iemOp_Grp15_lfence,   uint8_t, bRm)
+{
+    RT_NOREF_PV(bRm);
+    IEMOP_MNEMONIC(lfence, "lfence");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(0, 0);
+#ifndef RT_ARCH_ARM64
+    if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fSse2)
+#endif
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_lfence);
+#ifndef RT_ARCH_ARM64
+    else
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_alt_mem_fence);
+#endif
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0x0f 0xae 11b/6. */
+FNIEMOP_DEF_1(iemOp_Grp15_mfence,   uint8_t, bRm)
+{
+    RT_NOREF_PV(bRm);
+    IEMOP_MNEMONIC(mfence, "mfence");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(0, 0);
+#ifndef RT_ARCH_ARM64
+    if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fSse2)
+#endif
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_mfence);
+#ifndef RT_ARCH_ARM64
+    else
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_alt_mem_fence);
+#endif
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0x0f 0xae 11b/7. */
+FNIEMOP_DEF_1(iemOp_Grp15_sfence,   uint8_t, bRm)
+{
+    RT_NOREF_PV(bRm);
+    IEMOP_MNEMONIC(sfence, "sfence");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    IEM_MC_BEGIN(0, 0);
+#ifndef RT_ARCH_ARM64
+    if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fSse2)
+#endif
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_sfence);
+#ifndef RT_ARCH_ARM64
+    else
+        IEM_MC_CALL_VOID_AIMPL_0(iemAImpl_alt_mem_fence);
+#endif
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+    IEM_MC_END();
+}
+
+
+/** Opcode 0xf3 0x0f 0xae 11b/0. */
+FNIEMOP_DEF_1(iemOp_Grp15_rdfsbase, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(rdfsbase, "rdfsbase Ry");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint64_t, u64Dst, 0);
+        IEM_MC_FETCH_SREG_BASE_U64(u64Dst, X86_SREG_FS);
+        IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint32_t, u32Dst, 0);
+        IEM_MC_FETCH_SREG_BASE_U32(u32Dst, X86_SREG_FS);
+        IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0xf3 0x0f 0xae 11b/1. */
+FNIEMOP_DEF_1(iemOp_Grp15_rdgsbase, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(rdgsbase, "rdgsbase Ry");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint64_t, u64Dst, 0);
+        IEM_MC_FETCH_SREG_BASE_U64(u64Dst, X86_SREG_GS);
+        IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint32_t, u32Dst, 0);
+        IEM_MC_FETCH_SREG_BASE_U32(u32Dst, X86_SREG_GS);
+        IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0xf3 0x0f 0xae 11b/2. */
+FNIEMOP_DEF_1(iemOp_Grp15_wrfsbase, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(wrfsbase, "wrfsbase Ry");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint64_t, u64Dst, 0);
+        IEM_MC_FETCH_GREG_U64(u64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(u64Dst);
+        IEM_MC_STORE_SREG_BASE_U64(X86_SREG_FS, u64Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint32_t, u32Dst, 0);
+        IEM_MC_FETCH_GREG_U32(u32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_STORE_SREG_BASE_U64(X86_SREG_FS, u32Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0xf3 0x0f 0xae 11b/3. */
+FNIEMOP_DEF_1(iemOp_Grp15_wrgsbase, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(wrgsbase, "wrgsbase Ry");
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint64_t, u64Dst, 0);
+        IEM_MC_FETCH_GREG_U64(u64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_MAYBE_RAISE_NON_CANONICAL_ADDR_GP0(u64Dst);
+        IEM_MC_STORE_SREG_BASE_U64(X86_SREG_GS, u64Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(1, 0);
+        IEM_MC_MAYBE_RAISE_FSGSBASE_XCPT();
+        IEM_MC_ARG(uint32_t, u32Dst, 0);
+        IEM_MC_FETCH_GREG_U32(u32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_STORE_SREG_BASE_U64(X86_SREG_GS, u32Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * Group 15 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup15RegReg[] =
+{   /* pfx:  none,                          066h,                           0f3h,                           0f2h */
+    /* /0 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_Grp15_rdfsbase,           iemOp_InvalidWithRM,
+    /* /1 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_Grp15_rdgsbase,           iemOp_InvalidWithRM,
+    /* /2 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_Grp15_wrfsbase,           iemOp_InvalidWithRM,
+    /* /3 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_Grp15_wrgsbase,           iemOp_InvalidWithRM,
+    /* /4 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /5 */ iemOp_Grp15_lfence,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /6 */ iemOp_Grp15_mfence,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /7 */ iemOp_Grp15_sfence,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup15RegReg) == 8*4);
+
+
+/**
+ * Group 15 jump table for memory variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup15MemReg[] =
+{   /* pfx:  none,                          066h,                           0f3h,                           0f2h */
+    /* /0 */ iemOp_Grp15_fxsave,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /1 */ iemOp_Grp15_fxrstor,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /2 */ iemOp_Grp15_ldmxcsr,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /3 */ iemOp_Grp15_stmxcsr,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /4 */ iemOp_Grp15_xsave,             iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /5 */ iemOp_Grp15_xrstor,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /6 */ iemOp_Grp15_xsaveopt,          iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /7 */ iemOp_Grp15_clflush,           iemOp_Grp15_clflushopt,         iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup15MemReg) == 8*4);
+
+
+/** Opcode 0x0f 0xae. */
+FNIEMOP_DEF(iemOp_Grp15)
+{
+    IEMOP_HLP_MIN_586(); /* Not entirely accurate nor needed, but useful for debugging 286 code. */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnGroup15RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4
+                                                  + pVCpu->iem.s.idxPrefix], bRm);
+    /* memory, register */
+    return FNIEMOP_CALL_1(g_apfnGroup15MemReg[ IEM_GET_MODRM_REG_8(bRm) * 4
+                                              + pVCpu->iem.s.idxPrefix], bRm);
+}
+
+
+/** Opcode 0x0f 0xaf. */
+FNIEMOP_DEF(iemOp_imul_Gv_Ev)
+{
+    IEMOP_MNEMONIC(imul_Gv_Ev, "imul Gv,Ev");
+    IEMOP_HLP_MIN_386();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_imul_two_eflags));
+}
+
+
+/** Opcode 0x0f 0xb0. */
+FNIEMOP_DEF(iemOp_cmpxchg_Eb_Gb)
+{
+    IEMOP_MNEMONIC(cmpxchg_Eb_Gb, "cmpxchg Eb,Gb");
+    IEMOP_HLP_MIN_486();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING();
+        IEM_MC_BEGIN(4, 0);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,                 0);
+        IEM_MC_ARG(uint8_t *,       pu8Al,                  1);
+        IEM_MC_ARG(uint8_t,         u8Src,                  2);
+        IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+        IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_GREG_U8(pu8Al, X86_GREG_xAX);
+        IEM_MC_REF_EFLAGS(pEFlags);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8, pu8Dst, pu8Al, u8Src, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8_locked, pu8Dst, pu8Al, u8Src, pEFlags);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_ARG(uint8_t *,       pu8Dst,                 0);
+        IEM_MC_ARG(uint8_t *,       pu8Al,                  1);
+        IEM_MC_ARG(uint8_t,         u8Src,                  2);
+        IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        3);
+        IEM_MC_LOCAL(RTGCPTR,       GCPtrEffDst);
+        IEM_MC_LOCAL(uint8_t,       u8Al);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING();
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+        IEM_MC_FETCH_GREG_U8(u8Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_GREG_U8(u8Al, X86_GREG_xAX);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        IEM_MC_REF_LOCAL(pu8Al, u8Al);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8, pu8Dst, pu8Al, u8Src, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u8_locked, pu8Dst, pu8Al, u8Src, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_STORE_GREG_U8(X86_GREG_xAX, u8Al);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/** Opcode 0x0f 0xb1. */
+FNIEMOP_DEF(iemOp_cmpxchg_Ev_Gv)
+{
+    IEMOP_MNEMONIC(cmpxchg_Ev_Gv, "cmpxchg Ev,Gv");
+    IEMOP_HLP_MIN_486();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t *,      pu16Ax,                 1);
+                IEM_MC_ARG(uint16_t,        u16Src,                 2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U16(pu16Ax, X86_GREG_xAX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16, pu16Dst, pu16Ax, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16_locked, pu16Dst, pu16Ax, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t *,      pu32Eax,                1);
+                IEM_MC_ARG(uint32_t,        u32Src,                 2);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U32(pu32Eax, X86_GREG_xAX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32, pu32Dst, pu32Eax, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32_locked, pu32Dst, pu32Eax, u32Src, pEFlags);
+
+                IEM_MC_IF_EFL_BIT_SET(X86_EFL_ZF) {
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                } IEM_MC_ELSE() {
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Eax);
+                } IEM_MC_ENDIF();
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t *,      pu64Rax,                1);
+#ifdef RT_ARCH_X86
+                IEM_MC_ARG(uint64_t *,      pu64Src,                2);
+#else
+                IEM_MC_ARG(uint64_t,        u64Src,                 2);
+#endif
+                IEM_MC_ARG(uint32_t *,      pEFlags,                3);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U64(pu64Rax, X86_GREG_xAX);
+                IEM_MC_REF_EFLAGS(pEFlags);
+#ifdef RT_ARCH_X86
+                IEM_MC_REF_GREG_U64(pu64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, pu64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, pu64Src, pEFlags);
+#else
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, u64Src, pEFlags);
+#endif
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(4, 3);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint16_t *,      pu16Ax,                 1);
+                IEM_MC_ARG(uint16_t,        u16Src,                 2);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,       GCPtrEffDst);
+                IEM_MC_LOCAL(uint16_t,      u16Ax);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING();
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U16(u16Ax, X86_GREG_xAX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_REF_LOCAL(pu16Ax, u16Ax);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16, pu16Dst, pu16Ax, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u16_locked, pu16Dst, pu16Ax, u16Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U16(X86_GREG_xAX, u16Ax);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(4, 3);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t *,      pu32Eax,                 1);
+                IEM_MC_ARG(uint32_t,        u32Src,                 2);
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,       GCPtrEffDst);
+                IEM_MC_LOCAL(uint32_t,      u32Eax);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING();
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_FETCH_GREG_U32(u32Eax, X86_GREG_xAX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_REF_LOCAL(pu32Eax, u32Eax);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32, pu32Dst, pu32Eax, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u32_locked, pu32Dst, pu32Eax, u32Src, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+
+                IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF)
+                    IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u32Eax);
+                IEM_MC_ENDIF();
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(4, 3);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint64_t *,      pu64Rax,                1);
+#ifdef RT_ARCH_X86
+                IEM_MC_ARG(uint64_t *,      pu64Src,                2);
+#else
+                IEM_MC_ARG(uint64_t,        u64Src,                 2);
+#endif
+                IEM_MC_ARG_LOCAL_EFLAGS(    pEFlags, EFlags,        3);
+                IEM_MC_LOCAL(RTGCPTR,       GCPtrEffDst);
+                IEM_MC_LOCAL(uint64_t,      u64Rax);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING();
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                IEM_MC_FETCH_GREG_U64(u64Rax, X86_GREG_xAX);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_REF_LOCAL(pu64Rax, u64Rax);
+#ifdef RT_ARCH_X86
+                IEM_MC_REF_GREG_U64(pu64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, pu64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, pu64Src, pEFlags);
+#else
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_REG(pVCpu, bRm));
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64, pu64Dst, pu64Rax, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg_u64_locked, pu64Dst, pu64Rax, u64Src, pEFlags);
+#endif
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u64Rax);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+FNIEMOP_DEF_2(iemOpCommonLoadSRegAndGreg, uint8_t, iSegReg, uint8_t, bRm)
+{
+    Assert(IEM_IS_MODRM_MEM_MODE(bRm)); /* Caller checks this */
+    uint8_t const iGReg = IEM_GET_MODRM_REG(pVCpu, bRm);
+
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(5, 1);
+            IEM_MC_ARG(uint16_t,        uSel,                                    0);
+            IEM_MC_ARG(uint16_t,        offSeg,                                  1);
+            IEM_MC_ARG_CONST(uint8_t,   iSegRegArg,/*=*/iSegReg,                 2);
+            IEM_MC_ARG_CONST(uint8_t,   iGRegArg,  /*=*/iGReg,                   3);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 4);
+            IEM_MC_LOCAL(RTGCPTR,       GCPtrEff);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff);
+            IEM_MC_FETCH_MEM_U16_DISP(uSel, pVCpu->iem.s.iEffSeg, GCPtrEff, 2);
+            IEM_MC_CALL_CIMPL_5(iemCImpl_load_SReg_Greg, uSel, offSeg, iSegRegArg, iGRegArg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(5, 1);
+            IEM_MC_ARG(uint16_t,        uSel,                                    0);
+            IEM_MC_ARG(uint32_t,        offSeg,                                  1);
+            IEM_MC_ARG_CONST(uint8_t,   iSegRegArg,/*=*/iSegReg,                 2);
+            IEM_MC_ARG_CONST(uint8_t,   iGRegArg,  /*=*/iGReg,                   3);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 4);
+            IEM_MC_LOCAL(RTGCPTR,       GCPtrEff);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U32(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff);
+            IEM_MC_FETCH_MEM_U16_DISP(uSel, pVCpu->iem.s.iEffSeg, GCPtrEff, 4);
+            IEM_MC_CALL_CIMPL_5(iemCImpl_load_SReg_Greg, uSel, offSeg, iSegRegArg, iGRegArg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(5, 1);
+            IEM_MC_ARG(uint16_t,        uSel,                                    0);
+            IEM_MC_ARG(uint64_t,        offSeg,                                  1);
+            IEM_MC_ARG_CONST(uint8_t,   iSegRegArg,/*=*/iSegReg,                 2);
+            IEM_MC_ARG_CONST(uint8_t,   iGRegArg,  /*=*/iGReg,                   3);
+            IEM_MC_ARG_CONST(IEMMODE,   enmEffOpSize,/*=*/pVCpu->iem.s.enmEffOpSize, 4);
+            IEM_MC_LOCAL(RTGCPTR,       GCPtrEff);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            if (IEM_IS_GUEST_CPU_AMD(pVCpu)) /** @todo testcase: rev 3.15 of the amd manuals claims it only loads a 32-bit greg. */
+                IEM_MC_FETCH_MEM_U32_SX_U64(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff);
+            else
+                IEM_MC_FETCH_MEM_U64(offSeg, pVCpu->iem.s.iEffSeg, GCPtrEff);
+            IEM_MC_FETCH_MEM_U16_DISP(uSel, pVCpu->iem.s.iEffSeg, GCPtrEff, 8);
+            IEM_MC_CALL_CIMPL_5(iemCImpl_load_SReg_Greg, uSel, offSeg, iSegRegArg, iGRegArg, enmEffOpSize);
+            IEM_MC_END();
+            return VINF_SUCCESS;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcode 0x0f 0xb2. */
+FNIEMOP_DEF(iemOp_lss_Gv_Mp)
+{
+    IEMOP_MNEMONIC(lss_Gv_Mp, "lss Gv,Mp");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_SS, bRm);
+}
+
+
+/** Opcode 0x0f 0xb3. */
+FNIEMOP_DEF(iemOp_btr_Ev_Gv)
+{
+    IEMOP_MNEMONIC(btr_Ev_Gv, "btr Ev,Gv");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_btr);
+}
+
+
+/** Opcode 0x0f 0xb4. */
+FNIEMOP_DEF(iemOp_lfs_Gv_Mp)
+{
+    IEMOP_MNEMONIC(lfs_Gv_Mp, "lfs Gv,Mp");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_FS, bRm);
+}
+
+
+/** Opcode 0x0f 0xb5. */
+FNIEMOP_DEF(iemOp_lgs_Gv_Mp)
+{
+    IEMOP_MNEMONIC(lgs_Gv_Mp, "lgs Gv,Mp");
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        return IEMOP_RAISE_INVALID_OPCODE();
+    return FNIEMOP_CALL_2(iemOpCommonLoadSRegAndGreg, X86_SREG_GS, bRm);
+}
+
+
+/** Opcode 0x0f 0xb6. */
+FNIEMOP_DEF(iemOp_movzx_Gv_Eb)
+{
+    IEMOP_MNEMONIC(movzx_Gv_Eb, "movzx Gv,Eb");
+    IEMOP_HLP_MIN_386();
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_FETCH_GREG_U8_ZX_U16(u16Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_FETCH_GREG_U8_ZX_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_FETCH_GREG_U8_ZX_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_ZX_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_ZX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_ZX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0x0f 0xb7. */
+FNIEMOP_DEF(iemOp_movzx_Gv_Ew)
+{
+    IEMOP_MNEMONIC(movzx_Gv_Ew, "movzx Gv,Ew");
+    IEMOP_HLP_MIN_386();
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Not entirely sure how the operand size prefix is handled here,
+     *        assuming that it will be ignored. Would be nice to have a few
+     *        test for this. */
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_FETCH_GREG_U16_ZX_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_FETCH_GREG_U16_ZX_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16_ZX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16_ZX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode      0x0f 0xb8 - JMPE (reserved for emulator on IPF) */
+FNIEMOP_UD_STUB(iemOp_jmpe);
+
+
+/** Opcode 0xf3 0x0f 0xb8 - POPCNT Gv, Ev */
+FNIEMOP_DEF(iemOp_popcnt_Gv_Ev)
+{
+    IEMOP_MNEMONIC2(RM, POPCNT, popcnt, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fPopCnt)
+        return iemOp_InvalidNeedRM(pVCpu);
+#ifndef TST_IEM_CHECK_MC
+# if (defined(RT_ARCH_X86) || defined(RT_ARCH_AMD64)) && !defined(IEM_WITHOUT_ASSEMBLY)
+    static const IEMOPBINSIZES s_Native =
+    {   NULL, NULL, iemAImpl_popcnt_u16, NULL, iemAImpl_popcnt_u32, NULL, iemAImpl_popcnt_u64, NULL };
+# endif
+    static const IEMOPBINSIZES s_Fallback =
+    {   NULL, NULL, iemAImpl_popcnt_u16_fallback, NULL, iemAImpl_popcnt_u32_fallback, NULL, iemAImpl_popcnt_u64_fallback, NULL };
+#endif
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm, IEM_SELECT_HOST_OR_FALLBACK(fPopCnt, &s_Native, &s_Fallback));
+}
+
+
+/**
+ * @opcode      0xb9
+ * @opinvalid   intel-modrm
+ * @optest      ->
+ */
+FNIEMOP_DEF(iemOp_Grp10)
+{
+    /*
+     * AMD does not decode beyond the 0xb9 whereas intel does the modr/m bit
+     * too. See bs3-cpu-decoder-1.c32.  So, we can forward to iemOp_InvalidNeedRM.
+     */
+    Log(("iemOp_Grp10 aka UD1 -> #UD\n"));
+    IEMOP_MNEMONIC2EX(ud1, "ud1", RM, UD1, ud1, Gb, Eb, DISOPTYPE_INVALID, IEMOPHINT_IGNORES_OP_SIZES); /* just picked Gb,Eb here. */
+    return FNIEMOP_CALL(iemOp_InvalidNeedRM);
+}
+
+
+/** Opcode 0x0f 0xba. */
+FNIEMOP_DEF(iemOp_Grp8)
+{
+    IEMOP_HLP_MIN_386();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    PCIEMOPBINSIZES pImpl;
+    switch (IEM_GET_MODRM_REG_8(bRm))
+    {
+        case 0: case 1: case 2: case 3:
+            /* Both AMD and Intel want full modr/m decoding and imm8. */
+            return FNIEMOP_CALL_1(iemOp_InvalidWithRMAllNeedImm8, bRm);
+        case 4: pImpl = &g_iemAImpl_bt;  IEMOP_MNEMONIC(bt_Ev_Ib,  "bt  Ev,Ib"); break;
+        case 5: pImpl = &g_iemAImpl_bts; IEMOP_MNEMONIC(bts_Ev_Ib, "bts Ev,Ib"); break;
+        case 6: pImpl = &g_iemAImpl_btr; IEMOP_MNEMONIC(btr_Ev_Ib, "btr Ev,Ib"); break;
+        case 7: pImpl = &g_iemAImpl_btc; IEMOP_MNEMONIC(btc_Ev_Ib, "btc Ev,Ib"); break;
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF);
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register destination. */
+        uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                    0);
+                IEM_MC_ARG_CONST(uint16_t,  u16Src, /*=*/ u8Bit & 0x0f, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                    0);
+                IEM_MC_ARG_CONST(uint32_t,  u32Src, /*=*/ u8Bit & 0x1f, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                    0);
+                IEM_MC_ARG_CONST(uint64_t,  u64Src, /*=*/ u8Bit & 0x3f, 1);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                    2);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /* memory destination. */
+
+        uint32_t fAccess;
+        if (pImpl->pfnLockedU16)
+            fAccess = IEM_ACCESS_DATA_RW;
+        else /* BT */
+            fAccess = IEM_ACCESS_DATA_R;
+
+        /** @todo test negative bit offsets! */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *,              pu16Dst,                0);
+                IEM_MC_ARG(uint16_t,                u16Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit);
+                IEM_MC_ASSIGN(u16Src, u8Bit & 0x0f);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu16Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU16, pu16Dst, u16Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t *,              pu32Dst,                0);
+                IEM_MC_ARG(uint32_t,                u32Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit);
+                IEM_MC_ASSIGN(u32Src, u8Bit & 0x1f);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu32Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *,              pu64Dst,                0);
+                IEM_MC_ARG(uint64_t,                u64Src,                 1);
+                IEM_MC_ARG_LOCAL_EFLAGS(            pEFlags, EFlags,        2);
+                IEM_MC_LOCAL(RTGCPTR,               GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 1);
+                uint8_t u8Bit; IEM_OPCODE_GET_NEXT_U8(&u8Bit);
+                IEM_MC_ASSIGN(u64Src, u8Bit & 0x3f);
+                if (pImpl->pfnLockedU16)
+                    IEMOP_HLP_DONE_DECODING();
+                else
+                    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                IEM_MC_MEM_MAP(pu64Dst, fAccess, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnLockedU64, pu64Dst, u64Src, pEFlags);
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, fAccess);
+
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0x0f 0xbb. */
+FNIEMOP_DEF(iemOp_btc_Ev_Gv)
+{
+    IEMOP_MNEMONIC(btc_Ev_Gv, "btc Ev,Gv");
+    IEMOP_HLP_MIN_386();
+    return FNIEMOP_CALL_1(iemOpCommonBit_Ev_Gv, &g_iemAImpl_btc);
+}
+
+
+/**
+ * Common worker for BSF and BSR instructions.
+ *
+ * These cannot use iemOpHlpBinaryOperator_rv_rm because they don't always write
+ * the destination register, which means that for 32-bit operations the high
+ * bits must be left alone.
+ *
+ * @param   pImpl       Pointer to the instruction implementation (assembly).
+ */
+FNIEMOP_DEF_1(iemOpHlpBitScanOperator_rv_rm, PCIEMOPBINSIZES, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *, pu16Dst, 0);
+                IEM_MC_ARG(uint16_t,   u16Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *, pu32Dst, 0);
+                IEM_MC_ARG(uint32_t,   u32Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U32(u32Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+                IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF)
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ENDIF();
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *, pu64Dst, 0);
+                IEM_MC_ARG(uint64_t,   u64Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_FETCH_GREG_U64(u64Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint16_t *, pu16Dst, 0);
+                IEM_MC_ARG(uint16_t,   u16Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU16, pu16Dst, u16Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint32_t *, pu32Dst, 0);
+                IEM_MC_ARG(uint32_t,   u32Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U32(u32Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU32, pu32Dst, u32Src, pEFlags);
+
+                IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF)
+                    IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ENDIF();
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 1);
+                IEM_MC_ARG(uint64_t *, pu64Dst, 0);
+                IEM_MC_ARG(uint64_t,   u64Src,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+                IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U64(u64Src, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnNormalU64, pu64Dst, u64Src, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0x0f 0xbc. */
+FNIEMOP_DEF(iemOp_bsf_Gv_Ev)
+{
+    IEMOP_MNEMONIC(bsf_Gv_Ev, "bsf Gv,Ev");
+    IEMOP_HLP_MIN_386();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF);
+    return FNIEMOP_CALL_1(iemOpHlpBitScanOperator_rv_rm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_bsf_eflags));
+}
+
+
+/** Opcode 0xf3 0x0f 0xbc - TZCNT Gv, Ev */
+FNIEMOP_DEF(iemOp_tzcnt_Gv_Ev)
+{
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi1)
+        return FNIEMOP_CALL(iemOp_bsf_Gv_Ev);
+    IEMOP_MNEMONIC2(RM, TZCNT, tzcnt, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+
+#ifndef TST_IEM_CHECK_MC
+    static const IEMOPBINSIZES s_iemAImpl_tzcnt =
+    {   NULL, NULL,     iemAImpl_tzcnt_u16, NULL,       iemAImpl_tzcnt_u32, NULL,       iemAImpl_tzcnt_u64, NULL };
+    static const IEMOPBINSIZES s_iemAImpl_tzcnt_amd =
+    {   NULL, NULL,     iemAImpl_tzcnt_u16_amd, NULL,   iemAImpl_tzcnt_u32_amd, NULL,   iemAImpl_tzcnt_u64_amd, NULL };
+    static const IEMOPBINSIZES s_iemAImpl_tzcnt_intel =
+    {   NULL, NULL,     iemAImpl_tzcnt_u16_intel, NULL, iemAImpl_tzcnt_u32_intel, NULL, iemAImpl_tzcnt_u64_intel, NULL };
+    static const IEMOPBINSIZES * const s_iemAImpl_tzcnt_eflags[2][4] =
+    {
+        { &s_iemAImpl_tzcnt_intel, &s_iemAImpl_tzcnt_intel, &s_iemAImpl_tzcnt_amd, &s_iemAImpl_tzcnt_intel },
+        { &s_iemAImpl_tzcnt,       &s_iemAImpl_tzcnt_intel, &s_iemAImpl_tzcnt_amd, &s_iemAImpl_tzcnt }
+    };
+#endif
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm,
+                          IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(s_iemAImpl_tzcnt_eflags, IEM_GET_HOST_CPU_FEATURES(pVCpu)->fBmi1));
+}
+
+
+/** Opcode 0x0f 0xbd. */
+FNIEMOP_DEF(iemOp_bsr_Gv_Ev)
+{
+    IEMOP_MNEMONIC(bsr_Gv_Ev, "bsr Gv,Ev");
+    IEMOP_HLP_MIN_386();
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF);
+    return FNIEMOP_CALL_1(iemOpHlpBitScanOperator_rv_rm, IEMTARGETCPU_EFL_BEHAVIOR_SELECT(g_iemAImpl_bsr_eflags));
+}
+
+
+/** Opcode 0xf3 0x0f 0xbd - LZCNT Gv, Ev */
+FNIEMOP_DEF(iemOp_lzcnt_Gv_Ev)
+{
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi1)
+        return FNIEMOP_CALL(iemOp_bsr_Gv_Ev);
+    IEMOP_MNEMONIC2(RM, LZCNT, lzcnt, Gv, Ev, DISOPTYPE_HARMLESS, 0);
+
+#ifndef TST_IEM_CHECK_MC
+    static const IEMOPBINSIZES s_iemAImpl_lzcnt =
+    {   NULL, NULL,     iemAImpl_lzcnt_u16, NULL,       iemAImpl_lzcnt_u32, NULL,       iemAImpl_lzcnt_u64, NULL };
+    static const IEMOPBINSIZES s_iemAImpl_lzcnt_amd =
+    {   NULL,  NULL,    iemAImpl_lzcnt_u16_amd, NULL,   iemAImpl_lzcnt_u32_amd, NULL,   iemAImpl_lzcnt_u64_amd, NULL };
+    static const IEMOPBINSIZES s_iemAImpl_lzcnt_intel =
+    {   NULL,  NULL,    iemAImpl_lzcnt_u16_intel, NULL, iemAImpl_lzcnt_u32_intel, NULL, iemAImpl_lzcnt_u64_intel, NULL };
+    static const IEMOPBINSIZES * const s_iemAImpl_lzcnt_eflags[2][4] =
+    {
+        { &s_iemAImpl_lzcnt_intel, &s_iemAImpl_lzcnt_intel, &s_iemAImpl_lzcnt_amd, &s_iemAImpl_lzcnt_intel },
+        { &s_iemAImpl_lzcnt,       &s_iemAImpl_lzcnt_intel, &s_iemAImpl_lzcnt_amd, &s_iemAImpl_lzcnt }
+    };
+#endif
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF);
+    return FNIEMOP_CALL_1(iemOpHlpBinaryOperator_rv_rm,
+                          IEMTARGETCPU_EFL_BEHAVIOR_SELECT_EX(s_iemAImpl_lzcnt_eflags, IEM_GET_HOST_CPU_FEATURES(pVCpu)->fBmi1));
+}
+
+
+
+/** Opcode 0x0f 0xbe. */
+FNIEMOP_DEF(iemOp_movsx_Gv_Eb)
+{
+    IEMOP_MNEMONIC(movsx_Gv_Eb, "movsx Gv,Eb");
+    IEMOP_HLP_MIN_386();
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_FETCH_GREG_U8_SX_U16(u16Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_FETCH_GREG_U8_SX_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 1);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_FETCH_GREG_U8_SX_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint16_t, u16Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_SX_U16(u16Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_SX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                IEM_MC_FETCH_MEM_U8_SX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode 0x0f 0xbf. */
+FNIEMOP_DEF(iemOp_movsx_Gv_Ew)
+{
+    IEMOP_MNEMONIC(movsx_Gv_Ew, "movsx Gv,Ew");
+    IEMOP_HLP_MIN_386();
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /** @todo Not entirely sure how the operand size prefix is handled here,
+     *        assuming that it will be ignored. Would be nice to have a few
+     *        test for this. */
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_FETCH_GREG_U16_SX_U32(u32Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_FETCH_GREG_U16_SX_U64(u64Value, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * We're loading a register from memory.
+         */
+        if (pVCpu->iem.s.enmEffOpSize != IEMMODE_64BIT)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint32_t, u32Value);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16_SX_U32(u32Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(uint64_t, u64Value);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+            IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+            IEM_MC_FETCH_MEM_U16_SX_U64(u64Value, pVCpu->iem.s.iEffSeg, GCPtrEffDst);
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64Value);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode 0x0f 0xc0. */
+FNIEMOP_DEF(iemOp_xadd_Eb_Gb)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    IEMOP_HLP_MIN_486();
+    IEMOP_MNEMONIC(xadd_Eb_Gb, "xadd Eb,Gb");
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint8_t *,  pu8Dst,  0);
+        IEM_MC_ARG(uint8_t *,  pu8Reg,  1);
+        IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+        IEM_MC_REF_GREG_U8(pu8Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_GREG_U8(pu8Reg, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_EFLAGS(pEFlags);
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u8, pu8Dst, pu8Reg, pEFlags);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        IEM_MC_BEGIN(3, 3);
+        IEM_MC_ARG(uint8_t *,   pu8Dst,          0);
+        IEM_MC_ARG(uint8_t *,   pu8Reg,          1);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+        IEM_MC_LOCAL(uint8_t,  u8RegCopy);
+        IEM_MC_LOCAL(RTGCPTR,  GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEM_MC_MEM_MAP(pu8Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+        IEM_MC_FETCH_GREG_U8(u8RegCopy, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_LOCAL(pu8Reg, u8RegCopy);
+        IEM_MC_FETCH_EFLAGS(EFlags);
+        if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u8, pu8Dst, pu8Reg, pEFlags);
+        else
+            IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u8_locked, pu8Dst, pu8Reg, pEFlags);
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu8Dst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_STORE_GREG_U8(IEM_GET_MODRM_REG(pVCpu, bRm), u8RegCopy);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0xc1. */
+FNIEMOP_DEF(iemOp_xadd_Ev_Gv)
+{
+    IEMOP_MNEMONIC(xadd_Ev_Gv, "xadd Ev,Gv");
+    IEMOP_HLP_MIN_486();
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /*
+     * If rm is denoting a register, no more instruction bytes.
+     */
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint16_t *, pu16Dst,  0);
+                IEM_MC_ARG(uint16_t *, pu16Reg,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U16(pu16Reg, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u16, pu16Dst, pu16Reg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint32_t *, pu32Dst,  0);
+                IEM_MC_ARG(uint32_t *, pu32Reg,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U32(pu32Reg, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u32, pu32Dst, pu32Reg, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Reg);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 0);
+                IEM_MC_ARG(uint64_t *, pu64Dst,  0);
+                IEM_MC_ARG(uint64_t *, pu64Reg,  1);
+                IEM_MC_ARG(uint32_t *, pEFlags, 2);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_GREG_U64(pu64Reg, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u64, pu64Dst, pu64Reg, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+    {
+        /*
+         * We're accessing memory.
+         */
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(3, 3);
+                IEM_MC_ARG(uint16_t *,  pu16Dst,         0);
+                IEM_MC_ARG(uint16_t *,  pu16Reg,         1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(uint16_t,  u16RegCopy);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu16Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U16(u16RegCopy, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_LOCAL(pu16Reg, u16RegCopy);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u16, pu16Dst, pu16Reg, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u16_locked, pu16Dst, pu16Reg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu16Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U16(IEM_GET_MODRM_REG(pVCpu, bRm), u16RegCopy);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(3, 3);
+                IEM_MC_ARG(uint32_t *,  pu32Dst,         0);
+                IEM_MC_ARG(uint32_t *,  pu32Reg,         1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(uint32_t,  u32RegCopy);
+                IEM_MC_LOCAL(RTGCPTR,   GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu32Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U32(u32RegCopy, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_LOCAL(pu32Reg, u32RegCopy);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u32, pu32Dst, pu32Reg, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u32_locked, pu32Dst, pu32Reg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu32Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u32RegCopy);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(3, 3);
+                IEM_MC_ARG(uint64_t *,  pu64Dst,         0);
+                IEM_MC_ARG(uint64_t *,  pu64Reg,         1);
+                IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 2);
+                IEM_MC_LOCAL(uint64_t,  u64RegCopy);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEM_MC_MEM_MAP(pu64Dst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+                IEM_MC_FETCH_GREG_U64(u64RegCopy, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_REF_LOCAL(pu64Reg, u64RegCopy);
+                IEM_MC_FETCH_EFLAGS(EFlags);
+                if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u64, pu64Dst, pu64Reg, pEFlags);
+                else
+                    IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_xadd_u64_locked, pu64Dst, pu64Reg, pEFlags);
+
+                IEM_MC_MEM_COMMIT_AND_UNMAP(pu64Dst, IEM_ACCESS_DATA_RW);
+                IEM_MC_COMMIT_EFLAGS(EFlags);
+                IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), u64RegCopy);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+}
+
+
+/** Opcode      0x0f 0xc2 - cmpps Vps,Wps,Ib */
+FNIEMOP_DEF(iemOp_cmpps_Vps_Wps_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, CMPPS, cmpps, Vps, Wps, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 2);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpps_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_LOCAL(RTGCPTR,                       GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(Src.uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpps_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0xc2 - cmppd Vpd,Wpd,Ib */
+FNIEMOP_DEF(iemOp_cmppd_Vpd_Wpd_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, CMPPD, cmppd, Vpd, Wpd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 2);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmppd_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128].
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_LOCAL(RTGCPTR,                       GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_ALIGN_SSE(Src.uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmppd_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM(IEM_GET_MODRM_REG(pVCpu, bRm), Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0xf3 0x0f 0xc2 - cmpss Vss,Wss,Ib */
+FNIEMOP_DEF(iemOp_cmpss_Vss_Wss_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, CMPSS, cmpss, Vss, Wss, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM32, XMM32.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 2);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpss_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM_U32(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/, Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM32, [mem32].
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_LOCAL(RTGCPTR,                       GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U32(Src.uSrc2, 0 /*a_iDword */, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpss_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM_U32(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iDword*/, Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0xf2 0x0f 0xc2 - cmpsd Vsd,Wsd,Ib */
+FNIEMOP_DEF(iemOp_cmpsd_Vsd_Wsd_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, CMPSD, cmpsd, Vsd, Wsd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM64, XMM64.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(4, 2);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpsd_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iQword*/, Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM64, [mem64].
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(IEMMEDIAF2XMMSRC,              Src);
+        IEM_MC_LOCAL(X86XMMREG,                     Dst);
+        IEM_MC_ARG(uint32_t *,                      pfMxcsr,                0);
+        IEM_MC_ARG_LOCAL_REF(PX86XMMREG,            pDst,           Dst,    1);
+        IEM_MC_ARG_LOCAL_REF(PCIEMMEDIAF2XMMSRC,    pSrc,           Src,    2);
+        IEM_MC_LOCAL(RTGCPTR,                       GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,                   bImmArg, /*=*/ bImm,    3);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U64(Src.uSrc2, 0 /*a_iQword */, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_FETCH_XREG_XMM(Src.uSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpsd_u128, pfMxcsr, pDst, pSrc, bImmArg);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_STORE_XREG_XMM_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /*a_iQword*/, Dst);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x0f 0xc3. */
+FNIEMOP_DEF(iemOp_movnti_My_Gy)
+{
+    IEMOP_MNEMONIC(movnti_My_Gy, "movnti My,Gy");
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+
+    /* Only the register -> memory form makes sense, assuming #UD for the other form. */
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint32_t, u32Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2)
+                    return IEMOP_RAISE_INVALID_OPCODE();
+
+                IEM_MC_FETCH_GREG_U32(u32Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u32Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(0, 2);
+                IEM_MC_LOCAL(uint64_t, u64Value);
+                IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+                IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+                IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+                if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2)
+                    return IEMOP_RAISE_INVALID_OPCODE();
+
+                IEM_MC_FETCH_GREG_U64(u64Value, IEM_GET_MODRM_REG(pVCpu, bRm));
+                IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffDst, u64Value);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_16BIT:
+                /** @todo check this form.   */
+                return IEMOP_RAISE_INVALID_OPCODE();
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/*  Opcode 0x66 0x0f 0xc3 - invalid */
+/*  Opcode 0xf3 0x0f 0xc3 - invalid */
+/*  Opcode 0xf2 0x0f 0xc3 - invalid */
+
+
+/** Opcode      0x0f 0xc4 - pinsrw Pq, Ry/Mw,Ib */
+FNIEMOP_DEF(iemOp_pinsrw_Pq_RyMw_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PINSRW, pinsrw, Pq, Ey, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(uint64_t *,           pu64Dst,               0);
+        IEM_MC_ARG(uint16_t,             u16Src,                1);
+        IEM_MC_ARG_CONST(uint8_t,        bImmArg, /*=*/ bImm,   2);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+        IEM_MC_REF_MREG_U64(pu64Dst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pinsrw_u64, pu64Dst, u16Src, bImmArg);
+        IEM_MC_MODIFIED_MREG_BY_REF(pu64Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_ARG(uint64_t *,    pu64Dst,               0);
+        IEM_MC_ARG(uint16_t,      u16Src,                1);
+        IEM_MC_LOCAL(RTGCPTR,            GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm,   2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_REF_MREG_U64(pu64Dst, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pinsrw_u64, pu64Dst, u16Src, bImmArg);
+        IEM_MC_MODIFIED_MREG_BY_REF(pu64Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0xc4 - pinsrw Vdq, Ry/Mw,Ib */
+FNIEMOP_DEF(iemOp_pinsrw_Vdq_RyMw_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, PINSRW, pinsrw, Vq, Ey, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,          puDst,                 0);
+        IEM_MC_ARG(uint16_t,             u16Src,                1);
+        IEM_MC_ARG_CONST(uint8_t,        bImmArg, /*=*/ bImm,   2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pinsrw_u128, puDst, u16Src, bImmArg);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,   puDst,                 0);
+        IEM_MC_ARG(uint16_t,      u16Src,                1);
+        IEM_MC_LOCAL(RTGCPTR,            GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm,   2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+
+        IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_REF_XREG_U128(puDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pinsrw_u128, puDst, u16Src, bImmArg);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode 0xf3 0x0f 0xc4 - invalid */
+/*  Opcode 0xf2 0x0f 0xc4 - invalid */
+
+
+/** Opcode      0x0f 0xc5 - pextrw Gd, Nq, Ib */
+FNIEMOP_DEF(iemOp_pextrw_Gd_Nq_Ib)
+{
+    /*IEMOP_MNEMONIC3(RMI_REG, PEXTRW, pextrw, Gd, Nq, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);*/ /** @todo */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Greg32, MMX, imm8.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_LOCAL(uint16_t,              u16Dst);
+        IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Dst,  u16Dst,      0);
+        IEM_MC_ARG(uint64_t,                u64Src,                1);
+        IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/ bImm,   2);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+        IEM_MC_FETCH_MREG_U64(u64Src, IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pextrw_u64, pu16Dst, u64Src, bImmArg);
+        IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u16Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    /* No memory operand. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x66 0x0f 0xc5 - pextrw Gd, Udq, Ib */
+FNIEMOP_DEF(iemOp_pextrw_Gd_Udq_Ib)
+{
+    IEMOP_MNEMONIC3(RMI_REG, PEXTRW, pextrw, Gd, Ux, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Greg32, XMM, imm8.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_LOCAL(uint16_t,              u16Dst);
+        IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Dst,  u16Dst,      0);
+        IEM_MC_ARG(PCRTUINT128U,            puSrc,                 1);
+        IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/ bImm,   2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_pextrw_u128, pu16Dst, puSrc, bImmArg);
+        IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u16Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    /* No memory operand. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/*  Opcode 0xf3 0x0f 0xc5 - invalid */
+/*  Opcode 0xf2 0x0f 0xc5 - invalid */
+
+
+/** Opcode      0x0f 0xc6 - shufps Vps, Wps, Ib */
+FNIEMOP_DEF(iemOp_shufps_Vps_Wps_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, SHUFPS, shufps, Vps, Wps, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM, imm8.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+        IEM_MC_ARG_CONST(uint8_t,        bImmArg, /*=*/ bImm, 2);
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_shufps_u128, pDst, pSrc, bImmArg);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128], imm8.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_shufps_u128, pDst, pSrc, bImmArg);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode 0x66 0x0f 0xc6 - shufpd Vpd, Wpd, Ib */
+FNIEMOP_DEF(iemOp_shufpd_Vpd_Wpd_Ib)
+{
+    IEMOP_MNEMONIC3(RMI, SHUFPD, shufpd, Vpd, Wpd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * XMM, XMM, imm8.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(3, 0);
+        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+        IEM_MC_ARG_CONST(uint8_t,       bImmArg, /*=*/ bImm, 2);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_shufpd_u128, pDst, pSrc, bImmArg);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * XMM, [mem128], imm8.
+         */
+        IEM_MC_BEGIN(3, 2);
+        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 2);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_shufpd_u128, pDst, pSrc, bImmArg);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode 0xf3 0x0f 0xc6 - invalid */
+/*  Opcode 0xf2 0x0f 0xc6 - invalid */
+
+
+/** Opcode 0x0f 0xc7 !11/1. */
+FNIEMOP_DEF_1(iemOp_Grp9_cmpxchg8b_Mq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(cmpxchg8b, "cmpxchg8b Mq");
+
+    IEM_MC_BEGIN(4, 3);
+    IEM_MC_ARG(uint64_t *, pu64MemDst,     0);
+    IEM_MC_ARG(PRTUINT64U, pu64EaxEdx,     1);
+    IEM_MC_ARG(PRTUINT64U, pu64EbxEcx,     2);
+    IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 3);
+    IEM_MC_LOCAL(RTUINT64U, u64EaxEdx);
+    IEM_MC_LOCAL(RTUINT64U, u64EbxEcx);
+    IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING();
+    IEM_MC_MEM_MAP(pu64MemDst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+
+    IEM_MC_FETCH_GREG_U32(u64EaxEdx.s.Lo, X86_GREG_xAX);
+    IEM_MC_FETCH_GREG_U32(u64EaxEdx.s.Hi, X86_GREG_xDX);
+    IEM_MC_REF_LOCAL(pu64EaxEdx, u64EaxEdx);
+
+    IEM_MC_FETCH_GREG_U32(u64EbxEcx.s.Lo, X86_GREG_xBX);
+    IEM_MC_FETCH_GREG_U32(u64EbxEcx.s.Hi, X86_GREG_xCX);
+    IEM_MC_REF_LOCAL(pu64EbxEcx, u64EbxEcx);
+
+    IEM_MC_FETCH_EFLAGS(EFlags);
+    if (   !(pVCpu->iem.s.fDisregardLock)
+        && (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg8b_locked, pu64MemDst, pu64EaxEdx, pu64EbxEcx, pEFlags);
+    else
+        IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg8b, pu64MemDst, pu64EaxEdx, pu64EbxEcx, pEFlags);
+
+    IEM_MC_MEM_COMMIT_AND_UNMAP(pu64MemDst, IEM_ACCESS_DATA_RW);
+    IEM_MC_COMMIT_EFLAGS(EFlags);
+    IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF)
+        IEM_MC_STORE_GREG_U32(X86_GREG_xAX, u64EaxEdx.s.Lo);
+        IEM_MC_STORE_GREG_U32(X86_GREG_xDX, u64EaxEdx.s.Hi);
+    IEM_MC_ENDIF();
+    IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+    IEM_MC_END();
+}
+
+
+/** Opcode REX.W 0x0f 0xc7 !11/1. */
+FNIEMOP_DEF_1(iemOp_Grp9_cmpxchg16b_Mdq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(cmpxchg16b, "cmpxchg16b Mdq");
+    if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fMovCmpXchg16b)
+    {
+#if 0
+        RT_NOREF(bRm);
+        IEMOP_BITCH_ABOUT_STUB();
+        return VERR_IEM_INSTR_NOT_IMPLEMENTED;
+#else
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_ARG(PRTUINT128U, pu128MemDst,     0);
+        IEM_MC_ARG(PRTUINT128U, pu128RaxRdx,     1);
+        IEM_MC_ARG(PRTUINT128U, pu128RbxRcx,     2);
+        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 3);
+        IEM_MC_LOCAL(RTUINT128U, u128RaxRdx);
+        IEM_MC_LOCAL(RTUINT128U, u128RbxRcx);
+        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+        IEMOP_HLP_DONE_DECODING();
+        IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(GCPtrEffDst, 16);
+        IEM_MC_MEM_MAP(pu128MemDst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
+
+        IEM_MC_FETCH_GREG_U64(u128RaxRdx.s.Lo, X86_GREG_xAX);
+        IEM_MC_FETCH_GREG_U64(u128RaxRdx.s.Hi, X86_GREG_xDX);
+        IEM_MC_REF_LOCAL(pu128RaxRdx, u128RaxRdx);
+
+        IEM_MC_FETCH_GREG_U64(u128RbxRcx.s.Lo, X86_GREG_xBX);
+        IEM_MC_FETCH_GREG_U64(u128RbxRcx.s.Hi, X86_GREG_xCX);
+        IEM_MC_REF_LOCAL(pu128RbxRcx, u128RbxRcx);
+
+        IEM_MC_FETCH_EFLAGS(EFlags);
+# if defined(RT_ARCH_AMD64) || defined(RT_ARCH_ARM64)
+#  if defined(RT_ARCH_AMD64)
+        if (IEM_GET_HOST_CPU_FEATURES(pVCpu)->fMovCmpXchg16b)
+#  endif
+        {
+            if (   !(pVCpu->iem.s.fDisregardLock)
+                && (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_locked, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+            else
+                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+        }
+#  if defined(RT_ARCH_AMD64)
+        else
+#  endif
+# endif
+# if !defined(RT_ARCH_ARM64) /** @todo may need this for unaligned accesses... */
+        {
+            /* Note! The fallback for 32-bit systems and systems without CX16 is multiple
+                     accesses and not all all atomic, which works fine on in UNI CPU guest
+                     configuration (ignoring DMA).  If guest SMP is active we have no choice
+                     but to use a rendezvous callback here.  Sigh. */
+            if (pVCpu->CTX_SUFF(pVM)->cCpus == 1)
+                IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_fallback, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+            else
+            {
+                IEM_MC_CALL_CIMPL_4(iemCImpl_cmpxchg16b_fallback_rendezvous, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+                /* Does not get here, tail code is duplicated in iemCImpl_cmpxchg16b_fallback_rendezvous. */
+            }
+        }
+# endif
+
+        IEM_MC_MEM_COMMIT_AND_UNMAP(pu128MemDst, IEM_ACCESS_DATA_RW);
+        IEM_MC_COMMIT_EFLAGS(EFlags);
+        IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF)
+            IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u128RaxRdx.s.Lo);
+            IEM_MC_STORE_GREG_U64(X86_GREG_xDX, u128RaxRdx.s.Hi);
+        IEM_MC_ENDIF();
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+
+        IEM_MC_END();
+#endif
+    }
+    Log(("cmpxchg16b -> #UD\n"));
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+FNIEMOP_DEF_1(iemOp_Grp9_cmpxchg8bOr16b, uint8_t, bRm)
+{
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+        return FNIEMOP_CALL_1(iemOp_Grp9_cmpxchg16b_Mdq, bRm);
+    return FNIEMOP_CALL_1(iemOp_Grp9_cmpxchg8b_Mq, bRm);
+}
+
+
+/** Opcode 0x0f 0xc7 11/6. */
+FNIEMOP_DEF_1(iemOp_Grp9_rdrand_Rv, uint8_t, bRm)
+{
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fRdRand)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register destination. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(2, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                1);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdRand, iemAImpl_rdrand_u16, iemAImpl_rdrand_u16_fallback),
+                                         pu16Dst, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(2, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                1);
+
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdRand, iemAImpl_rdrand_u32, iemAImpl_rdrand_u32_fallback),
+                                         pu32Dst, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(2, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                1);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdRand, iemAImpl_rdrand_u64, iemAImpl_rdrand_u64_fallback),
+                                         pu64Dst, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    /* Register only. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/** Opcode 0x0f 0xc7 !11/6. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF_1(iemOp_Grp9_vmptrld_Mq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(vmptrld, "vmptrld");
+    IEMOP_HLP_IN_VMX_OPERATION("vmptrld", kVmxVDiag_Vmptrld);
+    IEMOP_HLP_VMX_INSTR("vmptrld", kVmxVDiag_Vmptrld);
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t, iEffSeg,     0);
+    IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_vmptrld, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_UD_STUB_1(iemOp_Grp9_vmptrld_Mq, uint8_t, bRm);
+#endif
+
+/** Opcode 0x66 0x0f 0xc7 !11/6. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF_1(iemOp_Grp9_vmclear_Mq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(vmclear, "vmclear");
+    IEMOP_HLP_IN_VMX_OPERATION("vmclear", kVmxVDiag_Vmclear);
+    IEMOP_HLP_VMX_INSTR("vmclear", kVmxVDiag_Vmclear);
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t, iEffSeg,     0);
+    IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_vmclear, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_UD_STUB_1(iemOp_Grp9_vmclear_Mq, uint8_t, bRm);
+#endif
+
+/** Opcode 0xf3 0x0f 0xc7 !11/6. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF_1(iemOp_Grp9_vmxon_Mq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(vmxon, "vmxon");
+    IEMOP_HLP_VMX_INSTR("vmxon", kVmxVDiag_Vmxon);
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t, iEffSeg,     0);
+    IEM_MC_ARG(RTGCPTR, GCPtrEffSrc, 1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+    IEMOP_HLP_DONE_DECODING();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_vmxon, iEffSeg, GCPtrEffSrc);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_UD_STUB_1(iemOp_Grp9_vmxon_Mq, uint8_t, bRm);
+#endif
+
+/** Opcode [0xf3] 0x0f 0xc7 !11/7. */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+FNIEMOP_DEF_1(iemOp_Grp9_vmptrst_Mq, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC(vmptrst, "vmptrst");
+    IEMOP_HLP_IN_VMX_OPERATION("vmptrst", kVmxVDiag_Vmptrst);
+    IEMOP_HLP_VMX_INSTR("vmptrst", kVmxVDiag_Vmptrst);
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t, iEffSeg,     0);
+    IEM_MC_ARG(RTGCPTR, GCPtrEffDst, 1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
+    IEMOP_HLP_DONE_DECODING_NO_SIZE_OP_REPZ_OR_REPNZ_PREFIXES();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_vmptrst, iEffSeg, GCPtrEffDst);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+#else
+FNIEMOP_UD_STUB_1(iemOp_Grp9_vmptrst_Mq, uint8_t, bRm);
+#endif
+
+/** Opcode 0x0f 0xc7 11/7. */
+FNIEMOP_DEF_1(iemOp_Grp9_rdseed_Rv, uint8_t, bRm)
+{
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fRdSeed)
+        return IEMOP_RAISE_INVALID_OPCODE();
+
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /* register destination. */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        switch (pVCpu->iem.s.enmEffOpSize)
+        {
+            case IEMMODE_16BIT:
+                IEM_MC_BEGIN(2, 0);
+                IEM_MC_ARG(uint16_t *,      pu16Dst,                0);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                1);
+
+                IEM_MC_REF_GREG_U16(pu16Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdSeed, iemAImpl_rdseed_u16, iemAImpl_rdseed_u16_fallback),
+                                         pu16Dst, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_32BIT:
+                IEM_MC_BEGIN(2, 0);
+                IEM_MC_ARG(uint32_t *,      pu32Dst,                0);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                1);
+
+                IEM_MC_REF_GREG_U32(pu32Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdSeed, iemAImpl_rdseed_u32, iemAImpl_rdseed_u32_fallback),
+                                         pu32Dst, pEFlags);
+
+                IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            case IEMMODE_64BIT:
+                IEM_MC_BEGIN(2, 0);
+                IEM_MC_ARG(uint64_t *,      pu64Dst,                0);
+                IEM_MC_ARG(uint32_t *,      pEFlags,                1);
+
+                IEM_MC_REF_GREG_U64(pu64Dst, IEM_GET_MODRM_RM(pVCpu, bRm));
+                IEM_MC_REF_EFLAGS(pEFlags);
+                IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fRdSeed, iemAImpl_rdseed_u64, iemAImpl_rdseed_u64_fallback),
+                                         pu64Dst, pEFlags);
+
+                IEM_MC_ADVANCE_RIP_AND_FINISH();
+                IEM_MC_END();
+                break;
+
+            IEM_NOT_REACHED_DEFAULT_CASE_RET();
+        }
+    }
+    /* Register only. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/**
+ * Group 9 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup9RegReg[] =
+{   /* pfx:  none,                          066h,                           0f3h,                           0f2h */
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /2 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /4 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /6 */ iemOp_Grp9_rdrand_Rv,          iemOp_Grp9_rdrand_Rv,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /7 */ iemOp_Grp9_rdseed_Rv,          iemOp_Grp9_rdseed_Rv,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup9RegReg) == 8*4);
+
+
+/**
+ * Group 9 jump table for memory variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnGroup9MemReg[] =
+{   /* pfx:  none,                          066h,                           0f3h,                           0f2h */
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /1 */ iemOp_Grp9_cmpxchg8bOr16b,     iemOp_Grp9_cmpxchg8bOr16b,      iemOp_Grp9_cmpxchg8bOr16b,      iemOp_Grp9_cmpxchg8bOr16b, /* see bs3-cpu-decoding-1 */
+    /* /2 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /4 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRM),
+    /* /6 */ iemOp_Grp9_vmptrld_Mq,         iemOp_Grp9_vmclear_Mq,          iemOp_Grp9_vmxon_Mq,            iemOp_InvalidWithRM,
+    /* /7 */ iemOp_Grp9_vmptrst_Mq,         iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+};
+AssertCompile(RT_ELEMENTS(g_apfnGroup9MemReg) == 8*4);
+
+
+/** Opcode 0x0f 0xc7. */
+FNIEMOP_DEF(iemOp_Grp9)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_RM(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnGroup9RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4
+                                                 + pVCpu->iem.s.idxPrefix], bRm);
+    /* memory, register */
+    return FNIEMOP_CALL_1(g_apfnGroup9MemReg[ IEM_GET_MODRM_REG_8(bRm) * 4
+                                             + pVCpu->iem.s.idxPrefix], bRm);
+}
+
+
+/**
+ * Common 'bswap register' helper.
+ */
+FNIEMOP_DEF_1(iemOpCommonBswapGReg, uint8_t, iReg)
+{
+    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+    switch (pVCpu->iem.s.enmEffOpSize)
+    {
+        case IEMMODE_16BIT:
+            IEM_MC_BEGIN(1, 0);
+            IEM_MC_ARG(uint32_t *,  pu32Dst, 0);
+            IEM_MC_REF_GREG_U32(pu32Dst, iReg);     /* Don't clear the high dword! */
+            IEM_MC_CALL_VOID_AIMPL_1(iemAImpl_bswap_u16, pu32Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_32BIT:
+            IEM_MC_BEGIN(1, 0);
+            IEM_MC_ARG(uint32_t *,  pu32Dst, 0);
+            IEM_MC_REF_GREG_U32(pu32Dst, iReg);
+            IEM_MC_CALL_VOID_AIMPL_1(iemAImpl_bswap_u32, pu32Dst);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pu32Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        case IEMMODE_64BIT:
+            IEM_MC_BEGIN(1, 0);
+            IEM_MC_ARG(uint64_t *,  pu64Dst, 0);
+            IEM_MC_REF_GREG_U64(pu64Dst, iReg);
+            IEM_MC_CALL_VOID_AIMPL_1(iemAImpl_bswap_u64, pu64Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+            break;
+
+        IEM_NOT_REACHED_DEFAULT_CASE_RET();
+    }
+}
+
+
+/** Opcode 0x0f 0xc8. */
+FNIEMOP_DEF(iemOp_bswap_rAX_r8)
+{
+    IEMOP_MNEMONIC(bswap_rAX_r8, "bswap rAX/r8");
+    /* Note! Intel manuals states that R8-R15 can be accessed by using a REX.X
+             prefix.  REX.B is the correct prefix it appears.  For a parallel
+             case, see iemOp_mov_AL_Ib and iemOp_mov_eAX_Iv. */
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xAX | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xc9. */
+FNIEMOP_DEF(iemOp_bswap_rCX_r9)
+{
+    IEMOP_MNEMONIC(bswap_rCX_r9, "bswap rCX/r9");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xCX | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xca. */
+FNIEMOP_DEF(iemOp_bswap_rDX_r10)
+{
+    IEMOP_MNEMONIC(bswap_rDX_r9, "bswap rDX/r10");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xDX | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xcb. */
+FNIEMOP_DEF(iemOp_bswap_rBX_r11)
+{
+    IEMOP_MNEMONIC(bswap_rBX_r9, "bswap rBX/r11");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xBX | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xcc. */
+FNIEMOP_DEF(iemOp_bswap_rSP_r12)
+{
+    IEMOP_MNEMONIC(bswap_rSP_r12, "bswap rSP/r12");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xSP | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xcd. */
+FNIEMOP_DEF(iemOp_bswap_rBP_r13)
+{
+    IEMOP_MNEMONIC(bswap_rBP_r13, "bswap rBP/r13");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xBP | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xce. */
+FNIEMOP_DEF(iemOp_bswap_rSI_r14)
+{
+    IEMOP_MNEMONIC(bswap_rSI_r14, "bswap rSI/r14");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xSI | pVCpu->iem.s.uRexB);
+}
+
+
+/** Opcode 0x0f 0xcf. */
+FNIEMOP_DEF(iemOp_bswap_rDI_r15)
+{
+    IEMOP_MNEMONIC(bswap_rDI_r15, "bswap rDI/r15");
+    IEMOP_HLP_MIN_486();
+    return FNIEMOP_CALL_1(iemOpCommonBswapGReg, X86_GREG_xDI | pVCpu->iem.s.uRexB);
+}
+
+
+/*  Opcode      0x0f 0xd0 - invalid */
+
+
+/** Opcode 0x66 0x0f 0xd0 - addsubpd Vpd, Wpd */
+FNIEMOP_DEF(iemOp_addsubpd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, ADDSUBPD, addsubpd, Vpd, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_addsubpd_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xd0 - invalid */
+
+
+/** Opcode 0xf2 0x0f 0xd0 - addsubps Vps, Wps */
+FNIEMOP_DEF(iemOp_addsubps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(RM, ADDSUBPS, addsubps, Vps, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse3Fp_FullFull_To_Full, iemAImpl_addsubps_u128);
+}
+
+
+
+/** Opcode      0x0f 0xd1 - psrlw Pq, Qq */
+FNIEMOP_DEF(iemOp_psrlw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSRLW, psrlw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psrlw_u64);
+}
+
+/** Opcode 0x66 0x0f 0xd1 - psrlw Vx, Wx */
+FNIEMOP_DEF(iemOp_psrlw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSRLW, psrlw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psrlw_u128);
+}
+
+/*  Opcode 0xf3 0x0f 0xd1 - invalid */
+/*  Opcode 0xf2 0x0f 0xd1 - invalid */
+
+/** Opcode      0x0f 0xd2 - psrld Pq, Qq */
+FNIEMOP_DEF(iemOp_psrld_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSRLD, psrld, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psrld_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xd2 - psrld Vx, Wx */
+FNIEMOP_DEF(iemOp_psrld_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSRLD, psrld, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psrld_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xd2 - invalid */
+/*  Opcode 0xf2 0x0f 0xd2 - invalid */
+
+/** Opcode      0x0f 0xd3 - psrlq Pq, Qq */
+FNIEMOP_DEF(iemOp_psrlq_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSRLQ, psrlq, Pq, Qq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psrlq_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xd3 - psrlq Vx, Wx */
+FNIEMOP_DEF(iemOp_psrlq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSRLQ, psrlq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psrlq_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xd3 - invalid */
+/*  Opcode 0xf2 0x0f 0xd3 - invalid */
+
+
+/** Opcode      0x0f 0xd4 - paddq Pq, Qq */
+FNIEMOP_DEF(iemOp_paddq_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PADDQ, paddq, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, iemAImpl_paddq_u64, IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2);
+}
+
+
+/** Opcode 0x66 0x0f 0xd4 - paddq Vx, Wx */
+FNIEMOP_DEF(iemOp_paddq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PADDQ, paddq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddq_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xd4 - invalid */
+/*  Opcode 0xf2 0x0f 0xd4 - invalid */
+
+/** Opcode      0x0f 0xd5 - pmullw Pq, Qq */
+FNIEMOP_DEF(iemOp_pmullw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMULLW, pmullw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pmullw_u64);
+}
+
+/** Opcode 0x66 0x0f 0xd5 - pmullw Vx, Wx */
+FNIEMOP_DEF(iemOp_pmullw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMULLW, pmullw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmullw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xd5 - invalid */
+/*  Opcode 0xf2 0x0f 0xd5 - invalid */
+
+/*  Opcode      0x0f 0xd6 - invalid */
+
+/**
+ * @opcode      0xd6
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_pcksclr_datamove
+ * @opxcpttype  none
+ * @optest      op1=-1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movq_Wq_Vq)
+{
+    IEMOP_MNEMONIC2(MR, MOVQ, movq, WqZxReg_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_RM(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0xd6
+ * @opcodesub   11 mr/reg
+ * @oppfx       f3
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_simdint_datamove
+ * @optest      op1=1 op2=2   -> op1=2   ftw=0xff
+ * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+ */
+FNIEMOP_DEF(iemOp_movq2dq_Vdq_Nq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_MNEMONIC2(RM_REG, MOVQ2DQ, movq2dq, VqZx_WO, Nq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_FETCH_MREG_U64(uSrc, IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_STORE_XREG_U64_ZX_U128(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udf30fd6mem
+     * @opcode      0xd6
+     * @opcodesub   !11 mr/reg
+     * @oppfx       f3
+     * @opunused    intel-modrm
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedDecode, bRm);
+}
+
+
+/**
+ * @opcode      0xd6
+ * @opcodesub   11 mr/reg
+ * @oppfx       f2
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_simdint_datamove
+ * @optest      op1=1 op2=2   -> op1=2   ftw=0xff
+ * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+ * @optest      op1=0 op2=0x1123456789abcdef -> op1=0x1123456789abcdef ftw=0xff
+ * @optest      op1=0 op2=0xfedcba9876543210 -> op1=0xfedcba9876543210 ftw=0xff
+ * @optest      op1=-42 op2=0xfedcba9876543210
+ *                  ->  op1=0xfedcba9876543210 ftw=0xff
+ */
+FNIEMOP_DEF(iemOp_movdq2q_Pq_Uq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_MNEMONIC2(RM_REG, MOVDQ2Q, movdq2q, Pq_WO, Uq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/);
+        IEM_MC_STORE_MREG_U64(IEM_GET_MODRM_REG_8(bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udf20fd6mem
+     * @opcode      0xd6
+     * @opcodesub   !11 mr/reg
+     * @oppfx       f2
+     * @opunused    intel-modrm
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedDecode, bRm);
+}
+
+
+/** Opcode      0x0f 0xd7 - pmovmskb Gd, Nq */
+FNIEMOP_DEF(iemOp_pmovmskb_Gd_Nq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    /* Docs says register only. */
+    if (IEM_IS_MODRM_REG_MODE(bRm)) /** @todo test that this is registers only. */
+    {
+        /* Note! Taking the lazy approch here wrt the high 32-bits of the GREG. */
+        IEMOP_MNEMONIC2(RM_REG, PMOVMSKB, pmovmskb, Gd, Nq, DISOPTYPE_MMX | DISOPTYPE_HARMLESS, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,              puDst, 0);
+        IEM_MC_ARG(uint64_t const *,        puSrc, 1);
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT_CHECK_SSE_OR_MMXEXT();
+        IEM_MC_PREPARE_FPU_USAGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_REF_GREG_U64(puDst,          IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_MREG_U64_CONST(puSrc,    IEM_GET_MODRM_RM_8(bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_pmovmskb_u64, puDst, puSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode 0x66 0x0f 0xd7 -  */
+FNIEMOP_DEF(iemOp_pmovmskb_Gd_Ux)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    /* Docs says register only. */
+    if (IEM_IS_MODRM_REG_MODE(bRm)) /** @todo test that this is registers only. */
+    {
+        /* Note! Taking the lazy approch here wrt the high 32-bits of the GREG. */
+        IEMOP_MNEMONIC2(RM_REG, PMOVMSKB, pmovmskb, Gd, Ux, DISOPTYPE_SSE | DISOPTYPE_HARMLESS, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(uint64_t *,              puDst, 0);
+        IEM_MC_ARG(PCRTUINT128U,            puSrc, 1);
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_PREPARE_SSE_USAGE();
+        IEM_MC_REF_GREG_U64(puDst,          IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,   IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_pmovmskb_u128, puDst, puSrc);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/*  Opcode 0xf3 0x0f 0xd7 - invalid */
+/*  Opcode 0xf2 0x0f 0xd7 - invalid */
+
+
+/** Opcode      0x0f 0xd8 - psubusb Pq, Qq */
+FNIEMOP_DEF(iemOp_psubusb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSUBUSB, psubusb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubusb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xd8 - psubusb Vx, Wx */
+FNIEMOP_DEF(iemOp_psubusb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSUBUSB, psubusb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubusb_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xd8 - invalid */
+/*  Opcode 0xf2 0x0f 0xd8 - invalid */
+
+/** Opcode      0x0f 0xd9 - psubusw Pq, Qq */
+FNIEMOP_DEF(iemOp_psubusw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSUBUSW, psubusw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubusw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xd9 - psubusw Vx, Wx */
+FNIEMOP_DEF(iemOp_psubusw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSUBUSW, psubusw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubusw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xd9 - invalid */
+/*  Opcode 0xf2 0x0f 0xd9 - invalid */
+
+/** Opcode      0x0f 0xda - pminub Pq, Qq */
+FNIEMOP_DEF(iemOp_pminub_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMINUB, pminub, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxSse_FullFull_To_Full, iemAImpl_pminub_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xda - pminub Vx, Wx */
+FNIEMOP_DEF(iemOp_pminub_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMINUB, pminub, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pminub_u128);
+}
+
+/*  Opcode 0xf3 0x0f 0xda - invalid */
+/*  Opcode 0xf2 0x0f 0xda - invalid */
+
+/** Opcode      0x0f 0xdb - pand Pq, Qq */
+FNIEMOP_DEF(iemOp_pand_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PAND, pand, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pand_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xdb - pand Vx, Wx */
+FNIEMOP_DEF(iemOp_pand_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PAND, pand, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pand_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xdb - invalid */
+/*  Opcode 0xf2 0x0f 0xdb - invalid */
+
+/** Opcode      0x0f 0xdc - paddusb Pq, Qq */
+FNIEMOP_DEF(iemOp_paddusb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PADDUSB, paddusb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddusb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xdc - paddusb Vx, Wx */
+FNIEMOP_DEF(iemOp_paddusb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PADDUSB, paddusb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddusb_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xdc - invalid */
+/*  Opcode 0xf2 0x0f 0xdc - invalid */
+
+/** Opcode      0x0f 0xdd - paddusw Pq, Qq */
+FNIEMOP_DEF(iemOp_paddusw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PADDUSW, paddusw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddusw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xdd - paddusw Vx, Wx */
+FNIEMOP_DEF(iemOp_paddusw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PADDUSW, paddusw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddusw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xdd - invalid */
+/*  Opcode 0xf2 0x0f 0xdd - invalid */
+
+/** Opcode      0x0f 0xde - pmaxub Pq, Qq */
+FNIEMOP_DEF(iemOp_pmaxub_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMAXUB, pmaxub, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxSse_FullFull_To_Full, iemAImpl_pmaxub_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xde - pmaxub Vx, W */
+FNIEMOP_DEF(iemOp_pmaxub_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMAXUB, pmaxub, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmaxub_u128);
+}
+
+/*  Opcode 0xf3 0x0f 0xde - invalid */
+/*  Opcode 0xf2 0x0f 0xde - invalid */
+
+
+/** Opcode      0x0f 0xdf - pandn Pq, Qq */
+FNIEMOP_DEF(iemOp_pandn_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PANDN, pandn, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pandn_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xdf - pandn Vx, Wx */
+FNIEMOP_DEF(iemOp_pandn_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PANDN, pandn, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pandn_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xdf - invalid */
+/*  Opcode 0xf2 0x0f 0xdf - invalid */
+
+/** Opcode      0x0f 0xe0 - pavgb Pq, Qq */
+FNIEMOP_DEF(iemOp_pavgb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PAVGB, pavgb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxSseOpt_FullFull_To_Full, iemAImpl_pavgb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xe0 - pavgb Vx, Wx */
+FNIEMOP_DEF(iemOp_pavgb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PAVGB, pavgb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_pavgb_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xe0 - invalid */
+/*  Opcode 0xf2 0x0f 0xe0 - invalid */
+
+/** Opcode      0x0f 0xe1 - psraw Pq, Qq */
+FNIEMOP_DEF(iemOp_psraw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSRAW, psraw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psraw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xe1 - psraw Vx, Wx */
+FNIEMOP_DEF(iemOp_psraw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSRAW, psraw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psraw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xe1 - invalid */
+/*  Opcode 0xf2 0x0f 0xe1 - invalid */
+
+/** Opcode      0x0f 0xe2 - psrad Pq, Qq */
+FNIEMOP_DEF(iemOp_psrad_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSRAD, psrad, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psrad_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xe2 - psrad Vx, Wx */
+FNIEMOP_DEF(iemOp_psrad_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSRAD, psrad, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psrad_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xe2 - invalid */
+/*  Opcode 0xf2 0x0f 0xe2 - invalid */
+
+/** Opcode      0x0f 0xe3 - pavgw Pq, Qq */
+FNIEMOP_DEF(iemOp_pavgw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PAVGW, pavgw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxSseOpt_FullFull_To_Full, iemAImpl_pavgw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xe3 - pavgw Vx, Wx */
+FNIEMOP_DEF(iemOp_pavgw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PAVGW, pavgw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_pavgw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xe3 - invalid */
+/*  Opcode 0xf2 0x0f 0xe3 - invalid */
+
+/** Opcode      0x0f 0xe4 - pmulhuw Pq, Qq */
+FNIEMOP_DEF(iemOp_pmulhuw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMULHUW, pmulhuw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxSseOpt_FullFull_To_Full, iemAImpl_pmulhuw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xe4 - pmulhuw Vx, Wx */
+FNIEMOP_DEF(iemOp_pmulhuw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMULHUW, pmulhuw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_pmulhuw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xe4 - invalid */
+/*  Opcode 0xf2 0x0f 0xe4 - invalid */
+
+/** Opcode      0x0f 0xe5 - pmulhw Pq, Qq */
+FNIEMOP_DEF(iemOp_pmulhw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMULHW, pmulhw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pmulhw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xe5 - pmulhw Vx, Wx */
+FNIEMOP_DEF(iemOp_pmulhw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMULHW, pmulhw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmulhw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xe5 - invalid */
+/*  Opcode 0xf2 0x0f 0xe5 - invalid */
+/*  Opcode      0x0f 0xe6 - invalid */
+
+
+/** Opcode 0x66 0x0f 0xe6 - cvttpd2dq Vx, Wpd */
+FNIEMOP_DEF(iemOp_cvttpd2dq_Vx_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, CVTTPD2DQ, cvttpd2dq, Vx, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvttpd2dq_u128);
+}
+
+
+/** Opcode 0xf3 0x0f 0xe6 - cvtdq2pd Vx, Wpd */
+FNIEMOP_DEF(iemOp_cvtdq2pd_Vx_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, CVTDQ2PD, cvtdq2pd, Vx, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtdq2pd_u128);
+}
+
+
+/** Opcode 0xf2 0x0f 0xe6 - cvtpd2dq Vx, Wpd */
+FNIEMOP_DEF(iemOp_cvtpd2dq_Vx_Wpd)
+{
+    IEMOP_MNEMONIC2(RM, CVTPD2DQ, cvtpd2dq, Vx, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Fp_FullFull_To_Full, iemAImpl_cvtpd2dq_u128);
+}
+
+
+/**
+ * @opcode      0xe7
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     sse
+ * @opgroup     og_sse1_cachect
+ * @opxcpttype  none
+ * @optest      op1=-1 op2=2  -> op1=2   ftw=0xff
+ * @optest      op1=0 op2=-42 -> op1=-42 ftw=0xff
+ */
+FNIEMOP_DEF(iemOp_movntq_Mq_Pq)
+{
+    IEMOP_MNEMONIC2(MR_MEM, MOVNTQ, movntq, Mq_WO, Pq, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        /* Register, memory. */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_MMX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_FPU_STATE_FOR_CHANGE();
+        IEM_MC_FPU_TO_MMX_MODE();
+
+        IEM_MC_FETCH_MREG_U64(uSrc, IEM_GET_MODRM_REG_8(bRm));
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    /**
+     * @opdone
+     * @opmnemonic  ud0fe7reg
+     * @opcode      0xe7
+     * @opcodesub   11 mr/reg
+     * @oppfx       none
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/**
+ * @opcode      0xe7
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     sse2
+ * @opgroup     og_sse2_cachect
+ * @opxcpttype  1
+ * @optest      op1=-1 op2=2  -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_movntdq_Mdq_Vdq)
+{
+    IEMOP_MNEMONIC2(MR_MEM, MOVNTDQ, movntdq, Mdq_WO, Vdq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        /* Register, memory. */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+
+        IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  ud660fe7reg
+     * @opcode      0xe7
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     sse
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/*  Opcode 0xf3 0x0f 0xe7 - invalid */
+/*  Opcode 0xf2 0x0f 0xe7 - invalid */
+
+
+/** Opcode      0x0f 0xe8 - psubsb Pq, Qq */
+FNIEMOP_DEF(iemOp_psubsb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSUBSB, psubsb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubsb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xe8 - psubsb Vx, Wx */
+FNIEMOP_DEF(iemOp_psubsb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSUBSB, psubsb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubsb_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xe8 - invalid */
+/*  Opcode 0xf2 0x0f 0xe8 - invalid */
+
+/** Opcode      0x0f 0xe9 - psubsw Pq, Qq */
+FNIEMOP_DEF(iemOp_psubsw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSUBSW, psubsw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubsw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xe9 - psubsw Vx, Wx */
+FNIEMOP_DEF(iemOp_psubsw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSUBSW, psubsw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubsw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xe9 - invalid */
+/*  Opcode 0xf2 0x0f 0xe9 - invalid */
+
+
+/** Opcode      0x0f 0xea - pminsw Pq, Qq */
+FNIEMOP_DEF(iemOp_pminsw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMINSW, pminsw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxSse_FullFull_To_Full, iemAImpl_pminsw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xea - pminsw Vx, Wx */
+FNIEMOP_DEF(iemOp_pminsw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMINSW, pminsw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pminsw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xea - invalid */
+/*  Opcode 0xf2 0x0f 0xea - invalid */
+
+
+/** Opcode      0x0f 0xeb - por Pq, Qq */
+FNIEMOP_DEF(iemOp_por_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, POR, por, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_por_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xeb - por Vx, Wx */
+FNIEMOP_DEF(iemOp_por_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, POR, por, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_por_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xeb - invalid */
+/*  Opcode 0xf2 0x0f 0xeb - invalid */
+
+/** Opcode      0x0f 0xec - paddsb Pq, Qq */
+FNIEMOP_DEF(iemOp_paddsb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PADDSB, paddsb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddsb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xec - paddsb Vx, Wx */
+FNIEMOP_DEF(iemOp_paddsb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PADDSB, paddsb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddsb_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xec - invalid */
+/*  Opcode 0xf2 0x0f 0xec - invalid */
+
+/** Opcode      0x0f 0xed - paddsw Pq, Qq */
+FNIEMOP_DEF(iemOp_paddsw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PADDSW, paddsw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddsw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xed - paddsw Vx, Wx */
+FNIEMOP_DEF(iemOp_paddsw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PADDSW, paddsw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddsw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xed - invalid */
+/*  Opcode 0xf2 0x0f 0xed - invalid */
+
+
+/** Opcode      0x0f 0xee - pmaxsw Pq, Qq */
+FNIEMOP_DEF(iemOp_pmaxsw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMAXSW, pmaxsw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxSse_FullFull_To_Full, iemAImpl_pmaxsw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xee - pmaxsw Vx, Wx */
+FNIEMOP_DEF(iemOp_pmaxsw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMAXSW, pmaxsw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmaxsw_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xee - invalid */
+/*  Opcode 0xf2 0x0f 0xee - invalid */
+
+
+/** Opcode      0x0f 0xef - pxor Pq, Qq */
+FNIEMOP_DEF(iemOp_pxor_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PXOR, pxor, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pxor_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xef - pxor Vx, Wx */
+FNIEMOP_DEF(iemOp_pxor_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PXOR, pxor, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pxor_u128);
+}
+
+
+/*  Opcode 0xf3 0x0f 0xef - invalid */
+/*  Opcode 0xf2 0x0f 0xef - invalid */
+
+/*  Opcode      0x0f 0xf0 - invalid */
+/*  Opcode 0x66 0x0f 0xf0 - invalid */
+
+
+/** Opcode 0xf2 0x0f 0xf0 - lddqu Vx, Mx */
+FNIEMOP_DEF(iemOp_lddqu_Vx_Mx)
+{
+    IEMOP_MNEMONIC2(RM_MEM, LDDQU, lddqu, Vdq_WO, Mx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register - (not implemented, assuming it raises \#UD).
+         */
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+        IEM_MC_MAYBE_RAISE_SSE3_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_SSE_STATE_FOR_CHANGE();
+        IEM_MC_FETCH_MEM_U128(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_XREG_U128(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode      0x0f 0xf1 - psllw Pq, Qq */
+FNIEMOP_DEF(iemOp_psllw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSLLW, psllw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psllw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xf1 - psllw Vx, Wx */
+FNIEMOP_DEF(iemOp_psllw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSLLW, psllw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psllw_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xf1 - invalid */
+
+/** Opcode      0x0f 0xf2 - pslld Pq, Qq */
+FNIEMOP_DEF(iemOp_pslld_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSLLD, pslld, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_pslld_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xf2 - pslld Vx, Wx */
+FNIEMOP_DEF(iemOp_pslld_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSLLD, pslld, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_pslld_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xf2 - invalid */
+
+/** Opcode      0x0f 0xf3 - psllq Pq, Qq */
+FNIEMOP_DEF(iemOp_psllq_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSLLQ, psllq, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmxOpt_FullFull_To_Full, iemAImpl_psllq_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xf3 - psllq Vx, Wx */
+FNIEMOP_DEF(iemOp_psllq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSLLQ, psllq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psllq_u128);
+}
+
+/*  Opcode 0xf2 0x0f 0xf3 - invalid */
+
+/** Opcode      0x0f 0xf4 - pmuludq Pq, Qq */
+FNIEMOP_DEF(iemOp_pmuludq_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMULUDQ, pmuludq, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pmuludq_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xf4 - pmuludq Vx, W */
+FNIEMOP_DEF(iemOp_pmuludq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMULUDQ, pmuludq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmuludq_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xf4 - invalid */
+
+/** Opcode      0x0f 0xf5 - pmaddwd Pq, Qq */
+FNIEMOP_DEF(iemOp_pmaddwd_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PMADDWD, pmaddwd, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, 0);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_pmaddwd_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xf5 - pmaddwd Vx, Wx */
+FNIEMOP_DEF(iemOp_pmaddwd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PMADDWD, pmaddwd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, 0);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_pmaddwd_u128);
+}
+
+/*  Opcode 0xf2 0x0f 0xf5 - invalid */
+
+/** Opcode      0x0f 0xf6 - psadbw Pq, Qq */
+FNIEMOP_DEF(iemOp_psadbw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSADBW, psadbw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmxSseOpt_FullFull_To_Full, iemAImpl_psadbw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xf6 - psadbw Vx, Wx */
+FNIEMOP_DEF(iemOp_psadbw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSADBW, psadbw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2Opt_FullFull_To_Full, iemAImpl_psadbw_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xf6 - invalid */
+
+/** Opcode      0x0f 0xf7 - maskmovq Pq, Nq */
+FNIEMOP_STUB(iemOp_maskmovq_Pq_Nq);
+/** Opcode 0x66 0x0f 0xf7 - maskmovdqu Vdq, Udq */
+FNIEMOP_STUB(iemOp_maskmovdqu_Vdq_Udq);
+/*  Opcode 0xf2 0x0f 0xf7 - invalid */
+
+
+/** Opcode      0x0f 0xf8 - psubb Pq, Qq */
+FNIEMOP_DEF(iemOp_psubb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSUBB, psubb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xf8 - psubb Vx, Wx */
+FNIEMOP_DEF(iemOp_psubb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSUBB, psubb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubb_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xf8 - invalid */
+
+
+/** Opcode      0x0f 0xf9 - psubw Pq, Qq */
+FNIEMOP_DEF(iemOp_psubw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSUBW, psubw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xf9 - psubw Vx, Wx */
+FNIEMOP_DEF(iemOp_psubw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSUBW, psubw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubw_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xf9 - invalid */
+
+
+/** Opcode      0x0f 0xfa - psubd Pq, Qq */
+FNIEMOP_DEF(iemOp_psubd_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSUBD, psubd, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_psubd_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xfa - psubd Vx, Wx */
+FNIEMOP_DEF(iemOp_psubd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSUBD, psubd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubd_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xfa - invalid */
+
+
+/** Opcode      0x0f 0xfb - psubq Pq, Qq */
+FNIEMOP_DEF(iemOp_psubq_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PSUBQ, psubq, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_2(iemOpCommonMmx_FullFull_To_Full_Ex, iemAImpl_psubq_u64, IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse2);
+}
+
+
+/** Opcode 0x66 0x0f 0xfb - psubq Vx, Wx */
+FNIEMOP_DEF(iemOp_psubq_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PSUBQ, psubq, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_psubq_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xfb - invalid */
+
+
+/** Opcode      0x0f 0xfc - paddb Pq, Qq */
+FNIEMOP_DEF(iemOp_paddb_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PADDB, paddb, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddb_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xfc - paddb Vx, Wx */
+FNIEMOP_DEF(iemOp_paddb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PADDB, paddb, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddb_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xfc - invalid */
+
+
+/** Opcode      0x0f 0xfd - paddw Pq, Qq */
+FNIEMOP_DEF(iemOp_paddw_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PADDW, paddw, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddw_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xfd - paddw Vx, Wx */
+FNIEMOP_DEF(iemOp_paddw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PADDW, paddw, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddw_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xfd - invalid */
+
+
+/** Opcode      0x0f 0xfe - paddd Pq, Qq */
+FNIEMOP_DEF(iemOp_paddd_Pq_Qq)
+{
+    IEMOP_MNEMONIC2(RM, PADDD, paddd, Pq, Qq, DISOPTYPE_HARMLESS | DISOPTYPE_MMX, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonMmx_FullFull_To_Full, iemAImpl_paddd_u64);
+}
+
+
+/** Opcode 0x66 0x0f 0xfe - paddd Vx, W */
+FNIEMOP_DEF(iemOp_paddd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(RM, PADDD, paddd, Vx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    return FNIEMOP_CALL_1(iemOpCommonSse2_FullFull_To_Full, iemAImpl_paddd_u128);
+}
+
+
+/*  Opcode 0xf2 0x0f 0xfe - invalid */
+
+
+/** Opcode **** 0x0f 0xff - UD0 */
+FNIEMOP_DEF(iemOp_ud0)
+{
+    IEMOP_MNEMONIC(ud0, "ud0");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        if (IEM_IS_MODRM_MEM_MODE(bRm))
+        {
+            RTGCPTR      GCPtrEff;
+            VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+            if (rcStrict != VINF_SUCCESS)
+                return rcStrict;
+        }
+#endif
+        IEMOP_HLP_DONE_DECODING();
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+
+/**
+ * Two byte opcode map, first byte 0x0f.
+ *
+ * @remarks The g_apfnVexMap1 table is currently a subset of this one, so please
+ *          check if it needs updating as well when making changes.
+ */
+IEM_STATIC const PFNIEMOP g_apfnTwoByteMap[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  IEMOP_X4(iemOp_Grp6),
+    /* 0x01 */  IEMOP_X4(iemOp_Grp7),
+    /* 0x02 */  IEMOP_X4(iemOp_lar_Gv_Ew),
+    /* 0x03 */  IEMOP_X4(iemOp_lsl_Gv_Ew),
+    /* 0x04 */  IEMOP_X4(iemOp_Invalid),
+    /* 0x05 */  IEMOP_X4(iemOp_syscall),
+    /* 0x06 */  IEMOP_X4(iemOp_clts),
+    /* 0x07 */  IEMOP_X4(iemOp_sysret),
+    /* 0x08 */  IEMOP_X4(iemOp_invd),
+    /* 0x09 */  IEMOP_X4(iemOp_wbinvd),
+    /* 0x0a */  IEMOP_X4(iemOp_Invalid),
+    /* 0x0b */  IEMOP_X4(iemOp_ud2),
+    /* 0x0c */  IEMOP_X4(iemOp_Invalid),
+    /* 0x0d */  IEMOP_X4(iemOp_nop_Ev_GrpP),
+    /* 0x0e */  IEMOP_X4(iemOp_femms),
+    /* 0x0f */  IEMOP_X4(iemOp_3Dnow),
+
+    /* 0x10 */  iemOp_movups_Vps_Wps,       iemOp_movupd_Vpd_Wpd,       iemOp_movss_Vss_Wss,        iemOp_movsd_Vsd_Wsd,
+    /* 0x11 */  iemOp_movups_Wps_Vps,       iemOp_movupd_Wpd_Vpd,       iemOp_movss_Wss_Vss,        iemOp_movsd_Wsd_Vsd,
+    /* 0x12 */  iemOp_movlps_Vq_Mq__movhlps, iemOp_movlpd_Vq_Mq,        iemOp_movsldup_Vdq_Wdq,     iemOp_movddup_Vdq_Wdq,
+    /* 0x13 */  iemOp_movlps_Mq_Vq,         iemOp_movlpd_Mq_Vq,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x14 */  iemOp_unpcklps_Vx_Wx,       iemOp_unpcklpd_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x15 */  iemOp_unpckhps_Vx_Wx,       iemOp_unpckhpd_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x16 */  iemOp_movhps_Vdq_Mq__movlhps_Vdq_Uq, iemOp_movhpd_Vdq_Mq, iemOp_movshdup_Vdq_Wdq,   iemOp_InvalidNeedRM,
+    /* 0x17 */  iemOp_movhps_Mq_Vq,         iemOp_movhpd_Mq_Vq,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x18 */  IEMOP_X4(iemOp_prefetch_Grp16),
+    /* 0x19 */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1a */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1b */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1c */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1d */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1e */  IEMOP_X4(iemOp_nop_Ev),
+    /* 0x1f */  IEMOP_X4(iemOp_nop_Ev),
+
+    /* 0x20 */  iemOp_mov_Rd_Cd,            iemOp_mov_Rd_Cd,            iemOp_mov_Rd_Cd,            iemOp_mov_Rd_Cd,
+    /* 0x21 */  iemOp_mov_Rd_Dd,            iemOp_mov_Rd_Dd,            iemOp_mov_Rd_Dd,            iemOp_mov_Rd_Dd,
+    /* 0x22 */  iemOp_mov_Cd_Rd,            iemOp_mov_Cd_Rd,            iemOp_mov_Cd_Rd,            iemOp_mov_Cd_Rd,
+    /* 0x23 */  iemOp_mov_Dd_Rd,            iemOp_mov_Dd_Rd,            iemOp_mov_Dd_Rd,            iemOp_mov_Dd_Rd,
+    /* 0x24 */  iemOp_mov_Rd_Td,            iemOp_mov_Rd_Td,            iemOp_mov_Rd_Td,            iemOp_mov_Rd_Td,
+    /* 0x25 */  iemOp_Invalid,              iemOp_Invalid,              iemOp_Invalid,              iemOp_Invalid,
+    /* 0x26 */  iemOp_mov_Td_Rd,            iemOp_mov_Td_Rd,            iemOp_mov_Td_Rd,            iemOp_mov_Td_Rd,
+    /* 0x27 */  iemOp_Invalid,              iemOp_Invalid,              iemOp_Invalid,              iemOp_Invalid,
+    /* 0x28 */  iemOp_movaps_Vps_Wps,       iemOp_movapd_Vpd_Wpd,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x29 */  iemOp_movaps_Wps_Vps,       iemOp_movapd_Wpd_Vpd,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2a */  iemOp_cvtpi2ps_Vps_Qpi,     iemOp_cvtpi2pd_Vpd_Qpi,     iemOp_cvtsi2ss_Vss_Ey,      iemOp_cvtsi2sd_Vsd_Ey,
+    /* 0x2b */  iemOp_movntps_Mps_Vps,      iemOp_movntpd_Mpd_Vpd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2c */  iemOp_cvttps2pi_Ppi_Wps,    iemOp_cvttpd2pi_Ppi_Wpd,    iemOp_cvttss2si_Gy_Wss,     iemOp_cvttsd2si_Gy_Wsd,
+    /* 0x2d */  iemOp_cvtps2pi_Ppi_Wps,     iemOp_cvtpd2pi_Qpi_Wpd,     iemOp_cvtss2si_Gy_Wss,      iemOp_cvtsd2si_Gy_Wsd,
+    /* 0x2e */  iemOp_ucomiss_Vss_Wss,      iemOp_ucomisd_Vsd_Wsd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2f */  iemOp_comiss_Vss_Wss,       iemOp_comisd_Vsd_Wsd,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x30 */  IEMOP_X4(iemOp_wrmsr),
+    /* 0x31 */  IEMOP_X4(iemOp_rdtsc),
+    /* 0x32 */  IEMOP_X4(iemOp_rdmsr),
+    /* 0x33 */  IEMOP_X4(iemOp_rdpmc),
+    /* 0x34 */  IEMOP_X4(iemOp_sysenter),
+    /* 0x35 */  IEMOP_X4(iemOp_sysexit),
+    /* 0x36 */  IEMOP_X4(iemOp_Invalid),
+    /* 0x37 */  IEMOP_X4(iemOp_getsec),
+    /* 0x38 */  IEMOP_X4(iemOp_3byte_Esc_0f_38),
+    /* 0x39 */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRM),
+    /* 0x3a */  IEMOP_X4(iemOp_3byte_Esc_0f_3a),
+    /* 0x3b */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRMImm8),
+    /* 0x3c */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRM),
+    /* 0x3d */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRM),
+    /* 0x3e */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRMImm8),
+    /* 0x3f */  IEMOP_X4(iemOp_InvalidNeed3ByteEscRMImm8),
+
+    /* 0x40 */  IEMOP_X4(iemOp_cmovo_Gv_Ev),
+    /* 0x41 */  IEMOP_X4(iemOp_cmovno_Gv_Ev),
+    /* 0x42 */  IEMOP_X4(iemOp_cmovc_Gv_Ev),
+    /* 0x43 */  IEMOP_X4(iemOp_cmovnc_Gv_Ev),
+    /* 0x44 */  IEMOP_X4(iemOp_cmove_Gv_Ev),
+    /* 0x45 */  IEMOP_X4(iemOp_cmovne_Gv_Ev),
+    /* 0x46 */  IEMOP_X4(iemOp_cmovbe_Gv_Ev),
+    /* 0x47 */  IEMOP_X4(iemOp_cmovnbe_Gv_Ev),
+    /* 0x48 */  IEMOP_X4(iemOp_cmovs_Gv_Ev),
+    /* 0x49 */  IEMOP_X4(iemOp_cmovns_Gv_Ev),
+    /* 0x4a */  IEMOP_X4(iemOp_cmovp_Gv_Ev),
+    /* 0x4b */  IEMOP_X4(iemOp_cmovnp_Gv_Ev),
+    /* 0x4c */  IEMOP_X4(iemOp_cmovl_Gv_Ev),
+    /* 0x4d */  IEMOP_X4(iemOp_cmovnl_Gv_Ev),
+    /* 0x4e */  IEMOP_X4(iemOp_cmovle_Gv_Ev),
+    /* 0x4f */  IEMOP_X4(iemOp_cmovnle_Gv_Ev),
+
+    /* 0x50 */  iemOp_movmskps_Gy_Ups,      iemOp_movmskpd_Gy_Upd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x51 */  iemOp_sqrtps_Vps_Wps,       iemOp_sqrtpd_Vpd_Wpd,       iemOp_sqrtss_Vss_Wss,       iemOp_sqrtsd_Vsd_Wsd,
+    /* 0x52 */  iemOp_rsqrtps_Vps_Wps,      iemOp_InvalidNeedRM,        iemOp_rsqrtss_Vss_Wss,      iemOp_InvalidNeedRM,
+    /* 0x53 */  iemOp_rcpps_Vps_Wps,        iemOp_InvalidNeedRM,        iemOp_rcpss_Vss_Wss,        iemOp_InvalidNeedRM,
+    /* 0x54 */  iemOp_andps_Vps_Wps,        iemOp_andpd_Vpd_Wpd,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x55 */  iemOp_andnps_Vps_Wps,       iemOp_andnpd_Vpd_Wpd,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x56 */  iemOp_orps_Vps_Wps,         iemOp_orpd_Vpd_Wpd,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x57 */  iemOp_xorps_Vps_Wps,        iemOp_xorpd_Vpd_Wpd,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x58 */  iemOp_addps_Vps_Wps,        iemOp_addpd_Vpd_Wpd,        iemOp_addss_Vss_Wss,        iemOp_addsd_Vsd_Wsd,
+    /* 0x59 */  iemOp_mulps_Vps_Wps,        iemOp_mulpd_Vpd_Wpd,        iemOp_mulss_Vss_Wss,        iemOp_mulsd_Vsd_Wsd,
+    /* 0x5a */  iemOp_cvtps2pd_Vpd_Wps,     iemOp_cvtpd2ps_Vps_Wpd,     iemOp_cvtss2sd_Vsd_Wss,     iemOp_cvtsd2ss_Vss_Wsd,
+    /* 0x5b */  iemOp_cvtdq2ps_Vps_Wdq,     iemOp_cvtps2dq_Vdq_Wps,     iemOp_cvttps2dq_Vdq_Wps,    iemOp_InvalidNeedRM,
+    /* 0x5c */  iemOp_subps_Vps_Wps,        iemOp_subpd_Vpd_Wpd,        iemOp_subss_Vss_Wss,        iemOp_subsd_Vsd_Wsd,
+    /* 0x5d */  iemOp_minps_Vps_Wps,        iemOp_minpd_Vpd_Wpd,        iemOp_minss_Vss_Wss,        iemOp_minsd_Vsd_Wsd,
+    /* 0x5e */  iemOp_divps_Vps_Wps,        iemOp_divpd_Vpd_Wpd,        iemOp_divss_Vss_Wss,        iemOp_divsd_Vsd_Wsd,
+    /* 0x5f */  iemOp_maxps_Vps_Wps,        iemOp_maxpd_Vpd_Wpd,        iemOp_maxss_Vss_Wss,        iemOp_maxsd_Vsd_Wsd,
+
+    /* 0x60 */  iemOp_punpcklbw_Pq_Qd,      iemOp_punpcklbw_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x61 */  iemOp_punpcklwd_Pq_Qd,      iemOp_punpcklwd_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x62 */  iemOp_punpckldq_Pq_Qd,      iemOp_punpckldq_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x63 */  iemOp_packsswb_Pq_Qq,       iemOp_packsswb_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x64 */  iemOp_pcmpgtb_Pq_Qq,        iemOp_pcmpgtb_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x65 */  iemOp_pcmpgtw_Pq_Qq,        iemOp_pcmpgtw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x66 */  iemOp_pcmpgtd_Pq_Qq,        iemOp_pcmpgtd_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x67 */  iemOp_packuswb_Pq_Qq,       iemOp_packuswb_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x68 */  iemOp_punpckhbw_Pq_Qq,      iemOp_punpckhbw_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x69 */  iemOp_punpckhwd_Pq_Qq,      iemOp_punpckhwd_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6a */  iemOp_punpckhdq_Pq_Qq,      iemOp_punpckhdq_Vx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6b */  iemOp_packssdw_Pq_Qd,       iemOp_packssdw_Vx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6c */  iemOp_InvalidNeedRM,        iemOp_punpcklqdq_Vx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6d */  iemOp_InvalidNeedRM,        iemOp_punpckhqdq_Vx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6e */  iemOp_movd_q_Pd_Ey,         iemOp_movd_q_Vy_Ey,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6f */  iemOp_movq_Pq_Qq,           iemOp_movdqa_Vdq_Wdq,       iemOp_movdqu_Vdq_Wdq,       iemOp_InvalidNeedRM,
+
+    /* 0x70 */  iemOp_pshufw_Pq_Qq_Ib,      iemOp_pshufd_Vx_Wx_Ib,      iemOp_pshufhw_Vx_Wx_Ib,     iemOp_pshuflw_Vx_Wx_Ib,
+    /* 0x71 */  IEMOP_X4(iemOp_Grp12),
+    /* 0x72 */  IEMOP_X4(iemOp_Grp13),
+    /* 0x73 */  IEMOP_X4(iemOp_Grp14),
+    /* 0x74 */  iemOp_pcmpeqb_Pq_Qq,        iemOp_pcmpeqb_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x75 */  iemOp_pcmpeqw_Pq_Qq,        iemOp_pcmpeqw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x76 */  iemOp_pcmpeqd_Pq_Qq,        iemOp_pcmpeqd_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x77 */  iemOp_emms,                 iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x78 */  iemOp_vmread_Ey_Gy,         iemOp_AmdGrp17,             iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x79 */  iemOp_vmwrite_Gy_Ey,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x7a */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x7b */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x7c */  iemOp_InvalidNeedRM,        iemOp_haddpd_Vpd_Wpd,       iemOp_InvalidNeedRM,        iemOp_haddps_Vps_Wps,
+    /* 0x7d */  iemOp_InvalidNeedRM,        iemOp_hsubpd_Vpd_Wpd,       iemOp_InvalidNeedRM,        iemOp_hsubps_Vps_Wps,
+    /* 0x7e */  iemOp_movd_q_Ey_Pd,         iemOp_movd_q_Ey_Vy,         iemOp_movq_Vq_Wq,           iemOp_InvalidNeedRM,
+    /* 0x7f */  iemOp_movq_Qq_Pq,           iemOp_movdqa_Wx_Vx,         iemOp_movdqu_Wx_Vx,         iemOp_InvalidNeedRM,
+
+    /* 0x80 */  IEMOP_X4(iemOp_jo_Jv),
+    /* 0x81 */  IEMOP_X4(iemOp_jno_Jv),
+    /* 0x82 */  IEMOP_X4(iemOp_jc_Jv),
+    /* 0x83 */  IEMOP_X4(iemOp_jnc_Jv),
+    /* 0x84 */  IEMOP_X4(iemOp_je_Jv),
+    /* 0x85 */  IEMOP_X4(iemOp_jne_Jv),
+    /* 0x86 */  IEMOP_X4(iemOp_jbe_Jv),
+    /* 0x87 */  IEMOP_X4(iemOp_jnbe_Jv),
+    /* 0x88 */  IEMOP_X4(iemOp_js_Jv),
+    /* 0x89 */  IEMOP_X4(iemOp_jns_Jv),
+    /* 0x8a */  IEMOP_X4(iemOp_jp_Jv),
+    /* 0x8b */  IEMOP_X4(iemOp_jnp_Jv),
+    /* 0x8c */  IEMOP_X4(iemOp_jl_Jv),
+    /* 0x8d */  IEMOP_X4(iemOp_jnl_Jv),
+    /* 0x8e */  IEMOP_X4(iemOp_jle_Jv),
+    /* 0x8f */  IEMOP_X4(iemOp_jnle_Jv),
+
+    /* 0x90 */  IEMOP_X4(iemOp_seto_Eb),
+    /* 0x91 */  IEMOP_X4(iemOp_setno_Eb),
+    /* 0x92 */  IEMOP_X4(iemOp_setc_Eb),
+    /* 0x93 */  IEMOP_X4(iemOp_setnc_Eb),
+    /* 0x94 */  IEMOP_X4(iemOp_sete_Eb),
+    /* 0x95 */  IEMOP_X4(iemOp_setne_Eb),
+    /* 0x96 */  IEMOP_X4(iemOp_setbe_Eb),
+    /* 0x97 */  IEMOP_X4(iemOp_setnbe_Eb),
+    /* 0x98 */  IEMOP_X4(iemOp_sets_Eb),
+    /* 0x99 */  IEMOP_X4(iemOp_setns_Eb),
+    /* 0x9a */  IEMOP_X4(iemOp_setp_Eb),
+    /* 0x9b */  IEMOP_X4(iemOp_setnp_Eb),
+    /* 0x9c */  IEMOP_X4(iemOp_setl_Eb),
+    /* 0x9d */  IEMOP_X4(iemOp_setnl_Eb),
+    /* 0x9e */  IEMOP_X4(iemOp_setle_Eb),
+    /* 0x9f */  IEMOP_X4(iemOp_setnle_Eb),
+
+    /* 0xa0 */  IEMOP_X4(iemOp_push_fs),
+    /* 0xa1 */  IEMOP_X4(iemOp_pop_fs),
+    /* 0xa2 */  IEMOP_X4(iemOp_cpuid),
+    /* 0xa3 */  IEMOP_X4(iemOp_bt_Ev_Gv),
+    /* 0xa4 */  IEMOP_X4(iemOp_shld_Ev_Gv_Ib),
+    /* 0xa5 */  IEMOP_X4(iemOp_shld_Ev_Gv_CL),
+    /* 0xa6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa8 */  IEMOP_X4(iemOp_push_gs),
+    /* 0xa9 */  IEMOP_X4(iemOp_pop_gs),
+    /* 0xaa */  IEMOP_X4(iemOp_rsm),
+    /* 0xab */  IEMOP_X4(iemOp_bts_Ev_Gv),
+    /* 0xac */  IEMOP_X4(iemOp_shrd_Ev_Gv_Ib),
+    /* 0xad */  IEMOP_X4(iemOp_shrd_Ev_Gv_CL),
+    /* 0xae */  IEMOP_X4(iemOp_Grp15),
+    /* 0xaf */  IEMOP_X4(iemOp_imul_Gv_Ev),
+
+    /* 0xb0 */  IEMOP_X4(iemOp_cmpxchg_Eb_Gb),
+    /* 0xb1 */  IEMOP_X4(iemOp_cmpxchg_Ev_Gv),
+    /* 0xb2 */  IEMOP_X4(iemOp_lss_Gv_Mp),
+    /* 0xb3 */  IEMOP_X4(iemOp_btr_Ev_Gv),
+    /* 0xb4 */  IEMOP_X4(iemOp_lfs_Gv_Mp),
+    /* 0xb5 */  IEMOP_X4(iemOp_lgs_Gv_Mp),
+    /* 0xb6 */  IEMOP_X4(iemOp_movzx_Gv_Eb),
+    /* 0xb7 */  IEMOP_X4(iemOp_movzx_Gv_Ew),
+    /* 0xb8 */  iemOp_jmpe,                 iemOp_InvalidNeedRM,        iemOp_popcnt_Gv_Ev,         iemOp_InvalidNeedRM,
+    /* 0xb9 */  IEMOP_X4(iemOp_Grp10),
+    /* 0xba */  IEMOP_X4(iemOp_Grp8),
+    /* 0xbb */  IEMOP_X4(iemOp_btc_Ev_Gv), // 0xf3?
+    /* 0xbc */  iemOp_bsf_Gv_Ev,            iemOp_bsf_Gv_Ev,            iemOp_tzcnt_Gv_Ev,          iemOp_bsf_Gv_Ev,
+    /* 0xbd */  iemOp_bsr_Gv_Ev,            iemOp_bsr_Gv_Ev,            iemOp_lzcnt_Gv_Ev,          iemOp_bsr_Gv_Ev,
+    /* 0xbe */  IEMOP_X4(iemOp_movsx_Gv_Eb),
+    /* 0xbf */  IEMOP_X4(iemOp_movsx_Gv_Ew),
+
+    /* 0xc0 */  IEMOP_X4(iemOp_xadd_Eb_Gb),
+    /* 0xc1 */  IEMOP_X4(iemOp_xadd_Ev_Gv),
+    /* 0xc2 */  iemOp_cmpps_Vps_Wps_Ib,     iemOp_cmppd_Vpd_Wpd_Ib,     iemOp_cmpss_Vss_Wss_Ib,     iemOp_cmpsd_Vsd_Wsd_Ib,
+    /* 0xc3 */  iemOp_movnti_My_Gy,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xc4 */  iemOp_pinsrw_Pq_RyMw_Ib,    iemOp_pinsrw_Vdq_RyMw_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xc5 */  iemOp_pextrw_Gd_Nq_Ib,      iemOp_pextrw_Gd_Udq_Ib,     iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xc6 */  iemOp_shufps_Vps_Wps_Ib,    iemOp_shufpd_Vpd_Wpd_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xc7 */  IEMOP_X4(iemOp_Grp9),
+    /* 0xc8 */  IEMOP_X4(iemOp_bswap_rAX_r8),
+    /* 0xc9 */  IEMOP_X4(iemOp_bswap_rCX_r9),
+    /* 0xca */  IEMOP_X4(iemOp_bswap_rDX_r10),
+    /* 0xcb */  IEMOP_X4(iemOp_bswap_rBX_r11),
+    /* 0xcc */  IEMOP_X4(iemOp_bswap_rSP_r12),
+    /* 0xcd */  IEMOP_X4(iemOp_bswap_rBP_r13),
+    /* 0xce */  IEMOP_X4(iemOp_bswap_rSI_r14),
+    /* 0xcf */  IEMOP_X4(iemOp_bswap_rDI_r15),
+
+    /* 0xd0 */  iemOp_InvalidNeedRM,        iemOp_addsubpd_Vpd_Wpd,     iemOp_InvalidNeedRM,        iemOp_addsubps_Vps_Wps,
+    /* 0xd1 */  iemOp_psrlw_Pq_Qq,          iemOp_psrlw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd2 */  iemOp_psrld_Pq_Qq,          iemOp_psrld_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd3 */  iemOp_psrlq_Pq_Qq,          iemOp_psrlq_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd4 */  iemOp_paddq_Pq_Qq,          iemOp_paddq_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd5 */  iemOp_pmullw_Pq_Qq,         iemOp_pmullw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd6 */  iemOp_InvalidNeedRM,        iemOp_movq_Wq_Vq,           iemOp_movq2dq_Vdq_Nq,       iemOp_movdq2q_Pq_Uq,
+    /* 0xd7 */  iemOp_pmovmskb_Gd_Nq,       iemOp_pmovmskb_Gd_Ux,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd8 */  iemOp_psubusb_Pq_Qq,        iemOp_psubusb_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd9 */  iemOp_psubusw_Pq_Qq,        iemOp_psubusw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xda */  iemOp_pminub_Pq_Qq,         iemOp_pminub_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdb */  iemOp_pand_Pq_Qq,           iemOp_pand_Vx_Wx,           iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdc */  iemOp_paddusb_Pq_Qq,        iemOp_paddusb_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdd */  iemOp_paddusw_Pq_Qq,        iemOp_paddusw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xde */  iemOp_pmaxub_Pq_Qq,         iemOp_pmaxub_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdf */  iemOp_pandn_Pq_Qq,          iemOp_pandn_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xe0 */  iemOp_pavgb_Pq_Qq,          iemOp_pavgb_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe1 */  iemOp_psraw_Pq_Qq,          iemOp_psraw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe2 */  iemOp_psrad_Pq_Qq,          iemOp_psrad_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe3 */  iemOp_pavgw_Pq_Qq,          iemOp_pavgw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe4 */  iemOp_pmulhuw_Pq_Qq,        iemOp_pmulhuw_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe5 */  iemOp_pmulhw_Pq_Qq,         iemOp_pmulhw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe6 */  iemOp_InvalidNeedRM,        iemOp_cvttpd2dq_Vx_Wpd,     iemOp_cvtdq2pd_Vx_Wpd,      iemOp_cvtpd2dq_Vx_Wpd,
+    /* 0xe7 */  iemOp_movntq_Mq_Pq,         iemOp_movntdq_Mdq_Vdq,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe8 */  iemOp_psubsb_Pq_Qq,         iemOp_psubsb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe9 */  iemOp_psubsw_Pq_Qq,         iemOp_psubsw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xea */  iemOp_pminsw_Pq_Qq,         iemOp_pminsw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xeb */  iemOp_por_Pq_Qq,            iemOp_por_Vx_Wx,            iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xec */  iemOp_paddsb_Pq_Qq,         iemOp_paddsb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xed */  iemOp_paddsw_Pq_Qq,         iemOp_paddsw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xee */  iemOp_pmaxsw_Pq_Qq,         iemOp_pmaxsw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xef */  iemOp_pxor_Pq_Qq,           iemOp_pxor_Vx_Wx,           iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xf0 */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_lddqu_Vx_Mx,
+    /* 0xf1 */  iemOp_psllw_Pq_Qq,          iemOp_psllw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf2 */  iemOp_pslld_Pq_Qq,          iemOp_pslld_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf3 */  iemOp_psllq_Pq_Qq,          iemOp_psllq_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf4 */  iemOp_pmuludq_Pq_Qq,        iemOp_pmuludq_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf5 */  iemOp_pmaddwd_Pq_Qq,        iemOp_pmaddwd_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf6 */  iemOp_psadbw_Pq_Qq,         iemOp_psadbw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf7 */  iemOp_maskmovq_Pq_Nq,       iemOp_maskmovdqu_Vdq_Udq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf8 */  iemOp_psubb_Pq_Qq,          iemOp_psubb_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf9 */  iemOp_psubw_Pq_Qq,          iemOp_psubw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfa */  iemOp_psubd_Pq_Qq,          iemOp_psubd_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfb */  iemOp_psubq_Pq_Qq,          iemOp_psubq_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfc */  iemOp_paddb_Pq_Qq,          iemOp_paddb_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfd */  iemOp_paddw_Pq_Qq,          iemOp_paddw_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfe */  iemOp_paddd_Pq_Qq,          iemOp_paddd_Vx_Wx,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xff */  IEMOP_X4(iemOp_ud0),
+};
+AssertCompile(RT_ELEMENTS(g_apfnTwoByteMap) == 1024);
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h
new file mode 100644
index 00000000..f60f30ea
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap1.cpp.h
@@ -0,0 +1,5642 @@
+/* $Id: IEMAllInstructionsVexMap1.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ *
+ * @remarks IEMAllInstructionsTwoByte0f.cpp.h is a legacy mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/** @name VEX Opcode Map 1
+ * @{
+ */
+
+/**
+ * Common worker for AVX2 instructions on the forms:
+ *     - vpxxx    xmm0, xmm1, xmm2/mem128
+ *     - vpxxx    ymm0, ymm1, ymm2/mem256
+ *
+ * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit.
+ */
+FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, PCIEMOPMEDIAF3, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_BEGIN(4, 3);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,  uDst,  1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1, 2);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2, 3);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_FETCH_YREG_U256(uSrc1,   IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_FETCH_YREG_U256(uSrc2,   IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_AVX_AIMPL_3(pImpl->pfnU256, puDst, puSrc1, puSrc2);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_BEGIN(4, 0);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG(PRTUINT128U,          puDst,  1);
+            IEM_MC_ARG(PCRTUINT128U,         puSrc1, 2);
+            IEM_MC_ARG(PCRTUINT128U,         puSrc2, 3);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_XREG_U128(puDst,        IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_AVX_AIMPL_3(pImpl->pfnU128, puDst, puSrc1, puSrc2);
+            IEM_MC_CLEAR_YREG_128_UP(          IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEM_MC_BEGIN(4, 4);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,  uDst,  1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1, 2);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2, 3);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_YREG_U256(uSrc1,      IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_CALL_AVX_AIMPL_3(pImpl->pfnU256, puDst, puSrc1, puSrc2);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(   IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(4, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG(PRTUINT128U,                 puDst,         1);
+            IEM_MC_ARG(PCRTUINT128U,                puSrc1,        2);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc2, uSrc2, 3);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc1,  IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_CALL_AVX_AIMPL_3(pImpl->pfnU128, puDst, puSrc1, puSrc2);
+            IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/**
+ * Common worker for AVX2 instructions on the forms:
+ *     - vpxxx    xmm0, xmm1, xmm2/mem128
+ *     - vpxxx    ymm0, ymm1, ymm2/mem256
+ *
+ * Takes function table for function w/o implicit state parameter.
+ *
+ * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit.
+ */
+FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, PCIEMOPMEDIAOPTF3, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_BEGIN(3, 3);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2);
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,  uDst,  0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1, 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2, 2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_FETCH_YREG_U256(uSrc1,   IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_FETCH_YREG_U256(uSrc2,   IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnU256, puDst, puSrc1, puSrc2);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(PRTUINT128U,          puDst,  0);
+            IEM_MC_ARG(PCRTUINT128U,         puSrc1, 1);
+            IEM_MC_ARG(PCRTUINT128U,         puSrc2, 2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_XREG_U128(puDst,        IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnU128, puDst, puSrc1, puSrc2);
+            IEM_MC_CLEAR_YREG_128_UP(          IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEM_MC_BEGIN(3, 4);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,  uDst,  0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1, 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2, 2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_YREG_U256(uSrc1,      IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnU256, puDst, puSrc1, puSrc2);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(   IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_ARG(PRTUINT128U,                 puDst,         0);
+            IEM_MC_ARG(PCRTUINT128U,                puSrc1,        1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc2, uSrc2, 2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc1,  IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_CALL_VOID_AIMPL_3(pImpl->pfnU128, puDst, puSrc1, puSrc2);
+            IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/**
+ * Common worker for AVX2 instructions on the forms:
+ *     - vpunpckhxx  xmm0, xmm1, xmm2/mem128
+ *     - vpunpckhxx  ymm0, ymm1, ymm2/mem256
+ *
+ * The 128-bit memory version of this instruction may elect to skip fetching the
+ * lower 64 bits of the operand.  We, however, do not.
+ *
+ * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit.
+ */
+FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_HighSrc, PCIEMOPMEDIAOPTF3, pImpl)
+{
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, pImpl);
+}
+
+
+/**
+ * Common worker for AVX2 instructions on the forms:
+ *     - vpunpcklxx  xmm0, xmm1, xmm2/mem128
+ *     - vpunpcklxx  ymm0, ymm1, ymm2/mem256
+ *
+ * The 128-bit memory version of this instruction may elect to skip fetching the
+ * higher 64 bits of the operand.  We, however, do not.
+ *
+ * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit.
+ */
+FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, PCIEMOPMEDIAOPTF3, pImpl)
+{
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, pImpl);
+}
+
+
+/**
+ * Common worker for AVX2 instructions on the forms:
+ *     - vpxxx    xmm0, xmm1/mem128
+ *     - vpxxx    ymm0, ymm1/mem256
+ *
+ * Takes function table for function w/o implicit state parameter.
+ *
+ * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit.
+ */
+FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Wx_Opt, PCIEMOPMEDIAOPTF2, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst, uDst, 0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc, uSrc, 1);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_FETCH_YREG_U256(uSrc,    IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnU256, puDst, puSrc);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(PRTUINT128U,          puDst,  0);
+            IEM_MC_ARG(PCRTUINT128U,         puSrc,  1);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_XREG_U128(puDst,        IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc,  IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnU128, puDst, puSrc);
+            IEM_MC_CLEAR_YREG_128_UP(          IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEM_MC_BEGIN(2, 3);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst, uDst, 0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc, uSrc, 1);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256_NO_AC(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnU256, puDst, puSrc);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(   IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_ARG(PRTUINT128U,                 puDst,       0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U128_NO_AC(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_2(pImpl->pfnU128, puDst, puSrc);
+            IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/*  Opcode VEX.0F 0x00 - invalid */
+/*  Opcode VEX.0F 0x01 - invalid */
+/*  Opcode VEX.0F 0x02 - invalid */
+/*  Opcode VEX.0F 0x03 - invalid */
+/*  Opcode VEX.0F 0x04 - invalid */
+/*  Opcode VEX.0F 0x05 - invalid */
+/*  Opcode VEX.0F 0x06 - invalid */
+/*  Opcode VEX.0F 0x07 - invalid */
+/*  Opcode VEX.0F 0x08 - invalid */
+/*  Opcode VEX.0F 0x09 - invalid */
+/*  Opcode VEX.0F 0x0a - invalid */
+
+/** Opcode VEX.0F 0x0b. */
+FNIEMOP_DEF(iemOp_vud2)
+{
+    IEMOP_MNEMONIC(vud2, "vud2");
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/*  Opcode VEX.0F 0x0c - invalid */
+/*  Opcode VEX.0F 0x0d - invalid */
+/*  Opcode VEX.0F 0x0e - invalid */
+/*  Opcode VEX.0F 0x0f - invalid */
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_vmovups_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVUPS, vmovups, Vps_WO, Wps, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * 128-bit: Register, Memory
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * 256-bit: Register, Memory
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x10
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_vmovupd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVUPD, vmovupd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * 128-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * 256-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+FNIEMOP_DEF(iemOp_vmovss_Vss_Hss_Wss)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /**
+         * @opcode      0x10
+         * @oppfx       0xf3
+         * @opcodesub   11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  5
+         * @optest      op1=1 op2=0  op3=2    -> op1=2
+         * @optest      op1=0 op2=0  op3=-22  -> op1=0xffffffea
+         * @optest      op1=3 op2=-1 op3=0x77 -> op1=-4294967177
+         * @optest      op1=3 op2=-2 op3=0x77 -> op1=-8589934473
+         * @note        HssHi refers to bits 127:32.
+         */
+        IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVSS, vmovss, Vss_WO, HssHi, Uss, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEMOP_HLP_DONE_VEX_DECODING();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm) /*U32*/,
+                                           IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x10
+         * @oppfx       0xf3
+         * @opcodesub   !11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovss_Vss_Hss_Wss
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-22 -> op1=-22
+         */
+        IEMOP_MNEMONIC2(VEX_RM_MEM, VMOVSS, vmovss, VssZx_WO, Md, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U32_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+FNIEMOP_DEF(iemOp_vmovsd_Vsd_Hsd_Wsd)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /**
+         * @opcode      0x10
+         * @oppfx       0xf2
+         * @opcodesub   11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  5
+         * @optest      op1=1 op2=0  op3=2    -> op1=2
+         * @optest      op1=0 op2=0  op3=-22  -> op1=0xffffffffffffffea
+         * @optest      op1=3 op2=-1 op3=0x77 ->
+         *              op1=0xffffffffffffffff0000000000000077
+         * @optest      op1=3 op2=0x42 op3=0x77 -> op1=0x420000000000000077
+         */
+        IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVSD, vmovsd, Vsd_WO, HsdHi, Usd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEMOP_HLP_DONE_VEX_DECODING();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm) /*U32*/,
+                                           IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x10
+         * @oppfx       0xf2
+         * @opcodesub   !11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovsd_Vsd_Hsd_Wsd
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-22 -> op1=-22
+         */
+        IEMOP_MNEMONIC2(VEX_RM_MEM, VMOVSD, vmovsd, VsdZx_WO, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_vmovups_Wps_Vps)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVUPS, vmovups, Wps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * 128-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * 256-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U256(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x11
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-22 -> op1=-22
+ */
+FNIEMOP_DEF(iemOp_vmovupd_Wpd_Vpd)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVUPD, vmovupd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * 128-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * 256-bit: Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U256(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+FNIEMOP_DEF(iemOp_vmovss_Wss_Hss_Vss)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /**
+         * @opcode      0x11
+         * @oppfx       0xf3
+         * @opcodesub   11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  5
+         * @optest      op1=1 op2=0  op3=2    -> op1=2
+         * @optest      op1=0 op2=0  op3=-22  -> op1=0xffffffea
+         * @optest      op1=3 op2=-1 op3=0x77 -> op1=-4294967177
+         * @optest      op1=3 op2=0x42 op3=0x77 -> op1=0x4200000077
+         */
+        IEMOP_MNEMONIC3(VEX_MVR_REG, VMOVSS, vmovss, Uss_WO, HssHi, Vss, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEMOP_HLP_DONE_VEX_DECODING();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U32_U96_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm) /*U32*/,
+                                           IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x11
+         * @oppfx       0xf3
+         * @opcodesub   !11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovss_Vss_Hss_Wss
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-22 -> op1=-22
+         */
+        IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVSS, vmovss, Md_WO, Vss, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint32_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U32(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+FNIEMOP_DEF(iemOp_vmovsd_Wsd_Hsd_Vsd)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /**
+         * @opcode      0x11
+         * @oppfx       0xf2
+         * @opcodesub   11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  5
+         * @optest      op1=1 op2=0  op3=2    -> op1=2
+         * @optest      op1=0 op2=0  op3=-22  -> op1=0xffffffffffffffea
+         * @optest      op1=3 op2=-1 op3=0x77 ->
+         *              op1=0xffffffffffffffff0000000000000077
+         * @optest      op2=0x42 op3=0x77 -> op1=0x420000000000000077
+         */
+        IEMOP_MNEMONIC3(VEX_MVR_REG, VMOVSD, vmovsd, Usd_WO, HsdHi, Vsd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEMOP_HLP_DONE_VEX_DECODING();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U64_U64_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hss*/);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x11
+         * @oppfx       0xf2
+         * @opcodesub   !11 mr/reg
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovsd_Wsd_Hsd_Vsd
+         * @optest      op1=1 op2=2 -> op1=2
+         * @optest      op1=0 op2=-22 -> op1=-22
+         */
+        IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVSD, vmovsd, Mq_WO, Vsd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_IGNORED);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+FNIEMOP_DEF(iemOp_vmovlps_Vq_Hq_Mq__vmovhlps)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /**
+         * @opcode      0x12
+         * @opcodesub   11 mr/reg
+         * @oppfx       none
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  7LZ
+         * @optest         op2=0x2200220122022203
+         *                 op3=0x3304330533063307
+         *              -> op1=0x22002201220222033304330533063307
+         * @optest      op2=-1  op3=-42 -> op1=-42
+         * @note        op3 and op2 are only the 8-byte high XMM register halfs.
+         */
+        IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVHLPS, vmovhlps, Vq_WO, HqHi, UqHi, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U64HI_U64HI_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                               IEM_GET_MODRM_RM(pVCpu, bRm),
+                                               IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x12
+         * @opcodesub   !11 mr/reg
+         * @oppfx       none
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5LZ
+         * @opfunction  iemOp_vmovlps_Vq_Hq_Mq__vmovhlps
+         * @optest      op1=1 op2=0 op3=0 -> op1=0
+         * @optest      op1=0 op2=-1 op3=-1 -> op1=-1
+         * @optest      op1=1 op2=2 op3=3 -> op1=0x20000000000000003
+         * @optest      op2=-1 op3=0x42 -> op1=0xffffffffffffffff0000000000000042
+         */
+        IEMOP_MNEMONIC3(VEX_RVM_MEM, VMOVLPS, vmovlps, Vq_WO, HqHi, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                                  uSrc,
+                                                  IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x12
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamerge
+ * @opxcpttype  5LZ
+ * @optest      op2=0 op3=2 -> op1=2
+ * @optest      op2=0x22 op3=0x33 -> op1=0x220000000000000033
+ * @optest      op2=0xfffffff0fffffff1 op3=0xeeeeeee8eeeeeee9
+ *              -> op1=0xfffffff0fffffff1eeeeeee8eeeeeee9
+ */
+FNIEMOP_DEF(iemOp_vmovlpd_Vq_Hq_Mq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC3(VEX_RVM_MEM, VMOVLPD, vmovlpd, Vq_WO, HqHi, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_MERGE_YREG_U64LOCAL_U64HI_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                                  uSrc,
+                                                  IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex660f12m3
+     * @opcode      0x12
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x12
+ * @oppfx       0xf3
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  4
+ * @optest      vex.l==0 / op1=-1 op2=0xdddddddd00000002eeeeeeee00000001
+ *              -> op1=0x00000002000000020000000100000001
+ * @optest      vex.l==1 /
+ *                 op2=0xbbbbbbbb00000004cccccccc00000003dddddddd00000002eeeeeeee00000001
+ *              -> op1=0x0000000400000004000000030000000300000002000000020000000100000001
+ */
+FNIEMOP_DEF(iemOp_vmovsldup_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVSLDUP, vmovsldup, Vx_WO, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 0);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 0);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 2);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 2);
+            IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_CONST(uint8_t,   iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1);
+            IEM_MC_ARG_CONST(uint8_t,   iYRegSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 2);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovsldup_256_rr, iYRegDst, iYRegSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 0);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 0);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 2);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 2);
+            IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_CONST(uint8_t,   iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc, uSrc, 2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovsldup_256_rm, iYRegDst, puSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x12
+ * @oppfx       0xf2
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      vex.l==0 / op2=0xddddddddeeeeeeee2222222211111111
+ *              ->         op1=0x22222222111111112222222211111111
+ * @optest      vex.l==1 / op2=0xbbbbbbbbcccccccc4444444433333333ddddddddeeeeeeee2222222211111111
+ *              ->         op1=0x4444444433333333444444443333333322222222111111112222222211111111
+ */
+FNIEMOP_DEF(iemOp_vmovddup_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVDDUP, vmovddup, Vx_WO, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(1, 0);
+            IEM_MC_ARG(uint64_t,                    uSrc, 0);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_XREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/);
+            IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc);
+            IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 1 /* a_iQword*/, uSrc);
+            IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_CONST(uint8_t,   iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1);
+            IEM_MC_ARG_CONST(uint8_t,   iYRegSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 2);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovddup_256_rr, iYRegDst, iYRegSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(1, 1);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_ARG(uint64_t,                    uSrc, 0);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 0 /* a_iQword*/, uSrc);
+            IEM_MC_STORE_XREG_U64(IEM_GET_MODRM_REG(pVCpu, bRm), 1 /* a_iQword*/, uSrc);
+            IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_CONST(uint8_t,   iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc, uSrc, 2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovddup_256_rm, iYRegDst, puSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x13
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovlps_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVLPS, vmovlps, Mq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex0f13m3
+     * @opcode      0x13
+     * @opcodesub   11 mr/reg
+     * @oppfx       none
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x13
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  5
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovlpd_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVLPD, vmovlpd, Mq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex660f13m3
+     * @opcode      0x13
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/*  Opcode VEX.F3.0F 0x13 - invalid */
+/*  Opcode VEX.F2.0F 0x13 - invalid */
+
+/** Opcode VEX.0F 0x14 - vunpcklps Vx, Hx, Wx*/
+FNIEMOP_DEF(iemOp_vunpcklps_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VUNPCKLPS, vunpcklps, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(           vunpcklps);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F 0x14 - vunpcklpd Vx,Hx,Wx   */
+FNIEMOP_DEF(iemOp_vunpcklpd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VUNPCKLPD, vunpcklpd, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(           vunpcklpd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x14 - invalid */
+/*  Opcode VEX.F2.0F 0x14 - invalid */
+
+
+/** Opcode VEX.0F 0x15 - vunpckhps Vx, Hx, Wx   */
+FNIEMOP_DEF(iemOp_vunpckhps_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VUNPCKHPS, vunpckhps, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(           vunpckhps);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F 0x15 - vunpckhpd Vx,Hx,Wx   */
+FNIEMOP_DEF(iemOp_vunpckhpd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VUNPCKHPD, vunpckhpd, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(           vunpckhpd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x15 - invalid */
+/*  Opcode VEX.F2.0F 0x15 - invalid */
+
+
+FNIEMOP_DEF(iemOp_vmovhps_Vdq_Hq_Mq__vmovlhps_Vdq_Hq_Uq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /**
+         * @opcode      0x16
+         * @opcodesub   11 mr/reg
+         * @oppfx       none
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamerge
+         * @opxcpttype  7LZ
+         */
+        IEMOP_MNEMONIC3(VEX_RVM_REG, VMOVLHPS, vmovlhps, Vq_WO, Hq, Uq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        IEM_MC_MERGE_YREG_U64LO_U64LO_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                               IEM_GET_MODRM_RM(pVCpu, bRm),
+                                               IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x16
+         * @opcodesub   !11 mr/reg
+         * @oppfx       none
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdfp_datamove
+         * @opxcpttype  5LZ
+         * @opfunction  iemOp_vmovhps_Vdq_Hq_Mq__vmovlhps_Vdq_Hq_Uq
+         */
+        IEMOP_MNEMONIC3(VEX_RVM_MEM, VMOVHPS, vmovhps, Vq_WO, Hq, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                                  IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/,
+                                                  uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x16
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamerge
+ * @opxcpttype  5LZ
+ */
+FNIEMOP_DEF(iemOp_vmovhpd_Vdq_Hq_Mq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC3(VEX_RVM_MEM, VMOVHPD, vmovhpd, Vq_WO, Hq, Mq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_MERGE_YREG_U64LO_U64LOCAL_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                                  IEM_GET_EFFECTIVE_VVVV(pVCpu) /*Hq*/,
+                                                  uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex660f16m3
+     * @opcode      0x12
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode VEX.F3.0F 0x16 - vmovshdup Vx, Wx   */
+/**
+ * @opcode      0x16
+ * @oppfx       0xf3
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  4
+ */
+FNIEMOP_DEF(iemOp_vmovshdup_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVSHDUP, vmovshdup, Vx_WO, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(RTUINT128U,                  uSrc);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 1);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 1);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 3);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 3);
+            IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_CONST(uint8_t,   iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1);
+            IEM_MC_ARG_CONST(uint8_t,   iYRegSrc, IEM_GET_MODRM_RM(pVCpu, bRm), 2);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovshdup_256_rr, iYRegDst, iYRegSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 0, uSrc, 1);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 1, uSrc, 1);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 2, uSrc, 3);
+            IEM_MC_STORE_XREG_U32_U128(IEM_GET_MODRM_REG(pVCpu, bRm), 3, uSrc, 3);
+            IEM_MC_CLEAR_YREG_128_UP(IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_IMPLICIT_AVX_AIMPL_ARGS();
+            IEM_MC_ARG_CONST(uint8_t,   iYRegDst, IEM_GET_MODRM_REG(pVCpu, bRm), 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc, uSrc, 2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_AVX_AIMPL_2(iemAImpl_vmovshdup_256_rm, iYRegDst, puSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/*  Opcode VEX.F2.0F 0x16 - invalid */
+
+
+/**
+ * @opcode      0x17
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdfp_datamove
+ * @opxcpttype  5
+ */
+FNIEMOP_DEF(iemOp_vmovhps_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVHPS, vmovhps, Mq_WO, VqHi, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_2ND_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex0f17m3
+     * @opcode      0x17
+     * @opcodesub   11 mr/reg
+     * @oppfx       none
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/**
+ * @opcode      0x17
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  5
+ */
+FNIEMOP_DEF(iemOp_vmovhpd_Mq_Vq)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVHPD, vmovhpd, Mq_WO, VqHi, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_2ND_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex660f17m3
+     * @opcode      0x17
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/*  Opcode VEX.F3.0F 0x17 - invalid */
+/*  Opcode VEX.F2.0F 0x17 - invalid */
+
+
+/*  Opcode VEX.0F 0x18 - invalid */
+/*  Opcode VEX.0F 0x19 - invalid */
+/*  Opcode VEX.0F 0x1a - invalid */
+/*  Opcode VEX.0F 0x1b - invalid */
+/*  Opcode VEX.0F 0x1c - invalid */
+/*  Opcode VEX.0F 0x1d - invalid */
+/*  Opcode VEX.0F 0x1e - invalid */
+/*  Opcode VEX.0F 0x1f - invalid */
+
+/*  Opcode VEX.0F 0x20 - invalid */
+/*  Opcode VEX.0F 0x21 - invalid */
+/*  Opcode VEX.0F 0x22 - invalid */
+/*  Opcode VEX.0F 0x23 - invalid */
+/*  Opcode VEX.0F 0x24 - invalid */
+/*  Opcode VEX.0F 0x25 - invalid */
+/*  Opcode VEX.0F 0x26 - invalid */
+/*  Opcode VEX.0F 0x27 - invalid */
+
+/**
+ * @opcode      0x28
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Almost identical to vmovapd.
+ */
+FNIEMOP_DEF(iemOp_vmovaps_Vps_Wps)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVAPS, vmovaps, Vps_WO, Wps, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(1, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U256_ALIGN_AVX(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x28
+ * @oppfx       66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Almost identical to vmovaps
+ */
+FNIEMOP_DEF(iemOp_vmovapd_Vpd_Wpd)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVAPD, vmovapd, Vpd_WO, Wpd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(1, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U256_ALIGN_AVX(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+/**
+ * @opmnemonic  udvexf30f28
+ * @opcode      0x28
+ * @oppfx       0xf3
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udvexf20f28
+ * @opcode      0x28
+ * @oppfx       0xf2
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opcode      0x29
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Almost identical to vmovapd.
+ */
+FNIEMOP_DEF(iemOp_vmovaps_Wps_Vps)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVAPS, vmovaps, Wps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(1, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                IEM_GET_MODRM_REG(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+/**
+ * @opcode      0x29
+ * @oppfx       66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Almost identical to vmovaps
+ */
+FNIEMOP_DEF(iemOp_vmovapd_Wpd_Vpd)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVAPD, vmovapd, Wpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(1, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                IEM_GET_MODRM_REG(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/**
+ * @opmnemonic  udvexf30f29
+ * @opcode      0x29
+ * @oppfx       0xf3
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udvexf20f29
+ * @opcode      0x29
+ * @oppfx       0xf2
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+
+/** Opcode VEX.0F 0x2a - invalid */
+/** Opcode VEX.66.0F 0x2a - invalid */
+/** Opcode VEX.F3.0F 0x2a - vcvtsi2ss Vss, Hss, Ey */
+FNIEMOP_STUB(iemOp_vcvtsi2ss_Vss_Hss_Ey);
+/** Opcode VEX.F2.0F 0x2a - vcvtsi2sd Vsd, Hsd, Ey */
+FNIEMOP_STUB(iemOp_vcvtsi2sd_Vsd_Hsd_Ey);
+
+
+/**
+ * @opcode      0x2b
+ * @opcodesub   !11 mr/reg
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_cachect
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Identical implementation to vmovntpd
+ */
+FNIEMOP_DEF(iemOp_vmovntps_Mps_Vps)
+{
+    IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVNTPS, vmovntps, Mps_WO, Vps, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        /*
+         * memory, register.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    /* The register, register encoding is invalid. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/**
+ * @opcode      0x2b
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_cachect
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ * @note        Identical implementation to vmovntps
+ */
+FNIEMOP_DEF(iemOp_vmovntpd_Mpd_Vpd)
+{
+    IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVNTPD, vmovntpd, Mpd_WO, Vpd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        /*
+         * memory, register.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_XREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    /* The register, register encoding is invalid. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/**
+ * @opmnemonic  udvexf30f2b
+ * @opcode      0x2b
+ * @oppfx       0xf3
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+/**
+ * @opmnemonic  udvexf20f2b
+ * @opcode      0x2b
+ * @oppfx       0xf2
+ * @opunused    vex.modrm
+ * @opcpuid     avx
+ * @optest      ->
+ * @opdone
+ */
+
+
+/*  Opcode VEX.0F 0x2c - invalid */
+/*  Opcode VEX.66.0F 0x2c - invalid */
+/** Opcode VEX.F3.0F 0x2c - vcvttss2si Gy, Wss */
+FNIEMOP_STUB(iemOp_vcvttss2si_Gy_Wss);
+/** Opcode VEX.F2.0F 0x2c - vcvttsd2si Gy, Wsd */
+FNIEMOP_STUB(iemOp_vcvttsd2si_Gy_Wsd);
+
+/*  Opcode VEX.0F 0x2d - invalid */
+/*  Opcode VEX.66.0F 0x2d - invalid */
+/** Opcode VEX.F3.0F 0x2d - vcvtss2si Gy, Wss */
+FNIEMOP_STUB(iemOp_vcvtss2si_Gy_Wss);
+/** Opcode VEX.F2.0F 0x2d - vcvtsd2si Gy, Wsd */
+FNIEMOP_STUB(iemOp_vcvtsd2si_Gy_Wsd);
+
+
+/** Opcode VEX.0F 0x2e - vucomiss Vss, Wss */
+FNIEMOP_DEF(iemOp_vucomiss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, VUCOMISS, vucomiss, Vss, Wss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_BEGIN(4, 1);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc2,             3);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(puSrc2,      IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vucomiss_u128, iemAImpl_vucomiss_u128_fallback),
+                                 pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_LOCAL(X86XMMREG,                 uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,       puSrc2, uSrc2,      3);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,       IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vucomiss_u128, iemAImpl_vucomiss_u128_fallback),
+                                 pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode VEX.66.0F 0x2e - vucomisd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_vucomisd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, VUCOMISD, vucomisd, Vsd, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_BEGIN(4, 1);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc2,             3);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(puSrc2,      IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vucomisd_u128, iemAImpl_vucomisd_u128_fallback),
+                                 pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_LOCAL(X86XMMREG,                 uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,       puSrc2, uSrc2,      3);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,       IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vucomisd_u128, iemAImpl_vucomisd_u128_fallback),
+                                 pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode VEX.F3.0F 0x2e - invalid */
+/*  Opcode VEX.F2.0F 0x2e - invalid */
+
+/** Opcode VEX.0F 0x2f - vcomiss Vss, Wss */
+FNIEMOP_DEF(iemOp_vcomiss_Vss_Wss)
+{
+    IEMOP_MNEMONIC2(RM, VCOMISS, vcomiss, Vss, Wss, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_BEGIN(4, 1);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc2,             3);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(puSrc2,      IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vcomiss_u128, iemAImpl_vcomiss_u128_fallback),
+                                 pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_LOCAL(X86XMMREG,                 uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,       puSrc2, uSrc2,      3);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,       IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vcomiss_u128, iemAImpl_vcomiss_u128_fallback),
+                                 pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/** Opcode VEX.66.0F 0x2f - vcomisd Vsd, Wsd */
+FNIEMOP_DEF(iemOp_vcomisd_Vsd_Wsd)
+{
+    IEMOP_MNEMONIC2(RM, VCOMISD, vcomisd, Vsd, Wsd, DISOPTYPE_HARMLESS | DISOPTYPE_SSE, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_BEGIN(4, 1);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc2,             3);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_XMM_CONST(puSrc2,      IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vcomisd_u128, iemAImpl_vcomisd_u128_fallback),
+                                 pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 3);
+        IEM_MC_LOCAL(uint32_t, fEFlags);
+        IEM_MC_ARG(uint32_t *,                  pfMxcsr,            0);
+        IEM_MC_ARG_LOCAL_REF(uint32_t *,        pEFlags, fEFlags,   1);
+        IEM_MC_ARG(PCX86XMMREG,                 puSrc1,             2);
+        IEM_MC_LOCAL(X86XMMREG,                 uSrc2);
+        IEM_MC_ARG_LOCAL_REF(PCX86XMMREG,       puSrc2, uSrc2,      3);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_FETCH_MEM_XMM_U32(uSrc2, 0 /*a_DWord*/, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_FETCH_EFLAGS(fEFlags);
+        IEM_MC_REF_MXCSR(pfMxcsr);
+        IEM_MC_REF_XREG_XMM_CONST(puSrc1,       IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vcomisd_u128, iemAImpl_vcomisd_u128_fallback),
+                                 pfMxcsr, pEFlags, puSrc1, puSrc2);
+        IEM_MC_IF_MXCSR_XCPT_PENDING()
+            IEM_MC_RAISE_SSE_AVX_SIMD_FP_OR_UD_XCPT();
+        IEM_MC_ELSE()
+            IEM_MC_COMMIT_EFLAGS(fEFlags);
+        IEM_MC_ENDIF();
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode VEX.F3.0F 0x2f - invalid */
+/*  Opcode VEX.F2.0F 0x2f - invalid */
+
+/*  Opcode VEX.0F 0x30 - invalid */
+/*  Opcode VEX.0F 0x31 - invalid */
+/*  Opcode VEX.0F 0x32 - invalid */
+/*  Opcode VEX.0F 0x33 - invalid */
+/*  Opcode VEX.0F 0x34 - invalid */
+/*  Opcode VEX.0F 0x35 - invalid */
+/*  Opcode VEX.0F 0x36 - invalid */
+/*  Opcode VEX.0F 0x37 - invalid */
+/*  Opcode VEX.0F 0x38 - invalid */
+/*  Opcode VEX.0F 0x39 - invalid */
+/*  Opcode VEX.0F 0x3a - invalid */
+/*  Opcode VEX.0F 0x3b - invalid */
+/*  Opcode VEX.0F 0x3c - invalid */
+/*  Opcode VEX.0F 0x3d - invalid */
+/*  Opcode VEX.0F 0x3e - invalid */
+/*  Opcode VEX.0F 0x3f - invalid */
+/*  Opcode VEX.0F 0x40 - invalid */
+/*  Opcode VEX.0F 0x41 - invalid */
+/*  Opcode VEX.0F 0x42 - invalid */
+/*  Opcode VEX.0F 0x43 - invalid */
+/*  Opcode VEX.0F 0x44 - invalid */
+/*  Opcode VEX.0F 0x45 - invalid */
+/*  Opcode VEX.0F 0x46 - invalid */
+/*  Opcode VEX.0F 0x47 - invalid */
+/*  Opcode VEX.0F 0x48 - invalid */
+/*  Opcode VEX.0F 0x49 - invalid */
+/*  Opcode VEX.0F 0x4a - invalid */
+/*  Opcode VEX.0F 0x4b - invalid */
+/*  Opcode VEX.0F 0x4c - invalid */
+/*  Opcode VEX.0F 0x4d - invalid */
+/*  Opcode VEX.0F 0x4e - invalid */
+/*  Opcode VEX.0F 0x4f - invalid */
+
+
+/** Opcode VEX.0F 0x50 - vmovmskps Gy, Ups */
+FNIEMOP_DEF(iemOp_vmovmskps_Gy_Ups)
+{
+    IEMOP_MNEMONIC2(VEX_RM_REG, VMOVMSKPS, vmovmskps, Gd, Ux, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEMOP_HLP_DONE_VEX_DECODING();
+            IEM_MC_BEGIN(2, 1);
+            IEM_MC_LOCAL(uint8_t,           u8Dst);
+            IEM_MC_ARG_LOCAL_REF(uint8_t *, pu8Dst,  u8Dst, 0);
+            IEM_MC_ARG(PCRTUINT128U,            puSrc,                 1);
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_XREG_U128_CONST(puSrc,  IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vmovmskps_u128, iemAImpl_vmovmskps_u128_fallback),
+                                     pu8Dst, puSrc);
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u8Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEMOP_HLP_DONE_VEX_DECODING();
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_LOCAL(uint8_t,               u8Dst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_ARG_LOCAL_REF(uint8_t *,     pu8Dst,  u8Dst, 0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc,   uSrc,  1);
+
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_FETCH_YREG_U256(uSrc,    IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vmovmskps_u256, iemAImpl_vmovmskps_u256_fallback),
+                                     pu8Dst, puSrc);
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u8Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    /* No memory operand. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode VEX.66.0F 0x50 - vmovmskpd Gy,Upd */
+FNIEMOP_DEF(iemOp_vmovmskpd_Gy_Upd)
+{
+    IEMOP_MNEMONIC2(VEX_RM_REG, VMOVMSKPD, vmovmskpd, Gd, Ux, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            IEMOP_HLP_DONE_VEX_DECODING();
+            IEM_MC_BEGIN(2, 1);
+            IEM_MC_LOCAL(uint8_t,           u8Dst);
+            IEM_MC_ARG_LOCAL_REF(uint8_t *, pu8Dst,  u8Dst, 0);
+            IEM_MC_ARG(PCRTUINT128U,            puSrc,                 1);
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_XREG_U128_CONST(puSrc,  IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vmovmskpd_u128, iemAImpl_vmovmskpd_u128_fallback),
+                                     pu8Dst, puSrc);
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u8Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEMOP_HLP_DONE_VEX_DECODING();
+            IEM_MC_BEGIN(2, 2);
+            IEM_MC_LOCAL(uint8_t,               u8Dst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_ARG_LOCAL_REF(uint8_t *,     pu8Dst,  u8Dst, 0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc,   uSrc,  1);
+
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_FETCH_YREG_U256(uSrc,    IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vmovmskpd_u256, iemAImpl_vmovmskpd_u256_fallback),
+                                     pu8Dst, puSrc);
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u8Dst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    /* No memory operand. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/*  Opcode VEX.F3.0F 0x50 - invalid */
+/*  Opcode VEX.F2.0F 0x50 - invalid */
+
+/** Opcode VEX.0F 0x51 - vsqrtps Vps, Wps */
+FNIEMOP_STUB(iemOp_vsqrtps_Vps_Wps);
+/** Opcode VEX.66.0F 0x51 - vsqrtpd Vpd, Wpd */
+FNIEMOP_STUB(iemOp_vsqrtpd_Vpd_Wpd);
+/** Opcode VEX.F3.0F 0x51 - vsqrtss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vsqrtss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x51 - vsqrtsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vsqrtsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x52 - vrsqrtps Vps, Wps */
+FNIEMOP_STUB(iemOp_vrsqrtps_Vps_Wps);
+/*  Opcode VEX.66.0F 0x52 - invalid */
+/** Opcode VEX.F3.0F 0x52 - vrsqrtss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vrsqrtss_Vss_Hss_Wss);
+/*  Opcode VEX.F2.0F 0x52 - invalid */
+
+/** Opcode VEX.0F 0x53 - vrcpps Vps, Wps */
+FNIEMOP_STUB(iemOp_vrcpps_Vps_Wps);
+/*  Opcode VEX.66.0F 0x53 - invalid */
+/** Opcode VEX.F3.0F 0x53 - vrcpss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vrcpss_Vss_Hss_Wss);
+/*  Opcode VEX.F2.0F 0x53 - invalid */
+
+
+/** Opcode VEX.0F 0x54 - vandps Vps, Hps, Wps */
+FNIEMOP_DEF(iemOp_vandps_Vps_Hps_Wps)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VANDPS, vandps, Vps, Hps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpand, &g_iemAImpl_vpand_fallback));
+}
+
+
+/** Opcode VEX.66.0F 0x54 - vandpd Vpd, Hpd, Wpd */
+FNIEMOP_DEF(iemOp_vandpd_Vpd_Hpd_Wpd)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VANDPD, vandpd, Vpd, Hpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpand, &g_iemAImpl_vpand_fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x54 - invalid */
+/*  Opcode VEX.F2.0F 0x54 - invalid */
+
+
+/** Opcode VEX.0F 0x55 - vandnps Vps, Hps, Wps */
+FNIEMOP_DEF(iemOp_vandnps_Vps_Hps_Wps)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VANDNPS, vandnps, Vps, Hps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpandn, &g_iemAImpl_vpandn_fallback));
+}
+
+
+/** Opcode VEX.66.0F 0x55 - vandnpd Vpd, Hpd, Wpd */
+FNIEMOP_DEF(iemOp_vandnpd_Vpd_Hpd_Wpd)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VANDNPD, vandnpd, Vpd, Hpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpandn, &g_iemAImpl_vpandn_fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x55 - invalid */
+/*  Opcode VEX.F2.0F 0x55 - invalid */
+
+/** Opcode VEX.0F 0x56 - vorps Vps, Hps, Wps */
+FNIEMOP_DEF(iemOp_vorps_Vps_Hps_Wps)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VORPS, vorps, Vps, Hps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpor, &g_iemAImpl_vpor_fallback));
+}
+
+
+/** Opcode VEX.66.0F 0x56 - vorpd Vpd, Hpd, Wpd */
+FNIEMOP_DEF(iemOp_vorpd_Vpd_Hpd_Wpd)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VORPD, vorpd, Vpd, Hpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpor, &g_iemAImpl_vpor_fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x56 - invalid */
+/*  Opcode VEX.F2.0F 0x56 - invalid */
+
+
+/** Opcode VEX.0F 0x57 - vxorps Vps, Hps, Wps */
+FNIEMOP_DEF(iemOp_vxorps_Vps_Hps_Wps)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VXORPS, vxorps, Vps, Hps, Wps, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpxor, &g_iemAImpl_vpxor_fallback));
+}
+
+
+/** Opcode VEX.66.0F 0x57 - vxorpd Vpd, Hpd, Wpd */
+FNIEMOP_DEF(iemOp_vxorpd_Vpd_Hpd_Wpd)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VXORPD, vxorpd, Vpd, Hpd, Wpd, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpxor, &g_iemAImpl_vpxor_fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x57 - invalid */
+/*  Opcode VEX.F2.0F 0x57 - invalid */
+
+/** Opcode VEX.0F 0x58 - vaddps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vaddps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x58 - vaddpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vaddpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x58 - vaddss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vaddss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x58 - vaddsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vaddsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x59 - vmulps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vmulps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x59 - vmulpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vmulpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x59 - vmulss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vmulss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x59 - vmulsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vmulsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x5a - vcvtps2pd Vpd, Wps */
+FNIEMOP_STUB(iemOp_vcvtps2pd_Vpd_Wps);
+/** Opcode VEX.66.0F 0x5a - vcvtpd2ps Vps, Wpd */
+FNIEMOP_STUB(iemOp_vcvtpd2ps_Vps_Wpd);
+/** Opcode VEX.F3.0F 0x5a - vcvtss2sd Vsd, Hx, Wss */
+FNIEMOP_STUB(iemOp_vcvtss2sd_Vsd_Hx_Wss);
+/** Opcode VEX.F2.0F 0x5a - vcvtsd2ss Vss, Hx, Wsd */
+FNIEMOP_STUB(iemOp_vcvtsd2ss_Vss_Hx_Wsd);
+
+/** Opcode VEX.0F 0x5b - vcvtdq2ps Vps, Wdq */
+FNIEMOP_STUB(iemOp_vcvtdq2ps_Vps_Wdq);
+/** Opcode VEX.66.0F 0x5b - vcvtps2dq Vdq, Wps */
+FNIEMOP_STUB(iemOp_vcvtps2dq_Vdq_Wps);
+/** Opcode VEX.F3.0F 0x5b - vcvttps2dq Vdq, Wps */
+FNIEMOP_STUB(iemOp_vcvttps2dq_Vdq_Wps);
+/*  Opcode VEX.F2.0F 0x5b - invalid */
+
+/** Opcode VEX.0F 0x5c - vsubps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vsubps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x5c - vsubpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vsubpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x5c - vsubss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vsubss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x5c - vsubsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vsubsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x5d - vminps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vminps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x5d - vminpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vminpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x5d - vminss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vminss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x5d - vminsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vminsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x5e - vdivps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vdivps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x5e - vdivpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vdivpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x5e - vdivss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vdivss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x5e - vdivsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vdivsd_Vsd_Hsd_Wsd);
+
+/** Opcode VEX.0F 0x5f - vmaxps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vmaxps_Vps_Hps_Wps);
+/** Opcode VEX.66.0F 0x5f - vmaxpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vmaxpd_Vpd_Hpd_Wpd);
+/** Opcode VEX.F3.0F 0x5f - vmaxss Vss, Hss, Wss */
+FNIEMOP_STUB(iemOp_vmaxss_Vss_Hss_Wss);
+/** Opcode VEX.F2.0F 0x5f - vmaxsd Vsd, Hsd, Wsd */
+FNIEMOP_STUB(iemOp_vmaxsd_Vsd_Hsd_Wsd);
+
+
+/*  Opcode VEX.0F 0x60 - invalid */
+
+
+/** Opcode VEX.66.0F 0x60 - vpunpcklbw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpunpcklbw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKLBW, vpunpcklbw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(           vpunpcklbw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x60 - invalid */
+
+
+/*  Opcode VEX.0F 0x61 - invalid */
+
+
+/** Opcode VEX.66.0F 0x61 - vpunpcklwd Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpunpcklwd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKLWD, vpunpcklwd, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(           vpunpcklwd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x61 - invalid */
+
+
+/*  Opcode VEX.0F 0x62 - invalid */
+
+/** Opcode VEX.66.0F 0x62 - vpunpckldq Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpunpckldq_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKLDQ, vpunpckldq, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(           vpunpckldq);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x62 - invalid */
+
+
+
+/*  Opcode VEX.0F 0x63 - invalid */
+
+
+/** Opcode VEX.66.0F 0x63 - vpacksswb Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpacksswb_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPACKSSWB, vpacksswb, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(          vpacksswb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x63 - invalid */
+
+/*  Opcode VEX.0F 0x64 - invalid */
+
+
+/** Opcode VEX.66.0F 0x64 - vpcmpgtb Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpcmpgtb_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPCMPGTB, vpcmpgtb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpcmpgtb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x64 - invalid */
+
+/*  Opcode VEX.0F 0x65 - invalid */
+
+
+/** Opcode VEX.66.0F 0x65 - vpcmpgtw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpcmpgtw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPCMPGTW, vpcmpgtw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpcmpgtw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x65 - invalid */
+
+/*  Opcode VEX.0F 0x66 - invalid */
+
+
+/** Opcode VEX.66.0F 0x66 - vpcmpgtd Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpcmpgtd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPCMPGTD, vpcmpgtd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpcmpgtd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x66 - invalid */
+
+/*  Opcode VEX.0F 0x67 - invalid */
+
+
+/** Opcode VEX.66.0F 0x67 - vpackuswb Vx, Hx, W */
+FNIEMOP_DEF(iemOp_vpackuswb_Vx_Hx_W)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPACKUSWB, vpackuswb, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(          vpackuswb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x67 - invalid */
+
+
+///**
+// * Common worker for SSE2 instructions on the form:
+// *      pxxxx xmm1, xmm2/mem128
+// *
+// * The 2nd operand is the second half of a register, which in the memory case
+// * means a 64-bit memory access for MMX, and for SSE a 128-bit aligned access
+// * where it may read the full 128 bits or only the upper 64 bits.
+// *
+// * Exceptions type 4.
+// */
+//FNIEMOP_DEF_1(iemOpCommonSse_HighHigh_To_Full, PCIEMOPMEDIAF1H1, pImpl)
+//{
+//    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+//    if (IEM_IS_MODRM_REG_MODE(bRm))
+//    {
+//        /*
+//         * Register, register.
+//         */
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_BEGIN(2, 0);
+//        IEM_MC_ARG(PRTUINT128U,          pDst, 0);
+//        IEM_MC_ARG(PCRTUINT128U,         pSrc, 1);
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+//        IEM_MC_REF_XREG_U128_CONST(pSrc, IEM_GET_MODRM_RM(pVCpu, bRm));
+//        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+//        IEM_MC_ADVANCE_RIP_AND_FINISH();
+//        IEM_MC_END();
+//    }
+//    else
+//    {
+//        /*
+//         * Register, memory.
+//         */
+//        IEM_MC_BEGIN(2, 2);
+//        IEM_MC_ARG(PRTUINT128U,                 pDst,       0);
+//        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+//        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      pSrc, uSrc, 1);
+//        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+//
+//        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+//        IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//        IEM_MC_MAYBE_RAISE_SSE2_RELATED_XCPT();
+//        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); /* Most CPUs probably only right high qword */
+//
+//        IEM_MC_PREPARE_SSE_USAGE();
+//        IEM_MC_REF_XREG_U128(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+//        IEM_MC_CALL_SSE_AIMPL_2(pImpl->pfnU128, pDst, pSrc);
+//
+//        IEM_MC_ADVANCE_RIP_AND_FINISH();
+//        IEM_MC_END();
+//    }
+//    return VINF_SUCCESS;
+//}
+
+
+/*  Opcode VEX.0F 0x68 - invalid */
+
+/** Opcode VEX.66.0F 0x68 - vpunpckhbw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpunpckhbw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKHBW, vpunpckhbw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(           vpunpckhbw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_HighSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x68 - invalid */
+
+
+/*  Opcode VEX.0F 0x69 - invalid */
+
+
+/** Opcode VEX.66.0F 0x69 - vpunpckhwd Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpunpckhwd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKHWD, vpunpckhwd, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(           vpunpckhwd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_HighSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x69 - invalid */
+
+
+/*  Opcode VEX.0F 0x6a - invalid */
+
+
+/** Opcode VEX.66.0F 0x6a - vpunpckhdq Vx, Hx, W */
+FNIEMOP_DEF(iemOp_vpunpckhdq_Vx_Hx_W)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKHDQ, vpunpckhdq, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(           vpunpckhdq);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_HighSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x6a - invalid */
+
+
+/*  Opcode VEX.0F 0x6b - invalid */
+
+
+/** Opcode VEX.66.0F 0x6b - vpackssdw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpackssdw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPACKSSDW, vpackssdw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(          vpackssdw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x6b - invalid */
+
+
+/*  Opcode VEX.0F 0x6c - invalid */
+
+
+/** Opcode VEX.66.0F 0x6c - vpunpcklqdq Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpunpcklqdq_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKLQDQ, vpunpcklqdq, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(            vpunpcklqdq);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_LowSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x6c - invalid */
+/*  Opcode VEX.F2.0F 0x6c - invalid */
+
+
+/*  Opcode VEX.0F 0x6d - invalid */
+
+
+/** Opcode VEX.66.0F 0x6d - vpunpckhqdq Vx, Hx, W */
+FNIEMOP_DEF(iemOp_vpunpckhqdq_Vx_Hx_W)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPUNPCKHQDQ, vpunpckhqdq, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(            vpunpckhqdq);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_HighSrc, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x6d - invalid */
+
+
+/*  Opcode VEX.0F 0x6e - invalid */
+
+FNIEMOP_DEF(iemOp_vmovd_q_Vy_Ey)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x6e
+         * @opcodesub   rex.w=1
+         * @oppfx       0x66
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdint_datamov
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(VEX_RM, VMOVQ, vmovq, Vq_WO, Eq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* XMM, greg64 */
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_GREG_U64(u64Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem64] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U64(u64Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x6e
+         * @opcodesub   rex.w=0
+         * @oppfx       0x66
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdint_datamov
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovd_q_Vy_Ey
+         * @optest      op1=1 op2=2   -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(VEX_RM, VMOVD, vmovd, Vd_WO, Ed, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* XMM, greg32 */
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_GREG_U32(u32Tmp, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* XMM, [mem32] */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U32(u32Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/*  Opcode VEX.F3.0F 0x6e - invalid */
+
+
+/*  Opcode VEX.0F 0x6f - invalid */
+
+/**
+ * @opcode      0x6f
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdint_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovdqa_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVDQA, vmovdqa, Vx_WO, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * Register, memory128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128_ALIGN_SSE(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory256.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U, u256Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U256_ALIGN_AVX(u256Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u256Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/**
+ * @opcode      0x6f
+ * @oppfx       0xf3
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdint_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovdqu_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVDQU, vmovdqu, Vx_WO, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                           IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * Register, memory128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory256.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U, u256Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U256(u256Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u256Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode VEX.0F 0x70 - invalid */
+
+
+/**
+ * Common worker for AVX/AVX2 instructions on the forms:
+ *     - vpxxx    xmm0, xmm2/mem128, imm8
+ *     - vpxxx    ymm0, ymm2/mem256, imm8
+ *
+ * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit.
+ */
+FNIEMOP_DEF_2(iemOpCommonAvxAvx2_vpshufXX_Vx_Wx_Ib, PFNIEMAIMPLMEDIAPSHUFU128, pfnU128, PFNIEMAIMPLMEDIAPSHUFU256, pfnU256)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx2);
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,  uDst,  0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc,  uSrc,  1);
+            IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/ bImm, 2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_FETCH_YREG_U256(uSrc,        IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_3(pfnU256, puDst, puSrc, bImmArg);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(    IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx);
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(PRTUINT128U,             puDst,                 0);
+            IEM_MC_ARG(PCRTUINT128U,            puSrc,                 1);
+            IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/ bImm, 2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc,   IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg);
+            IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEM_MC_BEGIN(3, 3);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,          uDst,  0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc,          uSrc,  1);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx2);
+            IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/ bImm, 2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256_NO_AC(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_CALL_VOID_AIMPL_3(pfnU256, puDst, puSrc, bImmArg);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(    IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 1);
+            IEM_MC_LOCAL(RTUINT128U,            uSrc);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_ARG(PRTUINT128U,             puDst,                 0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,  puSrc, uSrc,           1);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx);
+            IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/ bImm, 2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U128_NO_AC(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_3(pfnU128, puDst, puSrc, bImmArg);
+            IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode VEX.66.0F 0x70 - vpshufd Vx, Wx, Ib */
+FNIEMOP_DEF(iemOp_vpshufd_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(VEX_RMI, VPSHUFD, vpshufd, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonAvxAvx2_vpshufXX_Vx_Wx_Ib, iemAImpl_pshufd_u128,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vpshufd_u256, iemAImpl_vpshufd_u256_fallback));
+
+}
+
+
+/** Opcode VEX.F3.0F 0x70 - vpshufhw Vx, Wx, Ib */
+FNIEMOP_DEF(iemOp_vpshufhw_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(VEX_RMI, VPSHUFHW, vpshufhw, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonAvxAvx2_vpshufXX_Vx_Wx_Ib, iemAImpl_pshufhw_u128,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vpshufhw_u256, iemAImpl_vpshufhw_u256_fallback));
+
+}
+
+
+/** Opcode VEX.F2.0F 0x70 - vpshuflw Vx, Wx, Ib */
+FNIEMOP_DEF(iemOp_vpshuflw_Vx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(VEX_RMI, VPSHUFLW, vpshuflw, Vx, Wx, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    return FNIEMOP_CALL_2(iemOpCommonAvxAvx2_vpshufXX_Vx_Wx_Ib, iemAImpl_pshuflw_u128,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vpshuflw_u256, iemAImpl_vpshuflw_u256_fallback));
+}
+
+
+/*  Opcode VEX.0F 0x71 11/2 - invalid. */
+/** Opcode VEX.66.0F 0x71 11/2. */
+FNIEMOP_STUB_1(iemOp_VGrp12_vpsrlw_Hx_Ux_Ib, uint8_t, bRm);
+
+/*  Opcode VEX.0F 0x71 11/4 - invalid */
+/** Opcode VEX.66.0F 0x71 11/4. */
+FNIEMOP_STUB_1(iemOp_VGrp12_vpsraw_Hx_Ux_Ib, uint8_t, bRm);
+
+/*  Opcode VEX.0F 0x71 11/6 - invalid */
+/** Opcode VEX.66.0F 0x71 11/6. */
+FNIEMOP_STUB_1(iemOp_VGrp12_vpsllw_Hx_Ux_Ib, uint8_t, bRm);
+
+
+/**
+ * VEX Group 12 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnVexGroup12RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp12_vpsrlw_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /4 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp12_vpsraw_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp12_vpsllw_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8)
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexGroup12RegReg) == 8*4);
+
+
+/** Opcode VEX.0F 0x71. */
+FNIEMOP_DEF(iemOp_VGrp12)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnVexGroup12RegReg[  IEM_GET_MODRM_REG_8(bRm) * 4
+                                                     + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/*  Opcode VEX.0F 0x72 11/2 - invalid. */
+/** Opcode VEX.66.0F 0x72 11/2. */
+FNIEMOP_STUB_1(iemOp_VGrp13_vpsrld_Hx_Ux_Ib, uint8_t, bRm);
+
+/*  Opcode VEX.0F 0x72 11/4 - invalid. */
+/** Opcode VEX.66.0F 0x72 11/4. */
+FNIEMOP_STUB_1(iemOp_VGrp13_vpsrad_Hx_Ux_Ib, uint8_t, bRm);
+
+/*  Opcode VEX.0F 0x72 11/6 - invalid. */
+/** Opcode VEX.66.0F 0x72 11/6. */
+FNIEMOP_STUB_1(iemOp_VGrp13_vpslld_Hx_Ux_Ib, uint8_t, bRm);
+
+
+/**
+ * Group 13 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnVexGroup13RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp13_vpsrld_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /4 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp13_vpsrad_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_InvalidWithRMNeedImm8,   iemOp_VGrp13_vpslld_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8)
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexGroup13RegReg) == 8*4);
+
+/** Opcode VEX.0F 0x72. */
+FNIEMOP_DEF(iemOp_VGrp13)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnVexGroup13RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4
+                                                     + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/*  Opcode VEX.0F 0x73 11/2 - invalid. */
+/** Opcode VEX.66.0F 0x73 11/2. */
+FNIEMOP_STUB_1(iemOp_VGrp14_vpsrlq_Hx_Ux_Ib, uint8_t, bRm);
+
+/** Opcode VEX.66.0F 0x73 11/3. */
+FNIEMOP_STUB_1(iemOp_VGrp14_vpsrldq_Hx_Ux_Ib, uint8_t, bRm);
+
+/*  Opcode VEX.0F 0x73 11/6 - invalid. */
+/** Opcode VEX.66.0F 0x73 11/6. */
+FNIEMOP_STUB_1(iemOp_VGrp14_vpsllq_Hx_Ux_Ib, uint8_t, bRm);
+
+/** Opcode VEX.66.0F 0x73 11/7. */
+FNIEMOP_STUB_1(iemOp_VGrp14_vpslldq_Hx_Ux_Ib, uint8_t, bRm);
+
+/**
+ * Group 14 jump table for register variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnVexGroup14RegReg[] =
+{
+    /* /0 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /1 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /2 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpsrlq_Hx_Ux_Ib,  iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /3 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpsrldq_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /4 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /5 */ IEMOP_X4(iemOp_InvalidWithRMNeedImm8),
+    /* /6 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpsllq_Hx_Ux_Ib,  iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+    /* /7 */ iemOp_InvalidWithRMNeedImm8, iemOp_VGrp14_vpslldq_Hx_Ux_Ib, iemOp_InvalidWithRMNeedImm8, iemOp_InvalidWithRMNeedImm8,
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexGroup14RegReg) == 8*4);
+
+
+/** Opcode VEX.0F 0x73. */
+FNIEMOP_DEF(iemOp_VGrp14)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        /* register, register */
+        return FNIEMOP_CALL_1(g_apfnVexGroup14RegReg[ IEM_GET_MODRM_REG_8(bRm) * 4
+                                                     + pVCpu->iem.s.idxPrefix], bRm);
+    return FNIEMOP_CALL_1(iemOp_InvalidWithRMNeedImm8, bRm);
+}
+
+
+/*  Opcode VEX.0F 0x74 - invalid */
+
+
+/** Opcode VEX.66.0F 0x74 - vpcmpeqb Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpcmpeqb_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPCMPEQB, vpcmpeqb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpcmpeqb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+/*  Opcode VEX.F3.0F 0x74 - invalid */
+/*  Opcode VEX.F2.0F 0x74 - invalid */
+
+
+/*  Opcode VEX.0F 0x75 - invalid */
+
+
+/** Opcode VEX.66.0F 0x75 - vpcmpeqw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpcmpeqw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPCMPEQW, vpcmpeqw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpcmpeqw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x75 - invalid */
+/*  Opcode VEX.F2.0F 0x75 - invalid */
+
+
+/*  Opcode VEX.0F 0x76 - invalid */
+
+
+/** Opcode VEX.66.0F 0x76 - vpcmpeqd Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpcmpeqd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPCMPEQD, vpcmpeqd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpcmpeqd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0x76 - invalid */
+/*  Opcode VEX.F2.0F 0x76 - invalid */
+
+
+/** Opcode VEX.0F 0x77 - vzeroupperv vzeroallv */
+FNIEMOP_DEF(iemOp_vzeroupperv__vzeroallv)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * 128-bit: vzeroupper
+         */
+        IEMOP_MNEMONIC(vzeroupper, "vzeroupper");
+        IEM_MC_BEGIN(0, 0);
+
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_CLEAR_YREG_128_UP(0);
+        IEM_MC_CLEAR_YREG_128_UP(1);
+        IEM_MC_CLEAR_YREG_128_UP(2);
+        IEM_MC_CLEAR_YREG_128_UP(3);
+        IEM_MC_CLEAR_YREG_128_UP(4);
+        IEM_MC_CLEAR_YREG_128_UP(5);
+        IEM_MC_CLEAR_YREG_128_UP(6);
+        IEM_MC_CLEAR_YREG_128_UP(7);
+
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_CLEAR_YREG_128_UP( 8);
+            IEM_MC_CLEAR_YREG_128_UP( 9);
+            IEM_MC_CLEAR_YREG_128_UP(10);
+            IEM_MC_CLEAR_YREG_128_UP(11);
+            IEM_MC_CLEAR_YREG_128_UP(12);
+            IEM_MC_CLEAR_YREG_128_UP(13);
+            IEM_MC_CLEAR_YREG_128_UP(14);
+            IEM_MC_CLEAR_YREG_128_UP(15);
+        }
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * 256-bit: vzeroall
+         */
+        IEMOP_MNEMONIC(vzeroall, "vzeroall");
+        IEM_MC_BEGIN(0, 1);
+        IEM_MC_LOCAL(uint32_t,  uZero);
+
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_ASSIGN(uZero, 0);
+        IEM_MC_STORE_YREG_U32_ZX_VLMAX(0, uZero);
+        IEM_MC_STORE_YREG_U32_ZX_VLMAX(1, uZero);
+        IEM_MC_STORE_YREG_U32_ZX_VLMAX(2, uZero);
+        IEM_MC_STORE_YREG_U32_ZX_VLMAX(3, uZero);
+        IEM_MC_STORE_YREG_U32_ZX_VLMAX(4, uZero);
+        IEM_MC_STORE_YREG_U32_ZX_VLMAX(5, uZero);
+        IEM_MC_STORE_YREG_U32_ZX_VLMAX(6, uZero);
+        IEM_MC_STORE_YREG_U32_ZX_VLMAX(7, uZero);
+
+        if (pVCpu->iem.s.enmEffOpSize == IEMMODE_64BIT)
+        {
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX( 8, uZero);
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX( 9, uZero);
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX(10, uZero);
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX(11, uZero);
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX(12, uZero);
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX(13, uZero);
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX(14, uZero);
+            IEM_MC_STORE_YREG_U32_ZX_VLMAX(15, uZero);
+        }
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode VEX.66.0F 0x77 - invalid */
+/*  Opcode VEX.F3.0F 0x77 - invalid */
+/*  Opcode VEX.F2.0F 0x77 - invalid */
+
+/*  Opcode VEX.0F 0x78 - invalid */
+/*  Opcode VEX.66.0F 0x78 - invalid */
+/*  Opcode VEX.F3.0F 0x78 - invalid */
+/*  Opcode VEX.F2.0F 0x78 - invalid */
+
+/*  Opcode VEX.0F 0x79 - invalid */
+/*  Opcode VEX.66.0F 0x79 - invalid */
+/*  Opcode VEX.F3.0F 0x79 - invalid */
+/*  Opcode VEX.F2.0F 0x79 - invalid */
+
+/*  Opcode VEX.0F 0x7a - invalid */
+/*  Opcode VEX.66.0F 0x7a - invalid */
+/*  Opcode VEX.F3.0F 0x7a - invalid */
+/*  Opcode VEX.F2.0F 0x7a - invalid */
+
+/*  Opcode VEX.0F 0x7b - invalid */
+/*  Opcode VEX.66.0F 0x7b - invalid */
+/*  Opcode VEX.F3.0F 0x7b - invalid */
+/*  Opcode VEX.F2.0F 0x7b - invalid */
+
+/*  Opcode VEX.0F 0x7c - invalid */
+/** Opcode VEX.66.0F 0x7c - vhaddpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vhaddpd_Vpd_Hpd_Wpd);
+/*  Opcode VEX.F3.0F 0x7c - invalid */
+/** Opcode VEX.F2.0F 0x7c - vhaddps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vhaddps_Vps_Hps_Wps);
+
+/*  Opcode VEX.0F 0x7d - invalid */
+/** Opcode VEX.66.0F 0x7d - vhsubpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vhsubpd_Vpd_Hpd_Wpd);
+/*  Opcode VEX.F3.0F 0x7d - invalid */
+/** Opcode VEX.F2.0F 0x7d - vhsubps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vhsubps_Vps_Hps_Wps);
+
+
+/*  Opcode VEX.0F 0x7e - invalid */
+
+FNIEMOP_DEF(iemOp_vmovd_q_Ey_Vy)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+    {
+        /**
+         * @opcode      0x7e
+         * @opcodesub   rex.w=1
+         * @oppfx       0x66
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdint_datamov
+         * @opxcpttype  5
+         * @optest      64-bit / op1=1 op2=2   -> op1=2
+         * @optest      64-bit / op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(VEX_MR, VMOVQ, vmovq, Eq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg64, XMM */
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U64(u64Tmp, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_GREG_U64(IEM_GET_MODRM_RM(pVCpu, bRm), u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem64], XMM */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint64_t, u64Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U64(u64Tmp, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u64Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /**
+         * @opdone
+         * @opcode      0x7e
+         * @opcodesub   rex.w=0
+         * @oppfx       0x66
+         * @opcpuid     avx
+         * @opgroup     og_avx_simdint_datamov
+         * @opxcpttype  5
+         * @opfunction  iemOp_vmovd_q_Vy_Ey
+         * @optest      op1=1 op2=2   -> op1=2
+         * @optest      op1=0 op2=-42 -> op1=-42
+         */
+        IEMOP_MNEMONIC2(VEX_MR, VMOVD, vmovd, Ed_WO, Vd, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OZ_PFX | IEMOPHINT_VEX_L_ZERO);
+        if (IEM_IS_MODRM_REG_MODE(bRm))
+        {
+            /* greg32, XMM */
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_BEGIN(0, 1);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U32(u32Tmp, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_RM(pVCpu, bRm), u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /* [mem32], XMM */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffSrc);
+            IEM_MC_LOCAL(uint32_t, u32Tmp);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U32(u32Tmp, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U32(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u32Tmp);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/**
+ * @opcode      0x7e
+ * @oppfx       0xf3
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  none
+ * @optest      op1=1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovq_Vq_Wq)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VMOVQ, vmovq, Vq_WO, Wq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_COPY_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm),
+                                      IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+
+}
+/*  Opcode VEX.F2.0F 0x7e - invalid */
+
+
+/*  Opcode VEX.0F 0x7f - invalid */
+
+/**
+ * @opcode      0x7f
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdint_datamove
+ * @opxcpttype  1
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovdqa_Wx_Vx)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVDQA, vmovdqa, Wx_WO, Vx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * Register, memory128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U128(u128Tmp, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory256.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U, u256Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U256(u256Tmp, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u256Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/**
+ * @opcode      0x7f
+ * @oppfx       0xf3
+ * @opcpuid     avx
+ * @opgroup     og_avx_simdint_datamove
+ * @opxcpttype  4UA
+ * @optest      op1=1 op2=2   -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovdqu_Wx_Vx)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVDQU, vmovdqu, Wx_WO, Vx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+        if (pVCpu->iem.s.uVexLength == 0)
+            IEM_MC_COPY_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm));
+        else
+            IEM_MC_COPY_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                           IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * Register, memory128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U128(u128Tmp, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U128(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u128Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory256.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U, u256Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U256(u256Tmp, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U256(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, u256Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/*  Opcode VEX.F2.0F 0x7f - invalid */
+
+
+/*  Opcode VEX.0F 0x80 - invalid  */
+/*  Opcode VEX.0F 0x81 - invalid  */
+/*  Opcode VEX.0F 0x82 - invalid  */
+/*  Opcode VEX.0F 0x83 - invalid  */
+/*  Opcode VEX.0F 0x84 - invalid  */
+/*  Opcode VEX.0F 0x85 - invalid  */
+/*  Opcode VEX.0F 0x86 - invalid  */
+/*  Opcode VEX.0F 0x87 - invalid  */
+/*  Opcode VEX.0F 0x88 - invalid  */
+/*  Opcode VEX.0F 0x89 - invalid  */
+/*  Opcode VEX.0F 0x8a - invalid  */
+/*  Opcode VEX.0F 0x8b - invalid  */
+/*  Opcode VEX.0F 0x8c - invalid  */
+/*  Opcode VEX.0F 0x8d - invalid  */
+/*  Opcode VEX.0F 0x8e - invalid  */
+/*  Opcode VEX.0F 0x8f - invalid  */
+/*  Opcode VEX.0F 0x90 - invalid  */
+/*  Opcode VEX.0F 0x91 - invalid  */
+/*  Opcode VEX.0F 0x92 - invalid  */
+/*  Opcode VEX.0F 0x93 - invalid  */
+/*  Opcode VEX.0F 0x94 - invalid  */
+/*  Opcode VEX.0F 0x95 - invalid  */
+/*  Opcode VEX.0F 0x96 - invalid  */
+/*  Opcode VEX.0F 0x97 - invalid  */
+/*  Opcode VEX.0F 0x98 - invalid  */
+/*  Opcode VEX.0F 0x99 - invalid  */
+/*  Opcode VEX.0F 0x9a - invalid  */
+/*  Opcode VEX.0F 0x9b - invalid  */
+/*  Opcode VEX.0F 0x9c - invalid  */
+/*  Opcode VEX.0F 0x9d - invalid  */
+/*  Opcode VEX.0F 0x9e - invalid  */
+/*  Opcode VEX.0F 0x9f - invalid  */
+/*  Opcode VEX.0F 0xa0 - invalid  */
+/*  Opcode VEX.0F 0xa1 - invalid  */
+/*  Opcode VEX.0F 0xa2 - invalid  */
+/*  Opcode VEX.0F 0xa3 - invalid  */
+/*  Opcode VEX.0F 0xa4 - invalid  */
+/*  Opcode VEX.0F 0xa5 - invalid  */
+/*  Opcode VEX.0F 0xa6 - invalid  */
+/*  Opcode VEX.0F 0xa7 - invalid  */
+/*  Opcode VEX.0F 0xa8 - invalid  */
+/*  Opcode VEX.0F 0xa9 - invalid  */
+/*  Opcode VEX.0F 0xaa - invalid  */
+/*  Opcode VEX.0F 0xab - invalid  */
+/*  Opcode VEX.0F 0xac - invalid  */
+/*  Opcode VEX.0F 0xad - invalid  */
+
+
+/*  Opcode VEX.0F 0xae mem/0 - invalid. */
+/*  Opcode VEX.0F 0xae mem/1 - invalid. */
+
+/**
+ * @ opmaps      grp15
+ * @ opcode      !11/2
+ * @ oppfx       none
+ * @ opcpuid     sse
+ * @ opgroup     og_sse_mxcsrsm
+ * @ opxcpttype  5
+ * @ optest      op1=0      -> mxcsr=0
+ * @ optest      op1=0x2083 -> mxcsr=0x2083
+ * @ optest      op1=0xfffffffe -> value.xcpt=0xd
+ * @ optest      op1=0x2083 cr0|=ts -> value.xcpt=0x7
+ * @ optest      op1=0x2083 cr0|=em -> value.xcpt=0x6
+ * @ optest      op1=0x2083 cr0|=mp -> mxcsr=0x2083
+ * @ optest      op1=0x2083 cr4&~=osfxsr -> value.xcpt=0x6
+ * @ optest      op1=0x2083 cr0|=ts,em -> value.xcpt=0x6
+ * @ optest      op1=0x2083 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6
+ * @ optest      op1=0x2083 cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6
+ * @ optest      op1=0x2083 cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6
+ */
+FNIEMOP_STUB_1(iemOp_VGrp15_vldmxcsr, uint8_t, bRm);
+//FNIEMOP_DEF_1(iemOp_VGrp15_vldmxcsr, uint8_t, bRm)
+//{
+//    IEMOP_MNEMONIC1(M_MEM, VLDMXCSR, vldmxcsr, MdRO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES);
+//    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fSse)
+//        return IEMOP_RAISE_INVALID_OPCODE();
+//
+//    IEM_MC_BEGIN(2, 0);
+//    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+//    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+//    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+//    IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX();
+//    IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+//    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+//    IEM_MC_CALL_CIMPL_2(iemCImpl_ldmxcsr, iEffSeg, GCPtrEff);
+//    IEM_MC_END();
+//    return VINF_SUCCESS;
+//}
+
+
+/**
+ * @opmaps      vexgrp15
+ * @opcode      !11/3
+ * @oppfx       none
+ * @opcpuid     avx
+ * @opgroup     og_avx_mxcsrsm
+ * @opxcpttype  5
+ * @optest      mxcsr=0      -> op1=0
+ * @optest      mxcsr=0x2083 -> op1=0x2083
+ * @optest      mxcsr=0x2084 cr0|=ts -> value.xcpt=0x7
+ * @optest      !amd / mxcsr=0x2085 cr0|=em -> op1=0x2085
+ * @optest       amd / mxcsr=0x2085 cr0|=em -> value.xcpt=0x6
+ * @optest      mxcsr=0x2086 cr0|=mp -> op1=0x2086
+ * @optest      mxcsr=0x2087 cr4&~=osfxsr -> op1=0x2087
+ * @optest      mxcsr=0x208f cr4&~=osxsave -> value.xcpt=0x6
+ * @optest      mxcsr=0x2087 cr4&~=osfxsr,osxsave -> value.xcpt=0x6
+ * @optest      !amd / mxcsr=0x2088 cr0|=ts,em -> value.xcpt=0x7
+ * @optest      amd  / mxcsr=0x2088 cr0|=ts,em -> value.xcpt=0x6
+ * @optest      !amd / mxcsr=0x2089 cr0|=em cr4&~=osfxsr -> op1=0x2089
+ * @optest      amd  / mxcsr=0x2089 cr0|=em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      !amd / mxcsr=0x208a cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x7
+ * @optest      amd  / mxcsr=0x208a cr0|=ts,em cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      !amd / mxcsr=0x208b cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x7
+ * @optest      amd  / mxcsr=0x208b cr0|=ts,em,mp cr4&~=osfxsr -> value.xcpt=0x6
+ * @optest      !amd / mxcsr=0x208c xcr0&~=all_avx -> value.xcpt=0x6
+ * @optest      amd  / mxcsr=0x208c xcr0&~=all_avx -> op1=0x208c
+ * @optest      !amd / mxcsr=0x208d xcr0&~=all_avx_sse -> value.xcpt=0x6
+ * @optest      amd  / mxcsr=0x208d xcr0&~=all_avx_sse -> op1=0x208d
+ * @optest      !amd / mxcsr=0x208e xcr0&~=all_avx cr0|=ts -> value.xcpt=0x6
+ * @optest      amd  / mxcsr=0x208e xcr0&~=all_avx cr0|=ts -> value.xcpt=0x7
+ * @optest      mxcsr=0x2082 cr0|=ts cr4&~=osxsave -> value.xcpt=0x6
+ * @optest      mxcsr=0x2081 xcr0&~=all_avx cr0|=ts cr4&~=osxsave
+ *              -> value.xcpt=0x6
+ * @remarks     AMD Jaguar CPU (f0x16,m0,s1) \#UD when CR0.EM is set.  It also
+ *              doesn't seem to check XCR0[2:1] != 11b.  This does not match the
+ *              APMv4 rev 3.17 page 509.
+ * @todo        Test this instruction on AMD Ryzen.
+ */
+FNIEMOP_DEF_1(iemOp_VGrp15_vstmxcsr,  uint8_t, bRm)
+{
+    IEMOP_MNEMONIC1(VEX_M_MEM, VSTMXCSR, vstmxcsr, Md_WO, DISOPTYPE_HARMLESS, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+    IEM_MC_BEGIN(2, 0);
+    IEM_MC_ARG(uint8_t,         iEffSeg,                                 0);
+    IEM_MC_ARG(RTGCPTR,         GCPtrEff,                                1);
+    IEM_MC_CALC_RM_EFF_ADDR(GCPtrEff, bRm, 0);
+    IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+    IEM_MC_ACTUALIZE_SSE_STATE_FOR_READ();
+    IEM_MC_ASSIGN(iEffSeg, pVCpu->iem.s.iEffSeg);
+    IEM_MC_CALL_CIMPL_2(iemCImpl_vstmxcsr, iEffSeg, GCPtrEff);
+    IEM_MC_END();
+    return VINF_SUCCESS;
+}
+
+/*  Opcode VEX.0F 0xae mem/4 - invalid. */
+/*  Opcode VEX.0F 0xae mem/5 - invalid. */
+/*  Opcode VEX.0F 0xae mem/6 - invalid. */
+/*  Opcode VEX.0F 0xae mem/7 - invalid. */
+
+/*  Opcode VEX.0F 0xae 11b/0 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/1 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/2 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/3 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/4 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/5 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/6 - invalid. */
+/*  Opcode VEX.0F 0xae 11b/7 - invalid. */
+
+/**
+ * Vex group 15 jump table for memory variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnVexGroup15MemReg[] =
+{   /* pfx:  none,                          066h,                           0f3h,                           0f2h */
+    /* /0 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /1 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /2 */ iemOp_VGrp15_vldmxcsr,         iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /3 */ iemOp_VGrp15_vstmxcsr,         iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /4 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /5 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /6 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+    /* /7 */ iemOp_InvalidWithRM,           iemOp_InvalidWithRM,            iemOp_InvalidWithRM,            iemOp_InvalidWithRM,
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexGroup15MemReg) == 8*4);
+
+
+/** Opcode vex. 0xae. */
+FNIEMOP_DEF(iemOp_VGrp15)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+        /* register, register */
+        return FNIEMOP_CALL_1(iemOp_InvalidWithRM, bRm);
+
+    /* memory, register */
+    return FNIEMOP_CALL_1(g_apfnVexGroup15MemReg[ IEM_GET_MODRM_REG_8(bRm) * 4
+                                                 + pVCpu->iem.s.idxPrefix], bRm);
+}
+
+
+/*  Opcode VEX.0F 0xaf - invalid. */
+
+/*  Opcode VEX.0F 0xb0 - invalid. */
+/*  Opcode VEX.0F 0xb1 - invalid. */
+/*  Opcode VEX.0F 0xb2 - invalid. */
+/*  Opcode VEX.0F 0xb2 - invalid. */
+/*  Opcode VEX.0F 0xb3 - invalid. */
+/*  Opcode VEX.0F 0xb4 - invalid. */
+/*  Opcode VEX.0F 0xb5 - invalid. */
+/*  Opcode VEX.0F 0xb6 - invalid. */
+/*  Opcode VEX.0F 0xb7 - invalid. */
+/*  Opcode VEX.0F 0xb8 - invalid. */
+/*  Opcode VEX.0F 0xb9 - invalid. */
+/*  Opcode VEX.0F 0xba - invalid. */
+/*  Opcode VEX.0F 0xbb - invalid. */
+/*  Opcode VEX.0F 0xbc - invalid. */
+/*  Opcode VEX.0F 0xbd - invalid. */
+/*  Opcode VEX.0F 0xbe - invalid. */
+/*  Opcode VEX.0F 0xbf - invalid. */
+
+/*  Opcode VEX.0F 0xc0 - invalid. */
+/*  Opcode VEX.66.0F 0xc0 - invalid. */
+/*  Opcode VEX.F3.0F 0xc0 - invalid. */
+/*  Opcode VEX.F2.0F 0xc0 - invalid. */
+
+/*  Opcode VEX.0F 0xc1 - invalid. */
+/*  Opcode VEX.66.0F 0xc1 - invalid. */
+/*  Opcode VEX.F3.0F 0xc1 - invalid. */
+/*  Opcode VEX.F2.0F 0xc1 - invalid. */
+
+/** Opcode VEX.0F 0xc2 - vcmpps Vps,Hps,Wps,Ib */
+FNIEMOP_STUB(iemOp_vcmpps_Vps_Hps_Wps_Ib);
+/** Opcode VEX.66.0F 0xc2 - vcmppd Vpd,Hpd,Wpd,Ib */
+FNIEMOP_STUB(iemOp_vcmppd_Vpd_Hpd_Wpd_Ib);
+/** Opcode VEX.F3.0F 0xc2 - vcmpss Vss,Hss,Wss,Ib */
+FNIEMOP_STUB(iemOp_vcmpss_Vss_Hss_Wss_Ib);
+/** Opcode VEX.F2.0F 0xc2 - vcmpsd Vsd,Hsd,Wsd,Ib */
+FNIEMOP_STUB(iemOp_vcmpsd_Vsd_Hsd_Wsd_Ib);
+
+/*  Opcode VEX.0F 0xc3 - invalid */
+/*  Opcode VEX.66.0F 0xc3 - invalid */
+/*  Opcode VEX.F3.0F 0xc3 - invalid */
+/*  Opcode VEX.F2.0F 0xc3 - invalid */
+
+/*  Opcode VEX.0F 0xc4 - invalid */
+
+
+/** Opcode VEX.66.0F 0xc4 - vpinsrw Vdq,Hdq,Ry/Mw,Ib */
+FNIEMOP_DEF(iemOp_vpinsrw_Vdq_Hdq_RyMw_Ib)
+{
+    /*IEMOP_MNEMONIC4(VEX_RMV, VPINSRW, vpinsrw, Vdq, Vdq, Ey, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);*/ /** @todo */
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fAvx);
+        IEM_MC_BEGIN(4, 0);
+        IEM_MC_ARG(PRTUINT128U,          puDst,                 0);
+        IEM_MC_ARG(PCRTUINT128U,         puSrc,                 1);
+        IEM_MC_ARG(uint16_t,             u16Src,                2);
+        IEM_MC_ARG_CONST(uint8_t,        bImmArg, /*=*/ bImm,   3);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,        IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,  IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_FETCH_GREG_U16(u16Src, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vpinsrw_u128, iemAImpl_vpinsrw_u128_fallback),
+                                 puDst, puSrc, u16Src, bImmArg);
+        IEM_MC_CLEAR_YREG_128_UP(          IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 1);
+        IEM_MC_LOCAL(RTGCPTR,     GCPtrEffSrc);
+        IEM_MC_ARG(PRTUINT128U,   puDst,                 0);
+        IEM_MC_ARG(PCRTUINT128U, puSrc,                  1);
+        IEM_MC_ARG(uint16_t,      u16Src,                2);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t, bImmArg, /*=*/ bImm,   3);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fAvx);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+
+        IEM_MC_FETCH_MEM_U16(u16Src, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,  IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vpinsrw_u128, iemAImpl_vpinsrw_u128_fallback),
+                                 puDst, puSrc, u16Src, bImmArg);
+        IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm));
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode VEX.F3.0F 0xc4 - invalid */
+/*  Opcode VEX.F2.0F 0xc4 - invalid */
+
+/*  Opcode VEX.0F 0xc5 - invlid */
+
+
+/** Opcode VEX.66.0F 0xc5 - vpextrw Gd, Udq, Ib */
+FNIEMOP_DEF(iemOp_vpextrw_Gd_Udq_Ib)
+{
+    IEMOP_MNEMONIC3(VEX_RMI_REG, VPEXTRW, vpextrw, Gd, Ux, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fAvx);
+        IEM_MC_BEGIN(3, 1);
+        IEM_MC_LOCAL(uint16_t,              u16Dst);
+        IEM_MC_ARG_LOCAL_REF(uint16_t *,    pu16Dst,  u16Dst,      0);
+        IEM_MC_ARG(PCRTUINT128U,            puSrc,                 1);
+        IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/ bImm, 2);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_REF_XREG_U128_CONST(puSrc,  IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vpextrw_u128, iemAImpl_vpextrw_u128_fallback),
+                                 pu16Dst, puSrc, bImmArg);
+        IEM_MC_STORE_GREG_U32(IEM_GET_MODRM_REG(pVCpu, bRm), u16Dst);
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    /* No memory operand. */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/*  Opcode VEX.F3.0F 0xc5 - invalid */
+/*  Opcode VEX.F2.0F 0xc5 - invalid */
+
+
+#define VSHUFP_X(a_Instr) \
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \
+    if (IEM_IS_MODRM_REG_MODE(bRm)) \
+    { \
+        /* \
+         * Register, register. \
+         */ \
+        if (pVCpu->iem.s.uVexLength) \
+        { \
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); \
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); \
+            IEM_MC_BEGIN(4, 3); \
+            IEM_MC_LOCAL(RTUINT256U,            uDst); \
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1); \
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2); \
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,  uDst,          0); \
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1,         1); \
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2,         2); \
+            IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/ bImm,   3); \
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \
+            IEM_MC_PREPARE_AVX_USAGE(); \
+            IEM_MC_FETCH_YREG_U256(uSrc1,   IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_FETCH_YREG_U256(uSrc2,   IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u256, \
+                                                                 iemAImpl_ ## a_Instr ## _u256_fallback), puDst, puSrc1, puSrc2, bImmArg); \
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); \
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); \
+            IEM_MC_BEGIN(4, 0); \
+            IEM_MC_ARG(PRTUINT128U,          puDst,                 0); \
+            IEM_MC_ARG(PCRTUINT128U,         puSrc1,                1); \
+            IEM_MC_ARG(PCRTUINT128U,         puSrc2,                2); \
+            IEM_MC_ARG_CONST(uint8_t,        bImmArg, /*=*/ bImm,   3); \
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \
+            IEM_MC_PREPARE_AVX_USAGE(); \
+            IEM_MC_REF_XREG_U128(puDst,        IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u128, \
+                                                                 iemAImpl_ ## a_Instr ## _u128_fallback), puDst, puSrc1, puSrc2, bImmArg); \
+            IEM_MC_CLEAR_YREG_128_UP(          IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    else \
+    { \
+        /* \
+         * Register, memory. \
+         */ \
+        if (pVCpu->iem.s.uVexLength) \
+        { \
+            IEM_MC_BEGIN(4, 4); \
+            IEM_MC_LOCAL(RTUINT256U,            uDst); \
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1); \
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2); \
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc); \
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,          uDst,  0); \
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1,         uSrc1, 1); \
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2,         uSrc2, 2); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); \
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); \
+            IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/  bImm,  3); \
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2); \
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \
+            IEM_MC_PREPARE_AVX_USAGE(); \
+            IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_FETCH_YREG_U256(uSrc1,      IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u256, \
+                                                                 iemAImpl_ ## a_Instr ## _u256_fallback), puDst, puSrc1, puSrc2, bImmArg); \
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(   IEM_GET_MODRM_REG(pVCpu, bRm), uDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEM_MC_BEGIN(4, 2); \
+            IEM_MC_LOCAL(RTUINT128U,                uSrc2); \
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc); \
+            IEM_MC_ARG(PRTUINT128U,                 puDst,                 0); \
+            IEM_MC_ARG(PCRTUINT128U,                puSrc1,                1); \
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc2,         uSrc2, 2); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1); \
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm); \
+            IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/  bImm,  3); \
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx); \
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \
+            IEM_MC_PREPARE_AVX_USAGE(); \
+            IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_REF_XREG_U128_CONST(puSrc1,  IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u128, \
+                                                                 iemAImpl_ ## a_Instr ## _u128_fallback), puDst, puSrc1, puSrc2, bImmArg); \
+            IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    (void)0
+
+/** Opcode VEX.0F 0xc6 - vshufps Vps,Hps,Wps,Ib */
+FNIEMOP_DEF(iemOp_vshufps_Vps_Hps_Wps_Ib)
+{
+    IEMOP_MNEMONIC4(VEX_RMI, VSHUFPS, vshufps, Vpd, Hpd, Wpd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_SKIP_PYTHON); /** @todo */
+    VSHUFP_X(vshufps);
+}
+
+
+/** Opcode VEX.66.0F 0xc6 - vshufpd Vpd,Hpd,Wpd,Ib */
+FNIEMOP_DEF(iemOp_vshufpd_Vpd_Hpd_Wpd_Ib)
+{
+    IEMOP_MNEMONIC4(VEX_RMI, VSHUFPD, vshufpd, Vpd, Hpd, Wpd, Ib, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_SKIP_PYTHON); /** @todo */
+    VSHUFP_X(vshufpd);
+}
+#undef VSHUFP_X
+
+
+/*  Opcode VEX.F3.0F 0xc6 - invalid */
+/*  Opcode VEX.F2.0F 0xc6 - invalid */
+
+/*  Opcode VEX.0F 0xc7 - invalid */
+/*  Opcode VEX.66.0F 0xc7 - invalid */
+/*  Opcode VEX.F3.0F 0xc7 - invalid */
+/*  Opcode VEX.F2.0F 0xc7 - invalid */
+
+/*  Opcode VEX.0F 0xc8 - invalid */
+/*  Opcode VEX.0F 0xc9 - invalid */
+/*  Opcode VEX.0F 0xca - invalid */
+/*  Opcode VEX.0F 0xcb - invalid */
+/*  Opcode VEX.0F 0xcc - invalid */
+/*  Opcode VEX.0F 0xcd - invalid */
+/*  Opcode VEX.0F 0xce - invalid */
+/*  Opcode VEX.0F 0xcf - invalid */
+
+
+/*  Opcode VEX.0F 0xd0 - invalid */
+/** Opcode VEX.66.0F 0xd0 - vaddsubpd Vpd, Hpd, Wpd */
+FNIEMOP_STUB(iemOp_vaddsubpd_Vpd_Hpd_Wpd);
+/*  Opcode VEX.F3.0F 0xd0 - invalid */
+/** Opcode VEX.F2.0F 0xd0 - vaddsubps Vps, Hps, Wps */
+FNIEMOP_STUB(iemOp_vaddsubps_Vps_Hps_Wps);
+
+/*  Opcode VEX.0F 0xd1 - invalid */
+/** Opcode VEX.66.0F 0xd1 - vpsrlw Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsrlw_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xd1 - invalid */
+/*  Opcode VEX.F2.0F 0xd1 - invalid */
+
+/*  Opcode VEX.0F 0xd2 - invalid */
+/** Opcode VEX.66.0F 0xd2 - vpsrld Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsrld_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xd2 - invalid */
+/*  Opcode VEX.F2.0F 0xd2 - invalid */
+
+/*  Opcode VEX.0F 0xd3 - invalid */
+/** Opcode VEX.66.0F 0xd3 - vpsrlq Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsrlq_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xd3 - invalid */
+/*  Opcode VEX.F2.0F 0xd3 - invalid */
+
+/*  Opcode VEX.0F 0xd4 - invalid */
+
+
+/** Opcode VEX.66.0F 0xd4 - vpaddq Vx, Hx, W */
+FNIEMOP_DEF(iemOp_vpaddq_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPADDQ, vpaddq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpaddq);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xd4 - invalid */
+/*  Opcode VEX.F2.0F 0xd4 - invalid */
+
+/*  Opcode VEX.0F 0xd5 - invalid */
+
+
+/** Opcode VEX.66.0F 0xd5 - vpmullw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpmullw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMULLW, vpmullw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpmullw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xd5 - invalid */
+/*  Opcode VEX.F2.0F 0xd5 - invalid */
+
+/*  Opcode VEX.0F 0xd6 - invalid */
+
+/**
+ * @opcode      0xd6
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_pcksclr_datamove
+ * @opxcpttype  none
+ * @optest      op1=-1 op2=2 -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovq_Wq_Vq)
+{
+    IEMOP_MNEMONIC2(VEX_MR, VMOVQ, vmovq, Wq_WO, Vq, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES | IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_BEGIN(0, 0);
+
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_COPY_YREG_U64_ZX_VLMAX(IEM_GET_MODRM_RM(pVCpu, bRm),
+                                      IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Memory, register.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(uint64_t,                  uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+        IEM_MC_FETCH_YREG_U64(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_STORE_MEM_U64(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+/*  Opcode VEX.F3.0F 0xd6 - invalid */
+/*  Opcode VEX.F2.0F 0xd6 - invalid */
+
+
+/*  Opcode VEX.0F 0xd7 - invalid */
+
+/** Opcode VEX.66.0F 0xd7 -  */
+FNIEMOP_DEF(iemOp_vpmovmskb_Gd_Ux)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    /* Docs says register only. */
+    if (IEM_IS_MODRM_REG_MODE(bRm)) /** @todo test that this is registers only. */
+    {
+        /* Note! Taking the lazy approch here wrt the high 32-bits of the GREG. */
+        IEMOP_MNEMONIC2(RM_REG, VPMOVMSKB, vpmovmskb, Gd, Ux, DISOPTYPE_SSE | DISOPTYPE_HARMLESS, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEM_MC_BEGIN(2, 1);
+            IEM_MC_ARG(uint64_t *,              puDst, 0);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc, uSrc, 1);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_GREG_U64(puDst,          IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_FETCH_YREG_U256(uSrc,        IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_vpmovmskb_u256,
+                                                                 iemAImpl_vpmovmskb_u256_fallback), puDst, puSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(2, 0);
+            IEM_MC_ARG(uint64_t *,              puDst, 0);
+            IEM_MC_ARG(PCRTUINT128U,            puSrc, 1);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_GREG_U64(puDst,          IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc,   IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_2(iemAImpl_pmovmskb_u128, puDst, puSrc);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/*  Opcode VEX.F3.0F 0xd7 - invalid */
+/*  Opcode VEX.F2.0F 0xd7 - invalid */
+
+
+/*  Opcode VEX.0F 0xd8 - invalid */
+/** Opcode VEX.66.0F 0xd8 - vpsubusb Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsubusb_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xd8 - invalid */
+/*  Opcode VEX.F2.0F 0xd8 - invalid */
+
+/*  Opcode VEX.0F 0xd9 - invalid */
+/** Opcode VEX.66.0F 0xd9 - vpsubusw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsubusw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xd9 - invalid */
+/*  Opcode VEX.F2.0F 0xd9 - invalid */
+
+/*  Opcode VEX.0F 0xda - invalid */
+
+
+/** Opcode VEX.66.0F 0xda - vpminub Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpminub_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMINUB, vpminub, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpminub);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xda - invalid */
+/*  Opcode VEX.F2.0F 0xda - invalid */
+
+/*  Opcode VEX.0F 0xdb - invalid */
+
+
+/** Opcode VEX.66.0F 0xdb - vpand Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpand_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPAND, vpand, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpand, &g_iemAImpl_vpand_fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xdb - invalid */
+/*  Opcode VEX.F2.0F 0xdb - invalid */
+
+/*  Opcode VEX.0F 0xdc - invalid */
+/** Opcode VEX.66.0F 0xdc - vpaddusb Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpaddusb_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xdc - invalid */
+/*  Opcode VEX.F2.0F 0xdc - invalid */
+
+/*  Opcode VEX.0F 0xdd - invalid */
+/** Opcode VEX.66.0F 0xdd - vpaddusw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpaddusw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xdd - invalid */
+/*  Opcode VEX.F2.0F 0xdd - invalid */
+
+/*  Opcode VEX.0F 0xde - invalid */
+
+
+/** Opcode VEX.66.0F 0xde - vpmaxub Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpmaxub_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMAXUB, vpmaxub, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpmaxub);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xde - invalid */
+/*  Opcode VEX.F2.0F 0xde - invalid */
+
+/*  Opcode VEX.0F 0xdf - invalid */
+
+
+/** Opcode VEX.66.0F 0xdf - vpandn Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpandn_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPANDN, vpandn, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpandn, &g_iemAImpl_vpandn_fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xdf - invalid */
+/*  Opcode VEX.F2.0F 0xdf - invalid */
+
+/*  Opcode VEX.0F 0xe0 - invalid */
+
+
+/** Opcode VEX.66.0F 0xe0 - vpavgb Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpavgb_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPAVGB, vpavgb, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpavgb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xe0 - invalid */
+/*  Opcode VEX.F2.0F 0xe0 - invalid */
+
+/*  Opcode VEX.0F 0xe1 - invalid */
+/** Opcode VEX.66.0F 0xe1 - vpsraw Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsraw_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xe1 - invalid */
+/*  Opcode VEX.F2.0F 0xe1 - invalid */
+
+/*  Opcode VEX.0F 0xe2 - invalid */
+/** Opcode VEX.66.0F 0xe2 - vpsrad Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsrad_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xe2 - invalid */
+/*  Opcode VEX.F2.0F 0xe2 - invalid */
+
+/*  Opcode VEX.0F 0xe3 - invalid */
+
+
+/** Opcode VEX.66.0F 0xe3 - vpavgw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpavgw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPAVGW, vpavgw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpavgw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xe3 - invalid */
+/*  Opcode VEX.F2.0F 0xe3 - invalid */
+
+/*  Opcode VEX.0F 0xe4 - invalid */
+
+
+/** Opcode VEX.66.0F 0xe4 - vpmulhuw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpmulhuw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMULHUW, vpmulhuw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpmulhuw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xe4 - invalid */
+/*  Opcode VEX.F2.0F 0xe4 - invalid */
+
+/*  Opcode VEX.0F 0xe5 - invalid */
+
+
+/** Opcode VEX.66.0F 0xe5 - vpmulhw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpmulhw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMULHW, vpmulhw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpmulhw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xe5 - invalid */
+/*  Opcode VEX.F2.0F 0xe5 - invalid */
+
+/*  Opcode VEX.0F 0xe6 - invalid */
+/** Opcode VEX.66.0F 0xe6 - vcvttpd2dq Vx, Wpd */
+FNIEMOP_STUB(iemOp_vcvttpd2dq_Vx_Wpd);
+/** Opcode VEX.F3.0F 0xe6 - vcvtdq2pd Vx, Wpd */
+FNIEMOP_STUB(iemOp_vcvtdq2pd_Vx_Wpd);
+/** Opcode VEX.F2.0F 0xe6 - vcvtpd2dq Vx, Wpd */
+FNIEMOP_STUB(iemOp_vcvtpd2dq_Vx_Wpd);
+
+
+/* Opcode VEX.0F 0xe7 - invalid */
+
+/**
+ * @opcode      0xe7
+ * @opcodesub   !11 mr/reg
+ * @oppfx       0x66
+ * @opcpuid     avx
+ * @opgroup     og_avx_cachect
+ * @opxcpttype  1
+ * @optest      op1=-1 op2=2  -> op1=2
+ * @optest      op1=0 op2=-42 -> op1=-42
+ */
+FNIEMOP_DEF(iemOp_vmovntdq_Mx_Vx)
+{
+    IEMOP_MNEMONIC2(VEX_MR_MEM, VMOVNTDQ, vmovntdq, Mx_WO, Vx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            /*
+             * 128-bit: Memory, register.
+             */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U128(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U128_ALIGN_SSE(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /*
+             * 256-bit: Memory, register.
+             */
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_READ();
+
+            IEM_MC_FETCH_YREG_U256(uSrc, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_STORE_MEM_U256_ALIGN_AVX(pVCpu->iem.s.iEffSeg, GCPtrEffSrc, uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    /**
+     * @opdone
+     * @opmnemonic  udvex660fe7reg
+     * @opcode      0xe7
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+/*  Opcode VEX.F3.0F 0xe7 - invalid */
+/*  Opcode VEX.F2.0F 0xe7 - invalid */
+
+
+/*  Opcode VEX.0F 0xe8 - invalid */
+/** Opcode VEX.66.0F 0xe8 - vpsubsb Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsubsb_Vx_Hx_W);
+/*  Opcode VEX.F3.0F 0xe8 - invalid */
+/*  Opcode VEX.F2.0F 0xe8 - invalid */
+
+/*  Opcode VEX.0F 0xe9 - invalid */
+/** Opcode VEX.66.0F 0xe9 - vpsubsw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsubsw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xe9 - invalid */
+/*  Opcode VEX.F2.0F 0xe9 - invalid */
+
+/*  Opcode VEX.0F 0xea - invalid */
+
+
+/** Opcode VEX.66.0F 0xea - vpminsw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpminsw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMINSW, vpminsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpminsw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xea - invalid */
+/*  Opcode VEX.F2.0F 0xea - invalid */
+
+/*  Opcode VEX.0F 0xeb - invalid */
+
+
+/** Opcode VEX.66.0F 0xeb - vpor Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpor_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPOR, vpor, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpor, &g_iemAImpl_vpor_fallback));
+}
+
+
+
+/*  Opcode VEX.F3.0F 0xeb - invalid */
+/*  Opcode VEX.F2.0F 0xeb - invalid */
+
+/*  Opcode VEX.0F 0xec - invalid */
+/** Opcode VEX.66.0F 0xec - vpaddsb Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpaddsb_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xec - invalid */
+/*  Opcode VEX.F2.0F 0xec - invalid */
+
+/*  Opcode VEX.0F 0xed - invalid */
+/** Opcode VEX.66.0F 0xed - vpaddsw Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpaddsw_Vx_Hx_Wx);
+/*  Opcode VEX.F3.0F 0xed - invalid */
+/*  Opcode VEX.F2.0F 0xed - invalid */
+
+/*  Opcode VEX.0F 0xee - invalid */
+
+
+/** Opcode VEX.66.0F 0xee - vpmaxsw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpmaxsw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMAXSW, vpmaxsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpmaxsw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xee - invalid */
+/*  Opcode VEX.F2.0F 0xee - invalid */
+
+
+/*  Opcode VEX.0F 0xef - invalid */
+
+
+/** Opcode VEX.66.0F 0xef - vpxor Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpxor_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPXOR, vpxor, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx,
+                          IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &g_iemAImpl_vpxor, &g_iemAImpl_vpxor_fallback));
+}
+
+
+/*  Opcode VEX.F3.0F 0xef - invalid */
+/*  Opcode VEX.F2.0F 0xef - invalid */
+
+/*  Opcode VEX.0F 0xf0 - invalid */
+/*  Opcode VEX.66.0F 0xf0 - invalid */
+
+
+/** Opcode VEX.F2.0F 0xf0 - vlddqu Vx, Mx */
+FNIEMOP_DEF(iemOp_vlddqu_Vx_Mx)
+{
+    IEMOP_MNEMONIC2(VEX_RM_MEM, VLDDQU, vlddqu, Vx_WO, Mx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register - (not implemented, assuming it raises \#UD).
+         */
+        return IEMOP_RAISE_INVALID_OPCODE();
+    }
+    else if (pVCpu->iem.s.uVexLength == 0)
+    {
+        /*
+         * Register, memory128.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT128U, u128Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U128(u128Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u128Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory256.
+         */
+        IEM_MC_BEGIN(0, 2);
+        IEM_MC_LOCAL(RTUINT256U, u256Tmp);
+        IEM_MC_LOCAL(RTGCPTR,    GCPtrEffSrc);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+        IEM_MC_FETCH_MEM_U256(u256Tmp, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), u256Tmp);
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode VEX.0F 0xf1 - invalid */
+/** Opcode VEX.66.0F 0xf1 - vpsllw Vx, Hx, W */
+FNIEMOP_STUB(iemOp_vpsllw_Vx_Hx_W);
+/*  Opcode VEX.F2.0F 0xf1 - invalid */
+
+/*  Opcode VEX.0F 0xf2 - invalid */
+/** Opcode VEX.66.0F 0xf2 - vpslld Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpslld_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xf2 - invalid */
+
+/*  Opcode VEX.0F 0xf3 - invalid */
+/** Opcode VEX.66.0F 0xf3 - vpsllq Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpsllq_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xf3 - invalid */
+
+/*  Opcode VEX.0F 0xf4 - invalid */
+
+
+/** Opcode VEX.66.0F 0xf4 - vpmuludq Vx, Hx, W */
+FNIEMOP_DEF(iemOp_vpmuludq_Vx_Hx_W)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMULUDQ, vpmuludq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpmuludq);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F2.0F 0xf4 - invalid */
+
+/*  Opcode VEX.0F 0xf5 - invalid */
+/** Opcode VEX.66.0F 0xf5 - vpmaddwd Vx, Hx, Wx */
+FNIEMOP_STUB(iemOp_vpmaddwd_Vx_Hx_Wx);
+/*  Opcode VEX.F2.0F 0xf5 - invalid */
+
+/*  Opcode VEX.0F 0xf6 - invalid */
+
+
+/** Opcode VEX.66.0F 0xf6 - vpsadbw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpsadbw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPSADBW, vpsadbw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpsadbw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F2.0F 0xf6 - invalid */
+
+/*  Opcode VEX.0F 0xf7 - invalid */
+/** Opcode VEX.66.0F 0xf7 - vmaskmovdqu Vdq, Udq */
+FNIEMOP_STUB(iemOp_vmaskmovdqu_Vdq_Udq);
+/*  Opcode VEX.F2.0F 0xf7 - invalid */
+
+/*  Opcode VEX.0F 0xf8 - invalid */
+
+
+/** Opcode VEX.66.0F 0xf8 - vpsubb Vx, Hx, W */
+FNIEMOP_DEF(iemOp_vpsubb_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPSUBB, vpsubb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpsubb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F2.0F 0xf8 - invalid */
+
+/*  Opcode VEX.0F 0xf9 - invalid */
+
+
+/** Opcode VEX.66.0F 0xf9 - vpsubw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpsubw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPSUBW, vpsubw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpsubw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F2.0F 0xf9 - invalid */
+
+/*  Opcode VEX.0F 0xfa - invalid */
+
+
+/** Opcode VEX.66.0F 0xfa - vpsubd Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpsubd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPSUBD, vpsubd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpsubd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F2.0F 0xfa - invalid */
+
+/*  Opcode VEX.0F 0xfb - invalid */
+
+
+/** Opcode VEX.66.0F 0xfb - vpsubq Vx, Hx, W */
+FNIEMOP_DEF(iemOp_vpsubq_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPSUBQ, vpsubq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpsubq);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F2.0F 0xfb - invalid */
+
+/*  Opcode VEX.0F 0xfc - invalid */
+
+
+/** Opcode VEX.66.0F 0xfc - vpaddb Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpaddb_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPADDB, vpaddb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpaddb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F2.0F 0xfc - invalid */
+
+/*  Opcode VEX.0F 0xfd - invalid */
+
+
+/** Opcode VEX.66.0F 0xfd - vpaddw Vx, Hx, Wx */
+FNIEMOP_DEF(iemOp_vpaddw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPADDW, vpaddw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpaddw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F2.0F 0xfd - invalid */
+
+/*  Opcode VEX.0F 0xfe - invalid */
+
+
+/** Opcode VEX.66.0F 0xfe - vpaddd Vx, Hx, W */
+FNIEMOP_DEF(iemOp_vpaddd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPADDD, vpaddd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(  vpaddd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.F2.0F 0xfe - invalid */
+
+
+/** Opcode **** 0x0f 0xff - UD0 */
+FNIEMOP_DEF(iemOp_vud0)
+{
+    IEMOP_MNEMONIC(vud0, "vud0");
+    if (pVCpu->iem.s.enmCpuVendor == CPUMCPUVENDOR_INTEL)
+    {
+        uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); RT_NOREF(bRm);
+#ifndef TST_IEM_CHECK_MC
+        RTGCPTR      GCPtrEff;
+        VBOXSTRICTRC rcStrict = iemOpHlpCalcRmEffAddr(pVCpu, bRm, 0, &GCPtrEff);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+#endif
+        IEMOP_HLP_DONE_DECODING();
+    }
+    return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+
+/**
+ * VEX opcode map \#1.
+ *
+ * @sa  g_apfnTwoByteMap
+ */
+IEM_STATIC const PFNIEMOP g_apfnVexMap1[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x01 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x02 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x03 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x04 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x05 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x06 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x07 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x08 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x09 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0b */  IEMOP_X4(iemOp_vud2), /* ?? */
+    /* 0x0c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x0f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x10 */  iemOp_vmovups_Vps_Wps,      iemOp_vmovupd_Vpd_Wpd,      iemOp_vmovss_Vss_Hss_Wss,   iemOp_vmovsd_Vsd_Hsd_Wsd,
+    /* 0x11 */  iemOp_vmovups_Wps_Vps,      iemOp_vmovupd_Wpd_Vpd,      iemOp_vmovss_Wss_Hss_Vss,   iemOp_vmovsd_Wsd_Hsd_Vsd,
+    /* 0x12 */  iemOp_vmovlps_Vq_Hq_Mq__vmovhlps, iemOp_vmovlpd_Vq_Hq_Mq, iemOp_vmovsldup_Vx_Wx,    iemOp_vmovddup_Vx_Wx,
+    /* 0x13 */  iemOp_vmovlps_Mq_Vq,        iemOp_vmovlpd_Mq_Vq,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x14 */  iemOp_vunpcklps_Vx_Hx_Wx,   iemOp_vunpcklpd_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x15 */  iemOp_vunpckhps_Vx_Hx_Wx,   iemOp_vunpckhpd_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x16 */  iemOp_vmovhps_Vdq_Hq_Mq__vmovlhps_Vdq_Hq_Uq, iemOp_vmovhpd_Vdq_Hq_Mq, iemOp_vmovshdup_Vx_Wx, iemOp_InvalidNeedRM,
+    /* 0x17 */  iemOp_vmovhps_Mq_Vq,        iemOp_vmovhpd_Mq_Vq,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x18 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x19 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x20 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x21 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x22 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x23 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x24 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x25 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x26 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x27 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x28 */  iemOp_vmovaps_Vps_Wps,      iemOp_vmovapd_Vpd_Wpd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x29 */  iemOp_vmovaps_Wps_Vps,      iemOp_vmovapd_Wpd_Vpd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2a */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_vcvtsi2ss_Vss_Hss_Ey, iemOp_vcvtsi2sd_Vsd_Hsd_Ey,
+    /* 0x2b */  iemOp_vmovntps_Mps_Vps,     iemOp_vmovntpd_Mpd_Vpd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2c */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_vcvttss2si_Gy_Wss,    iemOp_vcvttsd2si_Gy_Wsd,
+    /* 0x2d */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_vcvtss2si_Gy_Wss,     iemOp_vcvtsd2si_Gy_Wsd,
+    /* 0x2e */  iemOp_vucomiss_Vss_Wss,     iemOp_vucomisd_Vsd_Wsd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2f */  iemOp_vcomiss_Vss_Wss,      iemOp_vcomisd_Vsd_Wsd,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x30 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x31 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x32 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x33 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x34 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x35 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x36 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x37 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x38 */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x39 */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3a */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3b */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3c */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3d */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3e */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+    /* 0x3f */  IEMOP_X4(iemOp_InvalidNeedRM),  /** @todo check that there is no escape table stuff here */
+
+    /* 0x40 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x41 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x42 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x43 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x44 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x45 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x46 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x47 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x48 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x49 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x50 */  iemOp_vmovmskps_Gy_Ups,     iemOp_vmovmskpd_Gy_Upd,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x51 */  iemOp_vsqrtps_Vps_Wps,      iemOp_vsqrtpd_Vpd_Wpd,      iemOp_vsqrtss_Vss_Hss_Wss,  iemOp_vsqrtsd_Vsd_Hsd_Wsd,
+    /* 0x52 */  iemOp_vrsqrtps_Vps_Wps,     iemOp_InvalidNeedRM,        iemOp_vrsqrtss_Vss_Hss_Wss, iemOp_InvalidNeedRM,
+    /* 0x53 */  iemOp_vrcpps_Vps_Wps,       iemOp_InvalidNeedRM,        iemOp_vrcpss_Vss_Hss_Wss,   iemOp_InvalidNeedRM,
+    /* 0x54 */  iemOp_vandps_Vps_Hps_Wps,   iemOp_vandpd_Vpd_Hpd_Wpd,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x55 */  iemOp_vandnps_Vps_Hps_Wps,  iemOp_vandnpd_Vpd_Hpd_Wpd,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x56 */  iemOp_vorps_Vps_Hps_Wps,    iemOp_vorpd_Vpd_Hpd_Wpd,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x57 */  iemOp_vxorps_Vps_Hps_Wps,   iemOp_vxorpd_Vpd_Hpd_Wpd,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x58 */  iemOp_vaddps_Vps_Hps_Wps,   iemOp_vaddpd_Vpd_Hpd_Wpd,   iemOp_vaddss_Vss_Hss_Wss,   iemOp_vaddsd_Vsd_Hsd_Wsd,
+    /* 0x59 */  iemOp_vmulps_Vps_Hps_Wps,   iemOp_vmulpd_Vpd_Hpd_Wpd,   iemOp_vmulss_Vss_Hss_Wss,   iemOp_vmulsd_Vsd_Hsd_Wsd,
+    /* 0x5a */  iemOp_vcvtps2pd_Vpd_Wps,    iemOp_vcvtpd2ps_Vps_Wpd,    iemOp_vcvtss2sd_Vsd_Hx_Wss, iemOp_vcvtsd2ss_Vss_Hx_Wsd,
+    /* 0x5b */  iemOp_vcvtdq2ps_Vps_Wdq,    iemOp_vcvtps2dq_Vdq_Wps,    iemOp_vcvttps2dq_Vdq_Wps,   iemOp_InvalidNeedRM,
+    /* 0x5c */  iemOp_vsubps_Vps_Hps_Wps,   iemOp_vsubpd_Vpd_Hpd_Wpd,   iemOp_vsubss_Vss_Hss_Wss,   iemOp_vsubsd_Vsd_Hsd_Wsd,
+    /* 0x5d */  iemOp_vminps_Vps_Hps_Wps,   iemOp_vminpd_Vpd_Hpd_Wpd,   iemOp_vminss_Vss_Hss_Wss,   iemOp_vminsd_Vsd_Hsd_Wsd,
+    /* 0x5e */  iemOp_vdivps_Vps_Hps_Wps,   iemOp_vdivpd_Vpd_Hpd_Wpd,   iemOp_vdivss_Vss_Hss_Wss,   iemOp_vdivsd_Vsd_Hsd_Wsd,
+    /* 0x5f */  iemOp_vmaxps_Vps_Hps_Wps,   iemOp_vmaxpd_Vpd_Hpd_Wpd,   iemOp_vmaxss_Vss_Hss_Wss,   iemOp_vmaxsd_Vsd_Hsd_Wsd,
+
+    /* 0x60 */  iemOp_InvalidNeedRM,        iemOp_vpunpcklbw_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x61 */  iemOp_InvalidNeedRM,        iemOp_vpunpcklwd_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x62 */  iemOp_InvalidNeedRM,        iemOp_vpunpckldq_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x63 */  iemOp_InvalidNeedRM,        iemOp_vpacksswb_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x64 */  iemOp_InvalidNeedRM,        iemOp_vpcmpgtb_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x65 */  iemOp_InvalidNeedRM,        iemOp_vpcmpgtw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x66 */  iemOp_InvalidNeedRM,        iemOp_vpcmpgtd_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x67 */  iemOp_InvalidNeedRM,        iemOp_vpackuswb_Vx_Hx_W,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x68 */  iemOp_InvalidNeedRM,        iemOp_vpunpckhbw_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x69 */  iemOp_InvalidNeedRM,        iemOp_vpunpckhwd_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6a */  iemOp_InvalidNeedRM,        iemOp_vpunpckhdq_Vx_Hx_W,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6b */  iemOp_InvalidNeedRM,        iemOp_vpackssdw_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6c */  iemOp_InvalidNeedRM,        iemOp_vpunpcklqdq_Vx_Hx_Wx, iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6d */  iemOp_InvalidNeedRM,        iemOp_vpunpckhqdq_Vx_Hx_W,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6e */  iemOp_InvalidNeedRM,        iemOp_vmovd_q_Vy_Ey,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x6f */  iemOp_InvalidNeedRM,        iemOp_vmovdqa_Vx_Wx,        iemOp_vmovdqu_Vx_Wx,        iemOp_InvalidNeedRM,
+
+    /* 0x70 */  iemOp_InvalidNeedRM,        iemOp_vpshufd_Vx_Wx_Ib,     iemOp_vpshufhw_Vx_Wx_Ib,    iemOp_vpshuflw_Vx_Wx_Ib,
+    /* 0x71 */  iemOp_InvalidNeedRM,        iemOp_VGrp12,               iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x72 */  iemOp_InvalidNeedRM,        iemOp_VGrp13,               iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x73 */  iemOp_InvalidNeedRM,        iemOp_VGrp14,               iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x74 */  iemOp_InvalidNeedRM,        iemOp_vpcmpeqb_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x75 */  iemOp_InvalidNeedRM,        iemOp_vpcmpeqw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x76 */  iemOp_InvalidNeedRM,        iemOp_vpcmpeqd_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x77 */  iemOp_vzeroupperv__vzeroallv, iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,       iemOp_InvalidNeedRM,
+    /* 0x78 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x79 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7c */  iemOp_InvalidNeedRM,        iemOp_vhaddpd_Vpd_Hpd_Wpd,  iemOp_InvalidNeedRM,        iemOp_vhaddps_Vps_Hps_Wps,
+    /* 0x7d */  iemOp_InvalidNeedRM,        iemOp_vhsubpd_Vpd_Hpd_Wpd,  iemOp_InvalidNeedRM,        iemOp_vhsubps_Vps_Hps_Wps,
+    /* 0x7e */  iemOp_InvalidNeedRM,        iemOp_vmovd_q_Ey_Vy,        iemOp_vmovq_Vq_Wq,          iemOp_InvalidNeedRM,
+    /* 0x7f */  iemOp_InvalidNeedRM,        iemOp_vmovdqa_Wx_Vx,        iemOp_vmovdqu_Wx_Vx,        iemOp_InvalidNeedRM,
+
+    /* 0x80 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x81 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x82 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x83 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x84 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x85 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x86 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x87 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x88 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x89 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x90 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x91 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x92 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x93 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x94 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x95 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x96 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x97 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x98 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x99 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x9f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xa0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xaa */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xab */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xac */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xad */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xae */  IEMOP_X4(iemOp_VGrp15),
+    /* 0xaf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xb0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xba */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbc */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbd */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbe */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xbf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xc0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc2 */  iemOp_vcmpps_Vps_Hps_Wps_Ib, iemOp_vcmppd_Vpd_Hpd_Wpd_Ib, iemOp_vcmpss_Vss_Hss_Wss_Ib, iemOp_vcmpsd_Vsd_Hsd_Wsd_Ib,
+    /* 0xc3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc4 */  iemOp_InvalidNeedRM,        iemOp_vpinsrw_Vdq_Hdq_RyMw_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0xc5 */  iemOp_InvalidNeedRM,        iemOp_vpextrw_Gd_Udq_Ib,       iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xc6 */  iemOp_vshufps_Vps_Hps_Wps_Ib, iemOp_vshufpd_Vpd_Hpd_Wpd_Ib, iemOp_InvalidNeedRMImm8,iemOp_InvalidNeedRMImm8,
+    /* 0xc7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xca */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcc */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcd */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xce */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xd0 */  iemOp_InvalidNeedRM,        iemOp_vaddsubpd_Vpd_Hpd_Wpd, iemOp_InvalidNeedRM,       iemOp_vaddsubps_Vps_Hps_Wps,
+    /* 0xd1 */  iemOp_InvalidNeedRM,        iemOp_vpsrlw_Vx_Hx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd2 */  iemOp_InvalidNeedRM,        iemOp_vpsrld_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd3 */  iemOp_InvalidNeedRM,        iemOp_vpsrlq_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd4 */  iemOp_InvalidNeedRM,        iemOp_vpaddq_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd5 */  iemOp_InvalidNeedRM,        iemOp_vpmullw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd6 */  iemOp_InvalidNeedRM,        iemOp_vmovq_Wq_Vq,          iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd7 */  iemOp_InvalidNeedRM,        iemOp_vpmovmskb_Gd_Ux,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd8 */  iemOp_InvalidNeedRM,        iemOp_vpsubusb_Vx_Hx_W,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xd9 */  iemOp_InvalidNeedRM,        iemOp_vpsubusw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xda */  iemOp_InvalidNeedRM,        iemOp_vpminub_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdb */  iemOp_InvalidNeedRM,        iemOp_vpand_Vx_Hx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdc */  iemOp_InvalidNeedRM,        iemOp_vpaddusb_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdd */  iemOp_InvalidNeedRM,        iemOp_vpaddusw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xde */  iemOp_InvalidNeedRM,        iemOp_vpmaxub_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdf */  iemOp_InvalidNeedRM,        iemOp_vpandn_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xe0 */  iemOp_InvalidNeedRM,        iemOp_vpavgb_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe1 */  iemOp_InvalidNeedRM,        iemOp_vpsraw_Vx_Hx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe2 */  iemOp_InvalidNeedRM,        iemOp_vpsrad_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe3 */  iemOp_InvalidNeedRM,        iemOp_vpavgw_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe4 */  iemOp_InvalidNeedRM,        iemOp_vpmulhuw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe5 */  iemOp_InvalidNeedRM,        iemOp_vpmulhw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe6 */  iemOp_InvalidNeedRM,        iemOp_vcvttpd2dq_Vx_Wpd,    iemOp_vcvtdq2pd_Vx_Wpd,     iemOp_vcvtpd2dq_Vx_Wpd,
+    /* 0xe7 */  iemOp_InvalidNeedRM,        iemOp_vmovntdq_Mx_Vx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe8 */  iemOp_InvalidNeedRM,        iemOp_vpsubsb_Vx_Hx_W,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xe9 */  iemOp_InvalidNeedRM,        iemOp_vpsubsw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xea */  iemOp_InvalidNeedRM,        iemOp_vpminsw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xeb */  iemOp_InvalidNeedRM,        iemOp_vpor_Vx_Hx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xec */  iemOp_InvalidNeedRM,        iemOp_vpaddsb_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xed */  iemOp_InvalidNeedRM,        iemOp_vpaddsw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xee */  iemOp_InvalidNeedRM,        iemOp_vpmaxsw_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xef */  iemOp_InvalidNeedRM,        iemOp_vpxor_Vx_Hx_Wx,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xf0 */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_vlddqu_Vx_Mx,
+    /* 0xf1 */  iemOp_InvalidNeedRM,        iemOp_vpsllw_Vx_Hx_W,       iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf2 */  iemOp_InvalidNeedRM,        iemOp_vpslld_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf3 */  iemOp_InvalidNeedRM,        iemOp_vpsllq_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf4 */  iemOp_InvalidNeedRM,        iemOp_vpmuludq_Vx_Hx_W,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf5 */  iemOp_InvalidNeedRM,        iemOp_vpmaddwd_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf6 */  iemOp_InvalidNeedRM,        iemOp_vpsadbw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf7 */  iemOp_InvalidNeedRM,        iemOp_vmaskmovdqu_Vdq_Udq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf8 */  iemOp_InvalidNeedRM,        iemOp_vpsubb_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf9 */  iemOp_InvalidNeedRM,        iemOp_vpsubw_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfa */  iemOp_InvalidNeedRM,        iemOp_vpsubd_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfb */  iemOp_InvalidNeedRM,        iemOp_vpsubq_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfc */  iemOp_InvalidNeedRM,        iemOp_vpaddb_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfd */  iemOp_InvalidNeedRM,        iemOp_vpaddw_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xfe */  iemOp_InvalidNeedRM,        iemOp_vpaddd_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xff */  IEMOP_X4(iemOp_vud0) /* ?? */
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexMap1) == 1024);
+/** @}  */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h
new file mode 100644
index 00000000..02fa3df9
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap2.cpp.h
@@ -0,0 +1,2107 @@
+/* $Id: IEMAllInstructionsVexMap2.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation.
+ *
+ * @remarks IEMAllInstructionsThree0f38.cpp.h is a VEX mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/** @name VEX Opcode Map 2
+ * @{
+ */
+
+/*  Opcode VEX.0F38 0x00 - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x00. */
+FNIEMOP_DEF(iemOp_vpshufb_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPSHUFB, vpshufb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpshufb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x01 - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x01. */
+FNIEMOP_DEF(iemOp_vphaddw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPHADDW, vphaddw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vphaddw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x02 - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x02. */
+FNIEMOP_DEF(iemOp_vphaddd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPHADDD, vphaddd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vphaddd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x03 - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x03. */
+FNIEMOP_DEF(iemOp_vphaddsw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPHADDSW, vphaddsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vphaddsw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x04 - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x04. */
+FNIEMOP_DEF(iemOp_vpmaddubsw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMADDUBSW, vpmaddubsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpmaddubsw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/* Opcode VEX.0F38 0x05 - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x05. */
+FNIEMOP_DEF(iemOp_vphsubw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPHSUBW, vphsubw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vphsubw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x06 - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x06. */
+FNIEMOP_DEF(iemOp_vphsubd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPHSUBD, vphsubd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vphsubd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x07 - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x07. */
+FNIEMOP_DEF(iemOp_vphsubsw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPHSUBSW, vphsubsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vphsubsw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x08 - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x08. */
+FNIEMOP_DEF(iemOp_vpsignb_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPSIGNB, vpsignb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpsignb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x09 - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x09. */
+FNIEMOP_DEF(iemOp_vpsignw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPSIGNW, vpsignw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpsignw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x0a - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x0a. */
+FNIEMOP_DEF(iemOp_vpsignd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPSIGND, vpsignd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpsignd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x0b - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x0b. */
+FNIEMOP_DEF(iemOp_vpmulhrsw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMULHRSW, vpmulhrsw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpmulhrsw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x0c - invalid. */
+/**  Opcode VEX.66.0F38 0x0c. */
+FNIEMOP_STUB(iemOp_vpermilps_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x0d - invalid. */
+/**  Opcode VEX.66.0F38 0x0d. */
+FNIEMOP_STUB(iemOp_vpermilpd_Vx_Hx_Wx);
+/*  Opcode VEX.0F38 0x0e - invalid. */
+/**  Opcode VEX.66.0F38 0x0e. */
+FNIEMOP_STUB(iemOp_vtestps_Vx_Wx);
+/*  Opcode VEX.0F38 0x0f - invalid. */
+/**  Opcode VEX.66.0F38 0x0f. */
+FNIEMOP_STUB(iemOp_vtestpd_Vx_Wx);
+
+
+/*  Opcode VEX.0F38 0x10 - invalid */
+/*  Opcode VEX.66.0F38 0x10 - invalid (legacy only). */
+/*  Opcode VEX.0F38 0x11 - invalid */
+/*  Opcode VEX.66.0F38 0x11 - invalid */
+/*  Opcode VEX.0F38 0x12 - invalid */
+/*  Opcode VEX.66.0F38 0x12 - invalid */
+/*  Opcode VEX.0F38 0x13 - invalid */
+/*  Opcode VEX.66.0F38 0x13 - invalid (vex only). */
+/*  Opcode VEX.0F38 0x14 - invalid */
+/*  Opcode VEX.66.0F38 0x14 - invalid (legacy only). */
+/*  Opcode VEX.0F38 0x15 - invalid */
+/*  Opcode VEX.66.0F38 0x15 - invalid (legacy only). */
+/*  Opcode VEX.0F38 0x16 - invalid */
+/** Opcode VEX.66.0F38 0x16. */
+FNIEMOP_STUB(iemOp_vpermps_Vqq_Hqq_Wqq);
+/*  Opcode VEX.0F38 0x17 - invalid */
+
+
+/** Opcode VEX.66.0F38 0x17 - invalid */
+FNIEMOP_DEF(iemOp_vptest_Vx_Wx)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1, 0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2, 1);
+            IEM_MC_ARG(uint32_t *,              pEFlags,       2);
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_FETCH_YREG_U256(uSrc1,   IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_FETCH_YREG_U256(uSrc2,   IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vptest_u256, iemAImpl_vptest_u256_fallback),
+                                     puSrc1, puSrc2, pEFlags);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(PCRTUINT128U,            puSrc1,  0);
+            IEM_MC_ARG(PCRTUINT128U,            puSrc2,  1);
+            IEM_MC_ARG(uint32_t *,              pEFlags, 2);
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_ptest_u128, puSrc1, puSrc2, pEFlags);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEM_MC_BEGIN(3, 3);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1, 0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2, 1);
+            IEM_MC_ARG(uint32_t *,              pEFlags,       2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_YREG_U256(uSrc1,      IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vptest_u256, iemAImpl_vptest_u256_fallback),
+                                     puSrc1, puSrc2, pEFlags);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 2);
+            IEM_MC_LOCAL(RTUINT128U,            uSrc2);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_ARG(PCRTUINT128U,            puSrc1,        0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,  puSrc2, uSrc2, 1);
+            IEM_MC_ARG(uint32_t *,              pEFlags,       2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2,  pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_XREG_U128_CONST(puSrc1,  IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_ptest_u128, puSrc1, puSrc2, pEFlags);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/*  Opcode VEX.0F38 0x18 - invalid */
+/** Opcode VEX.66.0F38 0x18. */
+FNIEMOP_STUB(iemOp_vbroadcastss_Vx_Wd);
+/*  Opcode VEX.0F38 0x19 - invalid */
+/** Opcode VEX.66.0F38 0x19. */
+FNIEMOP_STUB(iemOp_vbroadcastsd_Vqq_Wq);
+/*  Opcode VEX.0F38 0x1a - invalid */
+/** Opcode VEX.66.0F38 0x1a. */
+FNIEMOP_STUB(iemOp_vbroadcastf128_Vqq_Mdq);
+/*  Opcode VEX.0F38 0x1b - invalid */
+/*  Opcode VEX.66.0F38 0x1b - invalid */
+/*  Opcode VEX.0F38 0x1c - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x1c. */
+FNIEMOP_DEF(iemOp_vpabsb_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VPABSB, vpabsb, Vx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF2_INIT_VARS(vpabsb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x1d - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x1d. */
+FNIEMOP_DEF(iemOp_vpabsw_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VPABSW, vpabsw, Vx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF2_INIT_VARS(vpabsw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+/*  Opcode VEX.0F38 0x1e - invalid. */
+
+
+/** Opcode VEX.66.0F38 0x1e. */
+FNIEMOP_DEF(iemOp_vpabsd_Vx_Wx)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VPABSD, vpabsd, Vx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF2_INIT_VARS(vpabsd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.0F38 0x1f - invalid */
+/*  Opcode VEX.66.0F38 0x1f - invalid */
+
+
+/** Body for the vpmov{s,z}x* instructions. */
+#define IEMOP_BODY_VPMOV_S_Z(a_Instr, a_SrcWidth) \
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \
+    if (IEM_IS_MODRM_REG_MODE(bRm)) \
+    { \
+        /* \
+         * Register, register. \
+         */ \
+        if (pVCpu->iem.s.uVexLength) \
+        { \
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx2); \
+            IEM_MC_BEGIN(2, 1); \
+            IEM_MC_LOCAL(RTUINT256U,            uDst); \
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst, uDst, 0); \
+            IEM_MC_ARG(PCRTUINT128U,            puSrc,       1); \
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \
+            IEM_MC_PREPARE_AVX_USAGE(); \
+            IEM_MC_REF_XREG_U128_CONST(puSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u256, \
+                                                                 iemAImpl_ ## a_Instr ## _u256_fallback), \
+                                     puDst, puSrc); \
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx); \
+            IEM_MC_BEGIN(2, 0); \
+            IEM_MC_ARG(PRTUINT128U,          puDst,  0); \
+            IEM_MC_ARG(uint64_t,             uSrc,   1); \
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \
+            IEM_MC_PREPARE_AVX_USAGE(); \
+            IEM_MC_REF_XREG_U128(puDst,  IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_FETCH_XREG_U64(uSrc,  IEM_GET_MODRM_RM(pVCpu, bRm), 0 /* a_iQword*/); \
+            IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u128, \
+                                                                 iemAImpl_## a_Instr ## _u128_fallback), \
+                                     puDst, uSrc); \
+            IEM_MC_CLEAR_YREG_128_UP(          IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    else \
+    { \
+        /* \
+         * Register, memory. \
+         */ \
+        if (pVCpu->iem.s.uVexLength) \
+        { \
+            IEM_MC_BEGIN(2, 3); \
+            IEM_MC_LOCAL(RTUINT256U,            uDst); \
+            IEM_MC_LOCAL(RTUINT128U,            uSrc); \
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc); \
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst, uDst, 0); \
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,  puSrc, uSrc, 1); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx2); \
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \
+            IEM_MC_PREPARE_AVX_USAGE(); \
+            IEM_MC_FETCH_MEM_U128(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u256, \
+                                                                 iemAImpl_ ## a_Instr ## _u256_fallback), \
+                                     puDst, puSrc); \
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(   IEM_GET_MODRM_REG(pVCpu, bRm), uDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEM_MC_BEGIN(2, 1); \
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc); \
+            IEM_MC_ARG(PRTUINT128U,                 puDst,       0); \
+            IEM_MC_ARG(uint ## a_SrcWidth ##_t,     uSrc,        1); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV_EX(fAvx); \
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT(); \
+            IEM_MC_PREPARE_AVX_USAGE(); \
+            IEM_MC_REF_XREG_U128(puDst,  IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_FETCH_MEM_U ## a_SrcWidth (uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx2, iemAImpl_ ## a_Instr ## _u128, \
+                                                                 iemAImpl_ ## a_Instr ## _u128_fallback), \
+                                     puDst, uSrc); \
+            IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    (void)0
+
+/** Opcode VEX.66.0F38 0x20. */
+FNIEMOP_DEF(iemOp_vpmovsxbw_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVSXBW, vpmovsxbw, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovsxbw, 64);
+}
+
+
+/** Opcode VEX.66.0F38 0x21. */
+FNIEMOP_DEF(iemOp_vpmovsxbd_Vx_UxMd)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVSXBD, vpmovsxbd, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovsxbd, 32);
+}
+
+
+/** Opcode VEX.66.0F38 0x22. */
+FNIEMOP_DEF(iemOp_vpmovsxbq_Vx_UxMw)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVSXBQ, vpmovsxbq, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovsxbq, 16);
+}
+
+
+/** Opcode VEX.66.0F38 0x23. */
+FNIEMOP_DEF(iemOp_vpmovsxwd_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVSXWD, vpmovsxwd, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovsxwd, 64);
+}
+
+
+/** Opcode VEX.66.0F38 0x24. */
+FNIEMOP_DEF(iemOp_vpmovsxwq_Vx_UxMd)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVSXWQ, vpmovsxwq, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovsxwq, 32);
+}
+
+
+/** Opcode VEX.66.0F38 0x25. */
+FNIEMOP_DEF(iemOp_vpmovsxdq_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVSXDQ, vpmovsxdq, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovsxdq, 64);
+}
+
+
+/*  Opcode VEX.66.0F38 0x26 - invalid */
+/*  Opcode VEX.66.0F38 0x27 - invalid */
+
+
+/** Opcode VEX.66.0F38 0x28. */
+FNIEMOP_DEF(iemOp_vpmuldq_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMULDQ, vpmuldq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpmuldq);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x29. */
+FNIEMOP_DEF(iemOp_vpcmpeqq_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPCMPEQQ, vpcmpeqq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpcmpeqq);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+FNIEMOP_DEF(iemOp_vmovntdqa_Vx_Mx)
+{
+    Assert(pVCpu->iem.s.uVexLength <= 1);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_MEM_MODE(bRm))
+    {
+        if (pVCpu->iem.s.uVexLength == 0)
+        {
+            /**
+             * @opcode      0x2a
+             * @opcodesub   !11 mr/reg vex.l=0
+             * @oppfx       0x66
+             * @opcpuid     avx
+             * @opgroup     og_avx_cachect
+             * @opxcpttype  1
+             * @optest      op1=-1 op2=2  -> op1=2
+             * @optest      op1=0 op2=-42 -> op1=-42
+             */
+            /* 128-bit: Memory, register. */
+            IEMOP_MNEMONIC2EX(vmovntdqa_Vdq_WO_Mdq_L0, "vmovntdqa, Vdq_WO, Mdq", VEX_RM_MEM, VMOVNTDQA, vmovntdqa, Vx_WO, Mx,
+                              DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U128_ALIGN_SSE(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U128_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            /**
+             * @opdone
+             * @opcode      0x2a
+             * @opcodesub   !11 mr/reg vex.l=1
+             * @oppfx       0x66
+             * @opcpuid     avx2
+             * @opgroup     og_avx2_cachect
+             * @opxcpttype  1
+             * @optest      op1=-1 op2=2  -> op1=2
+             * @optest      op1=0 op2=-42 -> op1=-42
+             */
+            /* 256-bit: Memory, register. */
+            IEMOP_MNEMONIC2EX(vmovntdqa_Vqq_WO_Mqq_L1, "vmovntdqa, Vqq_WO,Mqq", VEX_RM_MEM, VMOVNTDQA, vmovntdqa, Vx_WO, Mx,
+                              DISOPTYPE_HARMLESS | DISOPTYPE_AVX, IEMOPHINT_IGNORES_OP_SIZES);
+            IEM_MC_BEGIN(0, 2);
+            IEM_MC_LOCAL(RTUINT256U,                uSrc);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_NO_VVVV();
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_ACTUALIZE_AVX_STATE_FOR_CHANGE();
+
+            IEM_MC_FETCH_MEM_U256_ALIGN_AVX(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uSrc);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+
+    /**
+     * @opdone
+     * @opmnemonic  udvex660f382arg
+     * @opcode      0x2a
+     * @opcodesub   11 mr/reg
+     * @oppfx       0x66
+     * @opunused    immediate
+     * @opcpuid     avx
+     * @optest      ->
+     */
+    else
+        return IEMOP_RAISE_INVALID_OPCODE();
+}
+
+
+/** Opcode VEX.66.0F38 0x2b. */
+FNIEMOP_DEF(iemOp_vpackusdw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPACKUSDW, vpackusdw, Vx, Hx, Wx, DISOPTYPE_HARMLESS | DISOPTYPE_AVX, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(          vpackusdw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x2c. */
+FNIEMOP_STUB(iemOp_vmaskmovps_Vx_Hx_Mx);
+/** Opcode VEX.66.0F38 0x2d. */
+FNIEMOP_STUB(iemOp_vmaskmovpd_Vx_Hx_Mx);
+/** Opcode VEX.66.0F38 0x2e. */
+FNIEMOP_STUB(iemOp_vmaskmovps_Mx_Hx_Vx);
+/** Opcode VEX.66.0F38 0x2f. */
+FNIEMOP_STUB(iemOp_vmaskmovpd_Mx_Hx_Vx);
+
+
+/** Opcode VEX.66.0F38 0x30. */
+FNIEMOP_DEF(iemOp_vpmovzxbw_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVZXBW, vpmovzxbw, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovzxbw, 64);
+}
+
+
+/** Opcode VEX.66.0F38 0x31. */
+FNIEMOP_DEF(iemOp_vpmovzxbd_Vx_UxMd)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVZXBD, vpmovzxbd, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovzxbd, 32);
+}
+
+
+/** Opcode VEX.66.0F38 0x32. */
+FNIEMOP_DEF(iemOp_vpmovzxbq_Vx_UxMw)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVZXBQ, vpmovzxbq, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovzxbq, 16);
+}
+
+
+/** Opcode VEX.66.0F38 0x33. */
+FNIEMOP_DEF(iemOp_vpmovzxwd_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVZXWD, vpmovzxwd, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovzxwd, 64);
+}
+
+
+/** Opcode VEX.66.0F38 0x34. */
+FNIEMOP_DEF(iemOp_vpmovzxwq_Vx_UxMd)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVZXWQ, vpmovzxwq, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovzxwq, 32);
+}
+
+
+/** Opcode VEX.66.0F38 0x35. */
+FNIEMOP_DEF(iemOp_vpmovzxdq_Vx_UxMq)
+{
+     /** @todo r=aeichner Review code, the naming of this function and the parameter type specifiers. */
+    IEMOP_MNEMONIC2(VEX_RM, VPMOVZXDQ, vpmovzxdq, Vx, Wq, DISOPTYPE_HARMLESS, 0);
+    IEMOP_BODY_VPMOV_S_Z(vpmovzxdq, 64);
+}
+
+
+/*  Opcode VEX.66.0F38 0x36. */
+FNIEMOP_STUB(iemOp_vpermd_Vqq_Hqq_Wqq);
+
+
+/** Opcode VEX.66.0F38 0x37. */
+FNIEMOP_DEF(iemOp_vpcmpgtq_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPCMPGTQ, vpcmpgtq, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpcmpgtq);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x38. */
+FNIEMOP_DEF(iemOp_vpminsb_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMINSB, vpminsb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpminsb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x39. */
+FNIEMOP_DEF(iemOp_vpminsd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMINSD, vpminsd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpminsd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x3a. */
+FNIEMOP_DEF(iemOp_vpminuw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMINUW, vpminuw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpminuw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x3b. */
+FNIEMOP_DEF(iemOp_vpminud_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMINUD, vpminud, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpminud);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x3c. */
+FNIEMOP_DEF(iemOp_vpmaxsb_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMAXSB, vpmaxsb, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpmaxsb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x3d. */
+FNIEMOP_DEF(iemOp_vpmaxsd_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMAXSD, vpmaxsd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpmaxsd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x3e. */
+FNIEMOP_DEF(iemOp_vpmaxuw_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMAXUW, vpmaxuw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpmaxuw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x3f. */
+FNIEMOP_DEF(iemOp_vpmaxud_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMAXUD, vpmaxud, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAF3_INIT_VARS(vpmaxud);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x40. */
+FNIEMOP_DEF(iemOp_vpmulld_Vx_Hx_Wx)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPMULLD, vpmulld, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0);
+    IEMOPMEDIAOPTF3_INIT_VARS(vpmulld);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F38 0x41. */
+FNIEMOP_DEF(iemOp_vphminposuw_Vdq_Wdq)
+{
+    IEMOP_MNEMONIC2(VEX_RM, VPHMINPOSUW, vphminposuw, Vdq, Wdq, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fAvx);
+        IEM_MC_BEGIN(2, 0);
+        IEM_MC_ARG(PRTUINT128U,          puDst,  0);
+        IEM_MC_ARG(PCRTUINT128U,         puSrc,  1);
+        IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,        IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc,  IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vphminposuw_u128, iemAImpl_vphminposuw_u128_fallback),
+                                 puDst, puSrc);
+        IEM_MC_CLEAR_YREG_128_UP(          IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(2, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,       0);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc, uSrc, 1);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fAvx);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+
+        IEM_MC_FETCH_MEM_U128_NO_AC(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_2(IEM_SELECT_HOST_OR_FALLBACK(fAvx, iemAImpl_vphminposuw_u128, iemAImpl_vphminposuw_u128_fallback),
+                                 puDst, puSrc);
+        IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm));
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode VEX.66.0F38 0x42 - invalid. */
+/*  Opcode VEX.66.0F38 0x43 - invalid. */
+/*  Opcode VEX.66.0F38 0x44 - invalid. */
+/** Opcode VEX.66.0F38 0x45. */
+FNIEMOP_STUB(iemOp_vpsrlvd_q_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x46. */
+FNIEMOP_STUB(iemOp_vsravd_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x47. */
+FNIEMOP_STUB(iemOp_vpsllvd_q_Vx_Hx_Wx);
+/*  Opcode VEX.66.0F38 0x48 - invalid. */
+/*  Opcode VEX.66.0F38 0x49 - invalid. */
+/*  Opcode VEX.66.0F38 0x4a - invalid. */
+/*  Opcode VEX.66.0F38 0x4b - invalid. */
+/*  Opcode VEX.66.0F38 0x4c - invalid. */
+/*  Opcode VEX.66.0F38 0x4d - invalid. */
+/*  Opcode VEX.66.0F38 0x4e - invalid. */
+/*  Opcode VEX.66.0F38 0x4f - invalid. */
+
+/*  Opcode VEX.66.0F38 0x50 - invalid. */
+/*  Opcode VEX.66.0F38 0x51 - invalid. */
+/*  Opcode VEX.66.0F38 0x52 - invalid. */
+/*  Opcode VEX.66.0F38 0x53 - invalid. */
+/*  Opcode VEX.66.0F38 0x54 - invalid. */
+/*  Opcode VEX.66.0F38 0x55 - invalid. */
+/*  Opcode VEX.66.0F38 0x56 - invalid. */
+/*  Opcode VEX.66.0F38 0x57 - invalid. */
+/** Opcode VEX.66.0F38 0x58. */
+FNIEMOP_STUB(iemOp_vpbroadcastd_Vx_Wx);
+/** Opcode VEX.66.0F38 0x59. */
+FNIEMOP_STUB(iemOp_vpbroadcastq_Vx_Wx);
+/** Opcode VEX.66.0F38 0x5a. */
+FNIEMOP_STUB(iemOp_vbroadcasti128_Vqq_Mdq);
+/*  Opcode VEX.66.0F38 0x5b - invalid. */
+/*  Opcode VEX.66.0F38 0x5c - invalid. */
+/*  Opcode VEX.66.0F38 0x5d - invalid. */
+/*  Opcode VEX.66.0F38 0x5e - invalid. */
+/*  Opcode VEX.66.0F38 0x5f - invalid. */
+
+/*  Opcode VEX.66.0F38 0x60 - invalid. */
+/*  Opcode VEX.66.0F38 0x61 - invalid. */
+/*  Opcode VEX.66.0F38 0x62 - invalid. */
+/*  Opcode VEX.66.0F38 0x63 - invalid. */
+/*  Opcode VEX.66.0F38 0x64 - invalid. */
+/*  Opcode VEX.66.0F38 0x65 - invalid. */
+/*  Opcode VEX.66.0F38 0x66 - invalid. */
+/*  Opcode VEX.66.0F38 0x67 - invalid. */
+/*  Opcode VEX.66.0F38 0x68 - invalid. */
+/*  Opcode VEX.66.0F38 0x69 - invalid. */
+/*  Opcode VEX.66.0F38 0x6a - invalid. */
+/*  Opcode VEX.66.0F38 0x6b - invalid. */
+/*  Opcode VEX.66.0F38 0x6c - invalid. */
+/*  Opcode VEX.66.0F38 0x6d - invalid. */
+/*  Opcode VEX.66.0F38 0x6e - invalid. */
+/*  Opcode VEX.66.0F38 0x6f - invalid. */
+
+/*  Opcode VEX.66.0F38 0x70 - invalid. */
+/*  Opcode VEX.66.0F38 0x71 - invalid. */
+/*  Opcode VEX.66.0F38 0x72 - invalid. */
+/*  Opcode VEX.66.0F38 0x73 - invalid. */
+/*  Opcode VEX.66.0F38 0x74 - invalid. */
+/*  Opcode VEX.66.0F38 0x75 - invalid. */
+/*  Opcode VEX.66.0F38 0x76 - invalid. */
+/*  Opcode VEX.66.0F38 0x77 - invalid. */
+/** Opcode VEX.66.0F38 0x78. */
+FNIEMOP_STUB(iemOp_vpboardcastb_Vx_Wx);
+/** Opcode VEX.66.0F38 0x79. */
+FNIEMOP_STUB(iemOp_vpboardcastw_Vx_Wx);
+/*  Opcode VEX.66.0F38 0x7a - invalid. */
+/*  Opcode VEX.66.0F38 0x7b - invalid. */
+/*  Opcode VEX.66.0F38 0x7c - invalid. */
+/*  Opcode VEX.66.0F38 0x7d - invalid. */
+/*  Opcode VEX.66.0F38 0x7e - invalid. */
+/*  Opcode VEX.66.0F38 0x7f - invalid. */
+
+/*  Opcode VEX.66.0F38 0x80 - invalid (legacy only). */
+/*  Opcode VEX.66.0F38 0x81 - invalid (legacy only). */
+/*  Opcode VEX.66.0F38 0x82 - invalid (legacy only). */
+/*  Opcode VEX.66.0F38 0x83 - invalid. */
+/*  Opcode VEX.66.0F38 0x84 - invalid. */
+/*  Opcode VEX.66.0F38 0x85 - invalid. */
+/*  Opcode VEX.66.0F38 0x86 - invalid. */
+/*  Opcode VEX.66.0F38 0x87 - invalid. */
+/*  Opcode VEX.66.0F38 0x88 - invalid. */
+/*  Opcode VEX.66.0F38 0x89 - invalid. */
+/*  Opcode VEX.66.0F38 0x8a - invalid. */
+/*  Opcode VEX.66.0F38 0x8b - invalid. */
+/** Opcode VEX.66.0F38 0x8c. */
+FNIEMOP_STUB(iemOp_vpmaskmovd_q_Vx_Hx_Mx);
+/*  Opcode VEX.66.0F38 0x8d - invalid. */
+/** Opcode VEX.66.0F38 0x8e. */
+FNIEMOP_STUB(iemOp_vpmaskmovd_q_Mx_Vx_Hx);
+/*  Opcode VEX.66.0F38 0x8f - invalid. */
+
+/** Opcode VEX.66.0F38 0x90 (vex only). */
+FNIEMOP_STUB(iemOp_vgatherdd_q_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x91 (vex only). */
+FNIEMOP_STUB(iemOp_vgatherqd_q_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x92 (vex only). */
+FNIEMOP_STUB(iemOp_vgatherdps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x93 (vex only). */
+FNIEMOP_STUB(iemOp_vgatherqps_d_Vx_Hx_Wx);
+/*  Opcode VEX.66.0F38 0x94 - invalid. */
+/*  Opcode VEX.66.0F38 0x95 - invalid. */
+/** Opcode VEX.66.0F38 0x96 (vex only). */
+FNIEMOP_STUB(iemOp_vfmaddsub132ps_q_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x97 (vex only). */
+FNIEMOP_STUB(iemOp_vfmsubadd132ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x98 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd132ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x99 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd132ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9a (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub132ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9b (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub132ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9c (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd132ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9d (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd132ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9e (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub132ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0x9f (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub132ss_d_Vx_Hx_Wx);
+
+/*  Opcode VEX.66.0F38 0xa0 - invalid. */
+/*  Opcode VEX.66.0F38 0xa1 - invalid. */
+/*  Opcode VEX.66.0F38 0xa2 - invalid. */
+/*  Opcode VEX.66.0F38 0xa3 - invalid. */
+/*  Opcode VEX.66.0F38 0xa4 - invalid. */
+/*  Opcode VEX.66.0F38 0xa5 - invalid. */
+/** Opcode VEX.66.0F38 0xa6 (vex only). */
+FNIEMOP_STUB(iemOp_vfmaddsub213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xa7 (vex only). */
+FNIEMOP_STUB(iemOp_vfmsubadd213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xa8 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xa9 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd213ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xaa (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xab (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub213ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xac (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xad (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd213ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xae (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub213ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xaf (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub213ss_d_Vx_Hx_Wx);
+
+/*  Opcode VEX.66.0F38 0xb0 - invalid. */
+/*  Opcode VEX.66.0F38 0xb1 - invalid. */
+/*  Opcode VEX.66.0F38 0xb2 - invalid. */
+/*  Opcode VEX.66.0F38 0xb3 - invalid. */
+/*  Opcode VEX.66.0F38 0xb4 - invalid. */
+/*  Opcode VEX.66.0F38 0xb5 - invalid. */
+/** Opcode VEX.66.0F38 0xb6 (vex only). */
+FNIEMOP_STUB(iemOp_vfmaddsub231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xb7 (vex only). */
+FNIEMOP_STUB(iemOp_vfmsubadd231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xb8 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xb9 (vex only). */
+FNIEMOP_STUB(iemOp_vfmadd231ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xba (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xbb (vex only). */
+FNIEMOP_STUB(iemOp_vfmsub231ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xbc (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xbd (vex only). */
+FNIEMOP_STUB(iemOp_vfnmadd231ss_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xbe (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub231ps_d_Vx_Hx_Wx);
+/** Opcode VEX.66.0F38 0xbf (vex only). */
+FNIEMOP_STUB(iemOp_vfnmsub231ss_d_Vx_Hx_Wx);
+
+/*  Opcode VEX.0F38 0xc0 - invalid. */
+/*  Opcode VEX.66.0F38 0xc0 - invalid. */
+/*  Opcode VEX.0F38 0xc1 - invalid. */
+/*  Opcode VEX.66.0F38 0xc1 - invalid. */
+/*  Opcode VEX.0F38 0xc2 - invalid. */
+/*  Opcode VEX.66.0F38 0xc2 - invalid. */
+/*  Opcode VEX.0F38 0xc3 - invalid. */
+/*  Opcode VEX.66.0F38 0xc3 - invalid. */
+/*  Opcode VEX.0F38 0xc4 - invalid. */
+/*  Opcode VEX.66.0F38 0xc4 - invalid. */
+/*  Opcode VEX.0F38 0xc5 - invalid. */
+/*  Opcode VEX.66.0F38 0xc5 - invalid. */
+/*  Opcode VEX.0F38 0xc6 - invalid. */
+/*  Opcode VEX.66.0F38 0xc6 - invalid. */
+/*  Opcode VEX.0F38 0xc7 - invalid. */
+/*  Opcode VEX.66.0F38 0xc7 - invalid. */
+/** Opcode VEX.0F38 0xc8. */
+FNIEMOP_STUB(iemOp_vsha1nexte_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xc8 - invalid. */
+/** Opcode VEX.0F38 0xc9. */
+FNIEMOP_STUB(iemOp_vsha1msg1_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xc9 - invalid. */
+/** Opcode VEX.0F38 0xca. */
+FNIEMOP_STUB(iemOp_vsha1msg2_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xca - invalid. */
+/** Opcode VEX.0F38 0xcb. */
+FNIEMOP_STUB(iemOp_vsha256rnds2_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xcb - invalid. */
+/** Opcode VEX.0F38 0xcc. */
+FNIEMOP_STUB(iemOp_vsha256msg1_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xcc - invalid. */
+/** Opcode VEX.0F38 0xcd. */
+FNIEMOP_STUB(iemOp_vsha256msg2_Vdq_Wdq);
+/*  Opcode VEX.66.0F38 0xcd - invalid. */
+/*  Opcode VEX.0F38 0xce - invalid. */
+/*  Opcode VEX.66.0F38 0xce - invalid. */
+/*  Opcode VEX.0F38 0xcf - invalid. */
+/*  Opcode VEX.66.0F38 0xcf - invalid. */
+
+/*  Opcode VEX.66.0F38 0xd0 - invalid. */
+/*  Opcode VEX.66.0F38 0xd1 - invalid. */
+/*  Opcode VEX.66.0F38 0xd2 - invalid. */
+/*  Opcode VEX.66.0F38 0xd3 - invalid. */
+/*  Opcode VEX.66.0F38 0xd4 - invalid. */
+/*  Opcode VEX.66.0F38 0xd5 - invalid. */
+/*  Opcode VEX.66.0F38 0xd6 - invalid. */
+/*  Opcode VEX.66.0F38 0xd7 - invalid. */
+/*  Opcode VEX.66.0F38 0xd8 - invalid. */
+/*  Opcode VEX.66.0F38 0xd9 - invalid. */
+/*  Opcode VEX.66.0F38 0xda - invalid. */
+/** Opcode VEX.66.0F38 0xdb. */
+FNIEMOP_STUB(iemOp_vaesimc_Vdq_Wdq);
+/** Opcode VEX.66.0F38 0xdc. */
+FNIEMOP_STUB(iemOp_vaesenc_Vdq_Wdq);
+/** Opcode VEX.66.0F38 0xdd. */
+FNIEMOP_STUB(iemOp_vaesenclast_Vdq_Wdq);
+/** Opcode VEX.66.0F38 0xde. */
+FNIEMOP_STUB(iemOp_vaesdec_Vdq_Wdq);
+/** Opcode VEX.66.0F38 0xdf. */
+FNIEMOP_STUB(iemOp_vaesdeclast_Vdq_Wdq);
+
+/*  Opcode VEX.66.0F38 0xe0 - invalid. */
+/*  Opcode VEX.66.0F38 0xe1 - invalid. */
+/*  Opcode VEX.66.0F38 0xe2 - invalid. */
+/*  Opcode VEX.66.0F38 0xe3 - invalid. */
+/*  Opcode VEX.66.0F38 0xe4 - invalid. */
+/*  Opcode VEX.66.0F38 0xe5 - invalid. */
+/*  Opcode VEX.66.0F38 0xe6 - invalid. */
+/*  Opcode VEX.66.0F38 0xe7 - invalid. */
+/*  Opcode VEX.66.0F38 0xe8 - invalid. */
+/*  Opcode VEX.66.0F38 0xe9 - invalid. */
+/*  Opcode VEX.66.0F38 0xea - invalid. */
+/*  Opcode VEX.66.0F38 0xeb - invalid. */
+/*  Opcode VEX.66.0F38 0xec - invalid. */
+/*  Opcode VEX.66.0F38 0xed - invalid. */
+/*  Opcode VEX.66.0F38 0xee - invalid. */
+/*  Opcode VEX.66.0F38 0xef - invalid. */
+
+
+/*  Opcode VEX.0F38 0xf0 - invalid (legacy only). */
+/*  Opcode VEX.66.0F38 0xf0 - invalid (legacy only). */
+/*  Opcode VEX.F3.0F38 0xf0 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf0 - invalid (legacy only). */
+
+/*  Opcode VEX.0F38 0xf1 - invalid (legacy only). */
+/*  Opcode VEX.66.0F38 0xf1 - invalid (legacy only). */
+/*  Opcode VEX.F3.0F38 0xf1 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf1 - invalid (legacy only). */
+
+/** Opcode VEX.0F38 0xf2 - ANDN (vex only). */
+FNIEMOP_DEF(iemOp_andn_Gy_By_Ey)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, ANDN, andn, Gy, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi1)
+        return iemOp_InvalidNeedRM(pVCpu);
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF | X86_EFL_PF);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+        {
+            IEM_MC_BEGIN(4, 0);
+            IEM_MC_ARG(uint64_t *,          pDst,    0);
+            IEM_MC_ARG(uint64_t,            uSrc1,   1);
+            IEM_MC_ARG(uint64_t,            uSrc2,   2);
+            IEM_MC_ARG(uint32_t *,          pEFlags, 3);
+            IEM_MC_REF_GREG_U64(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_andn_u64, iemAImpl_andn_u64_fallback),
+                                     pDst, uSrc1, uSrc2, pEFlags);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(4, 0);
+            IEM_MC_ARG(uint32_t *,          pDst,    0);
+            IEM_MC_ARG(uint32_t,            uSrc1,   1);
+            IEM_MC_ARG(uint32_t,            uSrc2,   2);
+            IEM_MC_ARG(uint32_t *,          pEFlags, 3);
+            IEM_MC_REF_GREG_U32(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_andn_u32, iemAImpl_andn_u32_fallback),
+                                     pDst, uSrc1, uSrc2, pEFlags);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+        {
+            IEM_MC_BEGIN(4, 1);
+            IEM_MC_ARG(uint64_t *,          pDst,    0);
+            IEM_MC_ARG(uint64_t,            uSrc1,   1);
+            IEM_MC_ARG(uint64_t,            uSrc2,   2);
+            IEM_MC_ARG(uint32_t *,          pEFlags, 3);
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0();
+            IEM_MC_FETCH_MEM_U64(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_REF_GREG_U64(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_andn_u64, iemAImpl_andn_u64_fallback),
+                                     pDst, uSrc1, uSrc2, pEFlags);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(4, 1);
+            IEM_MC_ARG(uint32_t *,          pDst,    0);
+            IEM_MC_ARG(uint32_t,            uSrc1,   1);
+            IEM_MC_ARG(uint32_t,            uSrc2,   2);
+            IEM_MC_ARG(uint32_t *,          pEFlags, 3);
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0();
+            IEM_MC_FETCH_MEM_U32(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_REF_GREG_U32(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_EFLAGS(pEFlags);
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_andn_u32, iemAImpl_andn_u32_fallback),
+                                     pDst, uSrc1, uSrc2, pEFlags);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+/*  Opcode VEX.66.0F38 0xf2 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf2 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf2 - invalid. */
+
+
+/*  Opcode VEX.0F38 0xf3 - invalid. */
+/*  Opcode VEX.66.0F38 0xf3 - invalid. */
+
+/*  Opcode VEX.F3.0F38 0xf3 /0 - invalid. */
+
+/** Body for the vex group 17 instructions. */
+#define IEMOP_BODY_By_Ey(a_Instr) \
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi1) \
+        return iemOp_InvalidWithRM(pVCpu, bRm); /* decode memory variant? */ \
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(X86_EFL_AF | X86_EFL_PF); \
+    if (IEM_IS_MODRM_REG_MODE(bRm)) \
+    { \
+        /* \
+         * Register, register. \
+         */ \
+        IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \
+        { \
+            IEM_MC_BEGIN(3, 0); \
+            IEM_MC_ARG(uint64_t *,          pDst,    0); \
+            IEM_MC_ARG(uint64_t,            uSrc,    1); \
+            IEM_MC_ARG(uint32_t *,          pEFlags, 2); \
+            IEM_MC_REF_GREG_U64(pDst,   IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_FETCH_GREG_U64(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_REF_EFLAGS(pEFlags); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_ ## a_Instr ## _u64, \
+                                                                 iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc, pEFlags); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEM_MC_BEGIN(3, 0); \
+            IEM_MC_ARG(uint32_t *,          pDst,    0); \
+            IEM_MC_ARG(uint32_t,            uSrc,    1); \
+            IEM_MC_ARG(uint32_t *,          pEFlags, 2); \
+            IEM_MC_REF_GREG_U32(pDst,   IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_FETCH_GREG_U32(uSrc, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_REF_EFLAGS(pEFlags); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_ ## a_Instr ## _u32, \
+                                                                 iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc, pEFlags); \
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    else \
+    { \
+        /* \
+         * Register, memory. \
+         */ \
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \
+        { \
+            IEM_MC_BEGIN(3, 1); \
+            IEM_MC_ARG(uint64_t *,          pDst,    0); \
+            IEM_MC_ARG(uint64_t,            uSrc,    1); \
+            IEM_MC_ARG(uint32_t *,          pEFlags, 2); \
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+            IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+            IEM_MC_FETCH_MEM_U64(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_REF_GREG_U64(pDst,  IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_EFLAGS(pEFlags); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_ ## a_Instr ## _u64, \
+                                                                 iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc, pEFlags); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEM_MC_BEGIN(3, 1); \
+            IEM_MC_ARG(uint32_t *,          pDst,    0); \
+            IEM_MC_ARG(uint32_t,            uSrc,    1); \
+            IEM_MC_ARG(uint32_t *,          pEFlags, 2); \
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+            IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+            IEM_MC_FETCH_MEM_U32(uSrc, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_REF_GREG_U32(pDst,  IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_EFLAGS(pEFlags); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(fBmi1, iemAImpl_ ## a_Instr ## _u32, \
+                                                                 iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc, pEFlags); \
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    (void)0
+
+
+/*  Opcode VEX.F3.0F38 0xf3 /1. */
+/** @opcode  /1
+ *  @opmaps  vexgrp17 */
+FNIEMOP_DEF_1(iemOp_VGrp17_blsr_By_Ey, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC2(VEX_VM, BLSR, blsr, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    IEMOP_BODY_By_Ey(blsr);
+}
+
+
+/*  Opcode VEX.F3.0F38 0xf3 /2. */
+/** @opcode  /2
+ *  @opmaps  vexgrp17 */
+FNIEMOP_DEF_1(iemOp_VGrp17_blsmsk_By_Ey, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC2(VEX_VM, BLSMSK, blsmsk, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    IEMOP_BODY_By_Ey(blsmsk);
+}
+
+
+/*  Opcode VEX.F3.0F38 0xf3 /3. */
+/** @opcode  /3
+ *  @opmaps  vexgrp17 */
+FNIEMOP_DEF_1(iemOp_VGrp17_blsi_By_Ey, uint8_t, bRm)
+{
+    IEMOP_MNEMONIC2(VEX_VM, BLSI, blsi, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    IEMOP_BODY_By_Ey(blsi);
+}
+
+
+/*  Opcode VEX.F3.0F38 0xf3 /4 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf3 /5 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf3 /6 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf3 /7 - invalid. */
+
+/**
+ * Group 17 jump table for the VEX.F3 variant.
+ */
+IEM_STATIC const PFNIEMOPRM g_apfnVexGroup17_f3[] =
+{
+    /* /0 */ iemOp_InvalidWithRM,
+    /* /1 */ iemOp_VGrp17_blsr_By_Ey,
+    /* /2 */ iemOp_VGrp17_blsmsk_By_Ey,
+    /* /3 */ iemOp_VGrp17_blsi_By_Ey,
+    /* /4 */ iemOp_InvalidWithRM,
+    /* /5 */ iemOp_InvalidWithRM,
+    /* /6 */ iemOp_InvalidWithRM,
+    /* /7 */ iemOp_InvalidWithRM
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexGroup17_f3) == 8);
+
+/**  Opcode VEX.F3.0F38 0xf3 - invalid (vex only - group 17). */
+FNIEMOP_DEF(iemOp_VGrp17_f3)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    return FNIEMOP_CALL_1(g_apfnVexGroup17_f3[IEM_GET_MODRM_REG_8(bRm)], bRm);
+}
+
+/*  Opcode VEX.F2.0F38 0xf3 - invalid (vex only - group 17). */
+
+
+/*  Opcode VEX.0F38 0xf4 - invalid. */
+/*  Opcode VEX.66.0F38 0xf4 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf4 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf4 - invalid. */
+
+/** Body for BZHI, BEXTR, ++; assumes VEX.L must be 0. */
+#define IEMOP_BODY_Gy_Ey_By(a_Instr, a_fFeatureMember, a_fUndefFlags) \
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->a_fFeatureMember) \
+        return iemOp_InvalidNeedRM(pVCpu); \
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(a_fUndefFlags); \
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \
+    if (IEM_IS_MODRM_REG_MODE(bRm)) \
+    { \
+        /* \
+         * Register, register. \
+         */ \
+        IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \
+        { \
+            IEM_MC_BEGIN(4, 0); \
+            IEM_MC_ARG(uint64_t *,          pDst,    0); \
+            IEM_MC_ARG(uint64_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint64_t,            uSrc2,   2); \
+            IEM_MC_ARG(uint32_t *,          pEFlags, 3); \
+            IEM_MC_REF_GREG_U64(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_EFLAGS(pEFlags); \
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u64, \
+                                                                 iemAImpl_ ## a_Instr ## _u64_fallback), \
+                                     pDst, uSrc1, uSrc2, pEFlags); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEM_MC_BEGIN(4, 0); \
+            IEM_MC_ARG(uint32_t *,          pDst,    0); \
+            IEM_MC_ARG(uint32_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint32_t,            uSrc2,   2); \
+            IEM_MC_ARG(uint32_t *,          pEFlags, 3); \
+            IEM_MC_REF_GREG_U32(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_EFLAGS(pEFlags); \
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u32, \
+                                                                 iemAImpl_ ## a_Instr ## _u32_fallback), \
+                                     pDst, uSrc1, uSrc2, pEFlags); \
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    else \
+    { \
+        /* \
+         * Register, memory. \
+         */ \
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \
+        { \
+            IEM_MC_BEGIN(4, 1); \
+            IEM_MC_ARG(uint64_t *,          pDst,    0); \
+            IEM_MC_ARG(uint64_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint64_t,            uSrc2,   2); \
+            IEM_MC_ARG(uint32_t *,          pEFlags, 3); \
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+            IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+            IEM_MC_FETCH_MEM_U64(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_GREG_U64(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_REF_EFLAGS(pEFlags); \
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u64, \
+                                                                 iemAImpl_ ## a_Instr ## _u64_fallback), \
+                                     pDst, uSrc1, uSrc2, pEFlags); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEM_MC_BEGIN(4, 1); \
+            IEM_MC_ARG(uint32_t *,          pDst,    0); \
+            IEM_MC_ARG(uint32_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint32_t,            uSrc2,   2); \
+            IEM_MC_ARG(uint32_t *,          pEFlags, 3); \
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+            IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+            IEM_MC_FETCH_MEM_U32(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_GREG_U32(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_REF_EFLAGS(pEFlags); \
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u32, \
+                                                                 iemAImpl_ ## a_Instr ## _u32_fallback), \
+                                     pDst, uSrc1, uSrc2, pEFlags); \
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    (void)0
+
+/** Body for SARX, SHLX, SHRX; assumes VEX.L must be 0. */
+#define IEMOP_BODY_Gy_Ey_By_NoEflags(a_Instr, a_fFeatureMember, a_fUndefFlags) \
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->a_fFeatureMember) \
+        return iemOp_InvalidNeedRM(pVCpu); \
+    IEMOP_VERIFICATION_UNDEFINED_EFLAGS(a_fUndefFlags); \
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \
+    if (IEM_IS_MODRM_REG_MODE(bRm)) \
+    { \
+        /* \
+         * Register, register. \
+         */ \
+        IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \
+        { \
+            IEM_MC_BEGIN(3, 0); \
+            IEM_MC_ARG(uint64_t *,          pDst,    0); \
+            IEM_MC_ARG(uint64_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint64_t,            uSrc2,   2); \
+            IEM_MC_REF_GREG_U64(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u64, \
+                                                                 iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc1, uSrc2); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEM_MC_BEGIN(3, 0); \
+            IEM_MC_ARG(uint32_t *,          pDst,    0); \
+            IEM_MC_ARG(uint32_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint32_t,            uSrc2,   2); \
+            IEM_MC_REF_GREG_U32(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u32, \
+                                                                 iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc1, uSrc2); \
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    else \
+    { \
+        /* \
+         * Register, memory. \
+         */ \
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \
+        { \
+            IEM_MC_BEGIN(3, 1); \
+            IEM_MC_ARG(uint64_t *,          pDst,    0); \
+            IEM_MC_ARG(uint64_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint64_t,            uSrc2,   2); \
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+            IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+            IEM_MC_FETCH_MEM_U64(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_GREG_U64(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u64, \
+                                                                 iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc1, uSrc2); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEM_MC_BEGIN(3, 1); \
+            IEM_MC_ARG(uint32_t *,          pDst,    0); \
+            IEM_MC_ARG(uint32_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint32_t,            uSrc2,   2); \
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+            IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+            IEM_MC_FETCH_MEM_U32(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_GREG_U32(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, iemAImpl_ ## a_Instr ## _u32, \
+                                                                 iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc1, uSrc2); \
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    (void)0
+
+/** Opcode VEX.0F38 0xf5 (vex only). */
+FNIEMOP_DEF(iemOp_bzhi_Gy_Ey_By)
+{
+    IEMOP_MNEMONIC3(VEX_RMV, BZHI, bzhi, Gy, Ey, By, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    IEMOP_BODY_Gy_Ey_By(bzhi, fBmi2, X86_EFL_AF | X86_EFL_PF);
+}
+
+/*  Opcode VEX.66.0F38 0xf5 - invalid. */
+
+/** Body for PDEP and PEXT (similar to ANDN, except no EFLAGS). */
+#define IEMOP_BODY_Gy_By_Ey_NoEflags(a_Instr, a_fFeatureMember) \
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->a_fFeatureMember) \
+        return iemOp_InvalidNeedRM(pVCpu); \
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm); \
+    if (IEM_IS_MODRM_REG_MODE(bRm)) \
+    { \
+        /* \
+         * Register, register. \
+         */ \
+        IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \
+        { \
+            IEM_MC_BEGIN(3, 0); \
+            IEM_MC_ARG(uint64_t *,          pDst,    0); \
+            IEM_MC_ARG(uint64_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint64_t,            uSrc2,   2); \
+            IEM_MC_REF_GREG_U64(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, \
+                                                                 iemAImpl_ ## a_Instr ## _u64, \
+                                                                 iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc1, uSrc2); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEM_MC_BEGIN(3, 0); \
+            IEM_MC_ARG(uint32_t *,          pDst,    0); \
+            IEM_MC_ARG(uint32_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint32_t,            uSrc2,   2); \
+            IEM_MC_REF_GREG_U32(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm)); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, \
+                                                                 iemAImpl_ ## a_Instr ## _u32, \
+                                                                 iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc1, uSrc2); \
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    else \
+    { \
+        /* \
+         * Register, memory. \
+         */ \
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W) \
+        { \
+            IEM_MC_BEGIN(3, 1); \
+            IEM_MC_ARG(uint64_t *,          pDst,    0); \
+            IEM_MC_ARG(uint64_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint64_t,            uSrc2,   2); \
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+            IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+            IEM_MC_FETCH_MEM_U64(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_GREG_U64(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, \
+                                                                 iemAImpl_ ## a_Instr ## _u64, \
+                                                                 iemAImpl_ ## a_Instr ## _u64_fallback), pDst, uSrc1, uSrc2); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+        else \
+        { \
+            IEM_MC_BEGIN(3, 1); \
+            IEM_MC_ARG(uint32_t *,          pDst,    0); \
+            IEM_MC_ARG(uint32_t,            uSrc1,   1); \
+            IEM_MC_ARG(uint32_t,            uSrc2,   2); \
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc); \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0); \
+            IEMOP_HLP_DONE_VEX_DECODING_L0(); \
+            IEM_MC_FETCH_MEM_U32(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc); \
+            IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu)); \
+            IEM_MC_REF_GREG_U32(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm)); \
+            IEM_MC_CALL_VOID_AIMPL_3(IEM_SELECT_HOST_OR_FALLBACK(a_fFeatureMember, \
+                                                                 iemAImpl_ ## a_Instr ## _u32, \
+                                                                 iemAImpl_ ## a_Instr ## _u32_fallback), pDst, uSrc1, uSrc2); \
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END(); \
+        } \
+    } \
+    (void)0
+
+
+/** Opcode VEX.F3.0F38 0xf5 (vex only). */
+FNIEMOP_DEF(iemOp_pext_Gy_By_Ey)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, PEXT, pext, Gy, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    IEMOP_BODY_Gy_By_Ey_NoEflags(pext, fBmi2);
+}
+
+
+/** Opcode VEX.F2.0F38 0xf5 (vex only). */
+FNIEMOP_DEF(iemOp_pdep_Gy_By_Ey)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, PDEP, pdep, Gy, By, Ey, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    IEMOP_BODY_Gy_By_Ey_NoEflags(pdep, fBmi2);
+}
+
+
+/*  Opcode VEX.0F38 0xf6 - invalid. */
+/*  Opcode VEX.66.0F38 0xf6 - invalid (legacy only). */
+/*  Opcode VEX.F3.0F38 0xf6 - invalid (legacy only). */
+
+
+/** Opcode VEX.F2.0F38 0xf6 (vex only) */
+FNIEMOP_DEF(iemOp_mulx_By_Gy_rDX_Ey)
+{
+    IEMOP_MNEMONIC4(VEX_RVM, MULX, mulx, Gy, By, Ey, rDX, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi2)
+        return iemOp_InvalidNeedRM(pVCpu);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        IEMOP_HLP_DONE_VEX_DECODING_L0();
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+        {
+            IEM_MC_BEGIN(4, 0);
+            IEM_MC_ARG(uint64_t *,          pDst1,   0);
+            IEM_MC_ARG(uint64_t *,          pDst2,   1);
+            IEM_MC_ARG(uint64_t,            uSrc1,   2);
+            IEM_MC_ARG(uint64_t,            uSrc2,   3);
+            IEM_MC_REF_GREG_U64(pDst1,   IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_GREG_U64(pDst2,   IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_FETCH_GREG_U64(uSrc1, X86_GREG_xDX);
+            IEM_MC_FETCH_GREG_U64(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi2, iemAImpl_mulx_u64, iemAImpl_mulx_u64_fallback),
+                                     pDst1, pDst2, uSrc1, uSrc2);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(4, 0);
+            IEM_MC_ARG(uint32_t *,          pDst1,   0);
+            IEM_MC_ARG(uint32_t *,          pDst2,   1);
+            IEM_MC_ARG(uint32_t,            uSrc1,   2);
+            IEM_MC_ARG(uint32_t,            uSrc2,   3);
+            IEM_MC_REF_GREG_U32(pDst1,   IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_GREG_U32(pDst2,   IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_FETCH_GREG_U32(uSrc1, X86_GREG_xDX);
+            IEM_MC_FETCH_GREG_U32(uSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi2, iemAImpl_mulx_u32, iemAImpl_mulx_u32_fallback),
+                                     pDst1, pDst2, uSrc1, uSrc2);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst2);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst1);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+        {
+            IEM_MC_BEGIN(4, 1);
+            IEM_MC_ARG(uint64_t *,          pDst1,   0);
+            IEM_MC_ARG(uint64_t *,          pDst2,   1);
+            IEM_MC_ARG(uint64_t,            uSrc1,   2);
+            IEM_MC_ARG(uint64_t,            uSrc2,   3);
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0();
+            IEM_MC_FETCH_MEM_U64(uSrc2,  pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_GREG_U64(uSrc1, X86_GREG_xDX);
+            IEM_MC_REF_GREG_U64(pDst2,   IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_REF_GREG_U64(pDst1,   IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi2, iemAImpl_mulx_u64, iemAImpl_mulx_u64_fallback),
+                                     pDst1, pDst2, uSrc1, uSrc2);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(4, 1);
+            IEM_MC_ARG(uint32_t *,          pDst1,   0);
+            IEM_MC_ARG(uint32_t *,          pDst2,   1);
+            IEM_MC_ARG(uint32_t,            uSrc1,   2);
+            IEM_MC_ARG(uint32_t,            uSrc2,   3);
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            IEMOP_HLP_DONE_VEX_DECODING_L0();
+            IEM_MC_FETCH_MEM_U32(uSrc2,  pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_GREG_U32(uSrc1, X86_GREG_xDX);
+            IEM_MC_REF_GREG_U32(pDst2,   IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_REF_GREG_U32(pDst1,   IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fBmi2, iemAImpl_mulx_u32, iemAImpl_mulx_u32_fallback),
+                                     pDst1, pDst2, uSrc1, uSrc2);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst2);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst1);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode VEX.0F38 0xf7 (vex only). */
+FNIEMOP_DEF(iemOp_bextr_Gy_Ey_By)
+{
+    IEMOP_MNEMONIC3(VEX_RMV, BEXTR, bextr, Gy, Ey, By, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    IEMOP_BODY_Gy_Ey_By(bextr, fBmi1, X86_EFL_SF | X86_EFL_AF | X86_EFL_PF);
+}
+
+
+/** Opcode VEX.66.0F38 0xf7 (vex only). */
+FNIEMOP_DEF(iemOp_shlx_Gy_Ey_By)
+{
+    IEMOP_MNEMONIC3(VEX_RMV, SHLX, shlx, Gy, Ey, By, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    IEMOP_BODY_Gy_Ey_By_NoEflags(shlx, fBmi2, 0);
+}
+
+
+/** Opcode VEX.F3.0F38 0xf7 (vex only). */
+FNIEMOP_DEF(iemOp_sarx_Gy_Ey_By)
+{
+    IEMOP_MNEMONIC3(VEX_RMV, SARX, sarx, Gy, Ey, By, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    IEMOP_BODY_Gy_Ey_By_NoEflags(sarx, fBmi2, 0);
+}
+
+
+/** Opcode VEX.F2.0F38 0xf7 (vex only). */
+FNIEMOP_DEF(iemOp_shrx_Gy_Ey_By)
+{
+    IEMOP_MNEMONIC3(VEX_RMV, SHRX, shrx, Gy, Ey, By, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO);
+    IEMOP_BODY_Gy_Ey_By_NoEflags(shrx, fBmi2, 0);
+}
+
+/*  Opcode VEX.0F38 0xf8 - invalid. */
+/*  Opcode VEX.66.0F38 0xf8 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf8 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf8 - invalid. */
+
+/*  Opcode VEX.0F38 0xf9 - invalid. */
+/*  Opcode VEX.66.0F38 0xf9 - invalid. */
+/*  Opcode VEX.F3.0F38 0xf9 - invalid. */
+/*  Opcode VEX.F2.0F38 0xf9 - invalid. */
+
+/*  Opcode VEX.0F38 0xfa - invalid. */
+/*  Opcode VEX.66.0F38 0xfa - invalid. */
+/*  Opcode VEX.F3.0F38 0xfa - invalid. */
+/*  Opcode VEX.F2.0F38 0xfa - invalid. */
+
+/*  Opcode VEX.0F38 0xfb - invalid. */
+/*  Opcode VEX.66.0F38 0xfb - invalid. */
+/*  Opcode VEX.F3.0F38 0xfb - invalid. */
+/*  Opcode VEX.F2.0F38 0xfb - invalid. */
+
+/*  Opcode VEX.0F38 0xfc - invalid. */
+/*  Opcode VEX.66.0F38 0xfc - invalid. */
+/*  Opcode VEX.F3.0F38 0xfc - invalid. */
+/*  Opcode VEX.F2.0F38 0xfc - invalid. */
+
+/*  Opcode VEX.0F38 0xfd - invalid. */
+/*  Opcode VEX.66.0F38 0xfd - invalid. */
+/*  Opcode VEX.F3.0F38 0xfd - invalid. */
+/*  Opcode VEX.F2.0F38 0xfd - invalid. */
+
+/*  Opcode VEX.0F38 0xfe - invalid. */
+/*  Opcode VEX.66.0F38 0xfe - invalid. */
+/*  Opcode VEX.F3.0F38 0xfe - invalid. */
+/*  Opcode VEX.F2.0F38 0xfe - invalid. */
+
+/*  Opcode VEX.0F38 0xff - invalid. */
+/*  Opcode VEX.66.0F38 0xff - invalid. */
+/*  Opcode VEX.F3.0F38 0xff - invalid. */
+/*  Opcode VEX.F2.0F38 0xff - invalid. */
+
+
+/**
+ * VEX opcode map \#2.
+ *
+ * @sa      g_apfnThreeByte0f38
+ */
+IEM_STATIC const PFNIEMOP g_apfnVexMap2[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  iemOp_InvalidNeedRM,        iemOp_vpshufb_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x01 */  iemOp_InvalidNeedRM,        iemOp_vphaddw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x02 */  iemOp_InvalidNeedRM,        iemOp_vphaddd_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x03 */  iemOp_InvalidNeedRM,        iemOp_vphaddsw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x04 */  iemOp_InvalidNeedRM,        iemOp_vpmaddubsw_Vx_Hx_Wx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x05 */  iemOp_InvalidNeedRM,        iemOp_vphsubw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x06 */  iemOp_InvalidNeedRM,        iemOp_vphsubd_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x07 */  iemOp_InvalidNeedRM,        iemOp_vphsubsw_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x08 */  iemOp_InvalidNeedRM,        iemOp_vpsignb_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x09 */  iemOp_InvalidNeedRM,        iemOp_vpsignw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0a */  iemOp_InvalidNeedRM,        iemOp_vpsignd_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0b */  iemOp_InvalidNeedRM,        iemOp_vpmulhrsw_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0c */  iemOp_InvalidNeedRM,        iemOp_vpermilps_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0d */  iemOp_InvalidNeedRM,        iemOp_vpermilpd_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0e */  iemOp_InvalidNeedRM,        iemOp_vtestps_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x0f */  iemOp_InvalidNeedRM,        iemOp_vtestpd_Vx_Wx,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x10 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x11 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x12 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x13 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x14 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x15 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x16 */  iemOp_InvalidNeedRM,        iemOp_vpermps_Vqq_Hqq_Wqq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x17 */  iemOp_InvalidNeedRM,        iemOp_vptest_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x18 */  iemOp_InvalidNeedRM,        iemOp_vbroadcastss_Vx_Wd,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x19 */  iemOp_InvalidNeedRM,        iemOp_vbroadcastsd_Vqq_Wq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1a */  iemOp_InvalidNeedRM,        iemOp_vbroadcastf128_Vqq_Mdq, iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x1b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x1c */  iemOp_InvalidNeedRM,        iemOp_vpabsb_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1d */  iemOp_InvalidNeedRM,        iemOp_vpabsw_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1e */  iemOp_InvalidNeedRM,        iemOp_vpabsd_Vx_Wx,         iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x1f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x20 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxbw_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x21 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxbd_Vx_UxMd,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x22 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxbq_Vx_UxMw,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x23 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxwd_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x24 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxwq_Vx_UxMd,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x25 */  iemOp_InvalidNeedRM,        iemOp_vpmovsxdq_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x26 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x27 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x28 */  iemOp_InvalidNeedRM,        iemOp_vpmuldq_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x29 */  iemOp_InvalidNeedRM,        iemOp_vpcmpeqq_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2a */  iemOp_InvalidNeedRM,        iemOp_vmovntdqa_Vx_Mx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2b */  iemOp_InvalidNeedRM,        iemOp_vpackusdw_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2c */  iemOp_InvalidNeedRM,        iemOp_vmaskmovps_Vx_Hx_Mx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2d */  iemOp_InvalidNeedRM,        iemOp_vmaskmovpd_Vx_Hx_Mx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2e */  iemOp_InvalidNeedRM,        iemOp_vmaskmovps_Mx_Hx_Vx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x2f */  iemOp_InvalidNeedRM,        iemOp_vmaskmovpd_Mx_Hx_Vx,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x30 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxbw_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x31 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxbd_Vx_UxMd,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x32 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxbq_Vx_UxMw,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x33 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxwd_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x34 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxwq_Vx_UxMd,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x35 */  iemOp_InvalidNeedRM,        iemOp_vpmovzxdq_Vx_UxMq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x36 */  iemOp_InvalidNeedRM,        iemOp_vpermd_Vqq_Hqq_Wqq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x37 */  iemOp_InvalidNeedRM,        iemOp_vpcmpgtq_Vx_Hx_Wx,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x38 */  iemOp_InvalidNeedRM,        iemOp_vpminsb_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x39 */  iemOp_InvalidNeedRM,        iemOp_vpminsd_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3a */  iemOp_InvalidNeedRM,        iemOp_vpminuw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3b */  iemOp_InvalidNeedRM,        iemOp_vpminud_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3c */  iemOp_InvalidNeedRM,        iemOp_vpmaxsb_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3d */  iemOp_InvalidNeedRM,        iemOp_vpmaxsd_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3e */  iemOp_InvalidNeedRM,        iemOp_vpmaxuw_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x3f */  iemOp_InvalidNeedRM,        iemOp_vpmaxud_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0x40 */  iemOp_InvalidNeedRM,        iemOp_vpmulld_Vx_Hx_Wx,     iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x41 */  iemOp_InvalidNeedRM,        iemOp_vphminposuw_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x42 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x43 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x44 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x45 */  iemOp_InvalidNeedRM,        iemOp_vpsrlvd_q_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x46 */  iemOp_InvalidNeedRM,        iemOp_vsravd_Vx_Hx_Wx,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x47 */  iemOp_InvalidNeedRM,        iemOp_vpsllvd_q_Vx_Hx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x48 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x49 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x4f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x50 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x51 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x52 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x53 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x54 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x55 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x56 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x57 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x58 */  iemOp_InvalidNeedRM,        iemOp_vpbroadcastd_Vx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x59 */  iemOp_InvalidNeedRM,        iemOp_vpbroadcastq_Vx_Wx,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0x5a */  iemOp_InvalidNeedRM,        iemOp_vbroadcasti128_Vqq_Mdq, iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x5b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x5f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x60 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x61 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x62 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x63 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x64 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x65 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x66 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x67 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x68 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x69 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x6f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x70 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x71 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x72 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x73 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x74 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x75 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x76 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x77 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x78 */  iemOp_InvalidNeedRM,        iemOp_vpboardcastb_Vx_Wx,   iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x79 */  iemOp_InvalidNeedRM,        iemOp_vpboardcastw_Vx_Wx,   iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x7a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7c */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7e */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x7f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x80 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x81 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x82 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x83 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x84 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x85 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x86 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x87 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x88 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x89 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8a */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8b */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8c */  iemOp_InvalidNeedRM,        iemOp_vpmaskmovd_q_Vx_Hx_Mx, iemOp_InvalidNeedRM,     iemOp_InvalidNeedRM,
+    /* 0x8d */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x8e */  iemOp_InvalidNeedRM,        iemOp_vpmaskmovd_q_Mx_Vx_Hx, iemOp_InvalidNeedRM,     iemOp_InvalidNeedRM,
+    /* 0x8f */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0x90 */  iemOp_InvalidNeedRM,        iemOp_vgatherdd_q_Vx_Hx_Wx, iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x91 */  iemOp_InvalidNeedRM,        iemOp_vgatherqd_q_Vx_Hx_Wx, iemOp_InvalidNeedRM,      iemOp_InvalidNeedRM,
+    /* 0x92 */  iemOp_InvalidNeedRM,        iemOp_vgatherdps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM,     iemOp_InvalidNeedRM,
+    /* 0x93 */  iemOp_InvalidNeedRM,        iemOp_vgatherqps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM,     iemOp_InvalidNeedRM,
+    /* 0x94 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x95 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0x96 */  iemOp_InvalidNeedRM,        iemOp_vfmaddsub132ps_q_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x97 */  iemOp_InvalidNeedRM,        iemOp_vfmsubadd132ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x98 */  iemOp_InvalidNeedRM,        iemOp_vfmadd132ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x99 */  iemOp_InvalidNeedRM,        iemOp_vfmadd132ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9a */  iemOp_InvalidNeedRM,        iemOp_vfmsub132ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9b */  iemOp_InvalidNeedRM,        iemOp_vfmsub132ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9c */  iemOp_InvalidNeedRM,        iemOp_vfnmadd132ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9d */  iemOp_InvalidNeedRM,        iemOp_vfnmadd132ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9e */  iemOp_InvalidNeedRM,        iemOp_vfnmsub132ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0x9f */  iemOp_InvalidNeedRM,        iemOp_vfnmsub132ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+
+    /* 0xa0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xa6 */  iemOp_InvalidNeedRM,        iemOp_vfmaddsub213ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xa7 */  iemOp_InvalidNeedRM,        iemOp_vfmsubadd213ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xa8 */  iemOp_InvalidNeedRM,        iemOp_vfmadd213ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xa9 */  iemOp_InvalidNeedRM,        iemOp_vfmadd213ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xaa */  iemOp_InvalidNeedRM,        iemOp_vfmsub213ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xab */  iemOp_InvalidNeedRM,        iemOp_vfmsub213ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xac */  iemOp_InvalidNeedRM,        iemOp_vfnmadd213ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xad */  iemOp_InvalidNeedRM,        iemOp_vfnmadd213ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xae */  iemOp_InvalidNeedRM,        iemOp_vfnmsub213ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xaf */  iemOp_InvalidNeedRM,        iemOp_vfnmsub213ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+
+    /* 0xb0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xb6 */  iemOp_InvalidNeedRM,        iemOp_vfmaddsub231ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xb7 */  iemOp_InvalidNeedRM,        iemOp_vfmsubadd231ps_d_Vx_Hx_Wx, iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xb8 */  iemOp_InvalidNeedRM,        iemOp_vfmadd231ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xb9 */  iemOp_InvalidNeedRM,        iemOp_vfmadd231ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xba */  iemOp_InvalidNeedRM,        iemOp_vfmsub231ps_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xbb */  iemOp_InvalidNeedRM,        iemOp_vfmsub231ss_d_Vx_Hx_Wx,    iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xbc */  iemOp_InvalidNeedRM,        iemOp_vfnmadd231ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xbd */  iemOp_InvalidNeedRM,        iemOp_vfnmadd231ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xbe */  iemOp_InvalidNeedRM,        iemOp_vfnmsub231ps_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+    /* 0xbf */  iemOp_InvalidNeedRM,        iemOp_vfnmsub231ss_d_Vx_Hx_Wx,   iemOp_InvalidNeedRM, iemOp_InvalidNeedRM,
+
+    /* 0xc0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xc8 */  iemOp_vsha1nexte_Vdq_Wdq,   iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xc9 */  iemOp_vsha1msg1_Vdq_Wdq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xca */  iemOp_vsha1msg2_Vdq_Wdq,    iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcb */  iemOp_vsha256rnds2_Vdq_Wdq, iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcc */  iemOp_vsha256msg1_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xcd */  iemOp_vsha256msg2_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xce */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xcf */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xd0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xd9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xda */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xdb */  iemOp_InvalidNeedRM,        iemOp_vaesimc_Vdq_Wdq,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdc */  iemOp_InvalidNeedRM,        iemOp_vaesenc_Vdq_Wdq,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdd */  iemOp_InvalidNeedRM,        iemOp_vaesenclast_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xde */  iemOp_InvalidNeedRM,        iemOp_vaesdec_Vdq_Wdq,      iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xdf */  iemOp_InvalidNeedRM,        iemOp_vaesdeclast_Vdq_Wdq,  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+
+    /* 0xe0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe2 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe3 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe5 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe6 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe7 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xe9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xea */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xeb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xec */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xed */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xee */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xef */  IEMOP_X4(iemOp_InvalidNeedRM),
+
+    /* 0xf0 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf1 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf2 */  iemOp_andn_Gy_By_Ey,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf3 */  iemOp_VGrp17_f3,            iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,
+    /* 0xf4 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf5 */  iemOp_bzhi_Gy_Ey_By,        iemOp_InvalidNeedRM,        iemOp_pext_Gy_By_Ey,        iemOp_pdep_Gy_By_Ey,
+    /* 0xf6 */  iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_InvalidNeedRM,        iemOp_mulx_By_Gy_rDX_Ey,
+    /* 0xf7 */  iemOp_bextr_Gy_Ey_By,       iemOp_shlx_Gy_Ey_By,        iemOp_sarx_Gy_Ey_By,        iemOp_shrx_Gy_Ey_By,
+    /* 0xf8 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xf9 */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfa */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfb */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfc */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfd */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xfe */  IEMOP_X4(iemOp_InvalidNeedRM),
+    /* 0xff */  IEMOP_X4(iemOp_InvalidNeedRM),
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexMap2) == 1024);
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h
new file mode 100644
index 00000000..6b6e5af4
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IEMAllInstructionsVexMap3.cpp.h
@@ -0,0 +1,1004 @@
+/* $Id: IEMAllInstructionsVexMap3.cpp.h $ */
+/** @file
+ * IEM - Instruction Decoding and Emulation, 0x0f 0x3a map.
+ *
+ * @remarks IEMAllInstructionsThree0f3a.cpp.h is a VEX mirror of this file.
+ *          Any update here is likely needed in that file too.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/** @name VEX Opcode Map 3
+ * @{
+ */
+
+/**
+ * Common worker for AVX2 instructions on the forms:
+ *     - vpxxx    xmm0, xmm1, xmm2/mem128, imm8
+ *     - vpxxx    ymm0, ymm1, ymm2/mem256, imm8
+ *
+ * Takes function table for function w/o implicit state parameter.
+ *
+ * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit.
+ */
+FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Ib_Opt, PCIEMOPMEDIAOPTF3IMM8, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_BEGIN(4, 3);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2);
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,  uDst,  0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1, 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2, 2);
+            IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/ bImm, 3);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_FETCH_YREG_U256(uSrc1,   IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_FETCH_YREG_U256(uSrc2,   IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU256, puDst, puSrc1, puSrc2, bImmArg);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_BEGIN(4, 0);
+            IEM_MC_ARG(PRTUINT128U,          puDst,  0);
+            IEM_MC_ARG(PCRTUINT128U,         puSrc1, 1);
+            IEM_MC_ARG(PCRTUINT128U,         puSrc2, 2);
+            IEM_MC_ARG_CONST(uint8_t,        bImmArg, /*=*/ bImm, 3);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_XREG_U128(puDst,        IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU128, puDst, puSrc1, puSrc2, bImmArg);
+            IEM_MC_CLEAR_YREG_128_UP(          IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEM_MC_BEGIN(4, 4);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,  uDst,  0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1, 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2, 2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEM_MC_ARG_CONST(uint8_t,           bImmArg, /*=*/ bImm, 3);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_YREG_U256(uSrc1,      IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU256, puDst, puSrc1, puSrc2, bImmArg);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(   IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(4, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_ARG(PRTUINT128U,                 puDst,         0);
+            IEM_MC_ARG(PCRTUINT128U,                puSrc1,        1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc2, uSrc2, 2);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+            IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 3);
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc1,  IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU128, puDst, puSrc1, puSrc2, bImmArg);
+            IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode VEX.66.0F3A 0x00. */
+FNIEMOP_STUB(iemOp_vpermq_Vqq_Wqq_Ib);
+/** Opcode VEX.66.0F3A 0x01. */
+FNIEMOP_STUB(iemOp_vpermqd_Vqq_Wqq_Ib);
+/** Opcode VEX.66.0F3A 0x02. */
+FNIEMOP_STUB(iemOp_vpblendd_Vx_Wx_Ib);
+/*  Opcode VEX.66.0F3A 0x03 - invalid */
+/** Opcode VEX.66.0F3A 0x04. */
+FNIEMOP_STUB(iemOp_vpermilps_Vx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x05. */
+FNIEMOP_STUB(iemOp_vpermilpd_Vx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x06 (vex only) */
+FNIEMOP_STUB(iemOp_vperm2f128_Vqq_Hqq_Wqq_Ib);
+/*  Opcode VEX.66.0F3A 0x07 - invalid */
+/** Opcode VEX.66.0F3A 0x08. */
+FNIEMOP_STUB(iemOp_vroundps_Vx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x09. */
+FNIEMOP_STUB(iemOp_vroundpd_Vx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x0a. */
+FNIEMOP_STUB(iemOp_vroundss_Vss_Wss_Ib);
+/** Opcode VEX.66.0F3A 0x0b. */
+FNIEMOP_STUB(iemOp_vroundsd_Vsd_Wsd_Ib);
+
+
+/** Opcode VEX.66.0F3A 0x0c. */
+FNIEMOP_DEF(iemOp_vblendps_Vx_Hx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VBLENDPS, vblendps, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); /* @todo */
+    IEMOPMEDIAOPTF3IMM8_INIT_VARS(vblendps);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Ib_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F3A 0x0d. */
+FNIEMOP_DEF(iemOp_vblendpd_Vx_Hx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VBLENDPD, vblendpd, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); /* @todo */
+    IEMOPMEDIAOPTF3IMM8_INIT_VARS(vblendpd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Ib_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F3A 0x0e. */
+FNIEMOP_DEF(iemOp_vpblendw_Vx_Hx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPBLENDW, vpblendw, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); /* @todo */
+    IEMOPMEDIAOPTF3IMM8_INIT_VARS(vpblendw);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Ib_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.0F3A 0x0f - invalid. */
+
+
+/** Opcode VEX.66.0F3A 0x0f. */
+FNIEMOP_DEF(iemOp_vpalignr_Vx_Hx_Wx_Ib)
+{
+    IEMOP_MNEMONIC3(VEX_RVM, VPALIGNR, vpalignr, Vx, Hx, Wx, DISOPTYPE_HARMLESS, 0); /* @todo */
+    IEMOPMEDIAOPTF3IMM8_INIT_VARS(vpalignr);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Ib_Opt, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.66.0F3A 0x10 - invalid */
+/*  Opcode VEX.66.0F3A 0x11 - invalid */
+/*  Opcode VEX.66.0F3A 0x12 - invalid */
+/*  Opcode VEX.66.0F3A 0x13 - invalid */
+/** Opcode VEX.66.0F3A 0x14. */
+FNIEMOP_STUB(iemOp_vpextrb_RdMb_Vdq_Ib);
+/** Opcode VEX.66.0F3A 0x15. */
+FNIEMOP_STUB(iemOp_vpextrw_RdMw_Vdq_Ib);
+/** Opcode VEX.66.0F3A 0x16. */
+FNIEMOP_STUB(iemOp_vpextrd_q_RdMw_Vdq_Ib);
+/** Opcode VEX.66.0F3A 0x17. */
+FNIEMOP_STUB(iemOp_vextractps_Ed_Vdq_Ib);
+/** Opcode VEX.66.0F3A 0x18 (vex only). */
+FNIEMOP_STUB(iemOp_vinsertf128_Vqq_Hqq_Wqq_Ib);
+/** Opcode VEX.66.0F3A 0x19 (vex only). */
+FNIEMOP_STUB(iemOp_vextractf128_Wdq_Vqq_Ib);
+/*  Opcode VEX.66.0F3A 0x1a - invalid */
+/*  Opcode VEX.66.0F3A 0x1b - invalid */
+/*  Opcode VEX.66.0F3A 0x1c - invalid */
+/** Opcode VEX.66.0F3A 0x1d (vex only). */
+FNIEMOP_STUB(iemOp_vcvtps2ph_Wx_Vx_Ib);
+/*  Opcode VEX.66.0F3A 0x1e - invalid */
+/*  Opcode VEX.66.0F3A 0x1f - invalid */
+
+
+/** Opcode VEX.66.0F3A 0x20. */
+FNIEMOP_STUB(iemOp_vpinsrb_Vdq_Hdq_RyMb_Ib);
+/** Opcode VEX.66.0F3A 0x21, */
+FNIEMOP_STUB(iemOp_vinsertps_Vdq_Hdq_UdqMd_Ib);
+/** Opcode VEX.66.0F3A 0x22. */
+FNIEMOP_STUB(iemOp_vpinsrd_q_Vdq_Hdq_Ey_Ib);
+/*  Opcode VEX.66.0F3A 0x23 - invalid */
+/*  Opcode VEX.66.0F3A 0x24 - invalid */
+/*  Opcode VEX.66.0F3A 0x25 - invalid */
+/*  Opcode VEX.66.0F3A 0x26 - invalid */
+/*  Opcode VEX.66.0F3A 0x27 - invalid */
+/*  Opcode VEX.66.0F3A 0x28 - invalid */
+/*  Opcode VEX.66.0F3A 0x29 - invalid */
+/*  Opcode VEX.66.0F3A 0x2a - invalid */
+/*  Opcode VEX.66.0F3A 0x2b - invalid */
+/*  Opcode VEX.66.0F3A 0x2c - invalid */
+/*  Opcode VEX.66.0F3A 0x2d - invalid */
+/*  Opcode VEX.66.0F3A 0x2e - invalid */
+/*  Opcode VEX.66.0F3A 0x2f - invalid */
+
+
+/*  Opcode VEX.66.0F3A 0x30 - invalid */
+/*  Opcode VEX.66.0F3A 0x31 - invalid */
+/*  Opcode VEX.66.0F3A 0x32 - invalid */
+/*  Opcode VEX.66.0F3A 0x33 - invalid */
+/*  Opcode VEX.66.0F3A 0x34 - invalid */
+/*  Opcode VEX.66.0F3A 0x35 - invalid */
+/*  Opcode VEX.66.0F3A 0x36 - invalid */
+/*  Opcode VEX.66.0F3A 0x37 - invalid */
+/** Opcode VEX.66.0F3A 0x38 (vex only). */
+FNIEMOP_STUB(iemOp_vinserti128_Vqq_Hqq_Wqq_Ib);
+/** Opcode VEX.66.0F3A 0x39 (vex only). */
+FNIEMOP_STUB(iemOp_vextracti128_Wdq_Vqq_Ib);
+/*  Opcode VEX.66.0F3A 0x3a - invalid */
+/*  Opcode VEX.66.0F3A 0x3b - invalid */
+/*  Opcode VEX.66.0F3A 0x3c - invalid */
+/*  Opcode VEX.66.0F3A 0x3d - invalid */
+/*  Opcode VEX.66.0F3A 0x3e - invalid */
+/*  Opcode VEX.66.0F3A 0x3f - invalid */
+
+
+/** Opcode VEX.66.0F3A 0x40. */
+FNIEMOP_STUB(iemOp_vdpps_Vx_Hx_Wx_Ib);
+/** Opcode VEX.66.0F3A 0x41, */
+FNIEMOP_STUB(iemOp_vdppd_Vdq_Hdq_Wdq_Ib);
+/** Opcode VEX.66.0F3A 0x42. */
+FNIEMOP_STUB(iemOp_vmpsadbw_Vx_Hx_Wx_Ib);
+/*  Opcode VEX.66.0F3A 0x43 - invalid */
+
+
+/** Opcode VEX.66.0F3A 0x44. */
+FNIEMOP_DEF(iemOp_vpclmulqdq_Vdq_Hdq_Wdq_Ib)
+{
+    //IEMOP_MNEMONIC3(VEX_RVM, VPCLMULQDQ, vpclmulqdq, Vdq, Hdq, Wdq, DISOPTYPE_HARMLESS, 0); /* @todo */
+
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fPclMul);
+        IEM_MC_BEGIN(4, 0);
+        IEM_MC_ARG(PRTUINT128U,          puDst,  0);
+        IEM_MC_ARG(PCRTUINT128U,         puSrc1, 1);
+        IEM_MC_ARG(PCRTUINT128U,         puSrc2, 2);
+        IEM_MC_ARG_CONST(uint8_t,        bImmArg, /*=*/ bImm, 3);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+        IEM_MC_REF_XREG_U128(puDst,        IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu));
+        IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fPclMul, iemAImpl_vpclmulqdq_u128, iemAImpl_vpclmulqdq_u128_fallback),
+                                                             puDst, puSrc1, puSrc2, bImmArg);
+        IEM_MC_CLEAR_YREG_128_UP(          IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        IEM_MC_BEGIN(4, 2);
+        IEM_MC_LOCAL(RTUINT128U,                uSrc2);
+        IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+        IEM_MC_ARG(PRTUINT128U,                 puDst,         0);
+        IEM_MC_ARG(PCRTUINT128U,                puSrc1,        1);
+        IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc2, uSrc2, 2);
+
+        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+        uint8_t bImm; IEM_OPCODE_GET_NEXT_U8(&bImm);
+        IEM_MC_ARG_CONST(uint8_t,               bImmArg, /*=*/ bImm, 3);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_EX(fPclMul);
+        IEM_MC_MAYBE_RAISE_AVX_RELATED_XCPT();
+        IEM_MC_PREPARE_AVX_USAGE();
+
+        IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+        IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+        IEM_MC_REF_XREG_U128_CONST(puSrc1,  IEM_GET_EFFECTIVE_VVVV(pVCpu));
+        IEM_MC_CALL_VOID_AIMPL_4(IEM_SELECT_HOST_OR_FALLBACK(fPclMul, iemAImpl_vpclmulqdq_u128, iemAImpl_vpclmulqdq_u128_fallback),
+                                                             puDst, puSrc1, puSrc2, bImmArg);
+        IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm));
+
+        IEM_MC_ADVANCE_RIP_AND_FINISH();
+        IEM_MC_END();
+    }
+}
+
+
+/*  Opcode VEX.66.0F3A 0x45 - invalid */
+/** Opcode VEX.66.0F3A 0x46 (vex only)  */
+FNIEMOP_STUB(iemOp_vperm2i128_Vqq_Hqq_Wqq_Ib);
+/*  Opcode VEX.66.0F3A 0x47 - invalid */
+/** Opcode VEX.66.0F3A 0x48 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vperlmilzz2ps_Vx_Hx_Wp_Lx);
+/** Opcode VEX.66.0F3A 0x49 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vperlmilzz2pd_Vx_Hx_Wp_Lx);
+
+
+/**
+ * Common worker for AVX2 instructions on the forms:
+ *     - vpxxx    xmm0, xmm1, xmm2/mem128, xmm4
+ *     - vpxxx    ymm0, ymm1, ymm2/mem256, ymm4
+ *
+ * Exceptions type 4. AVX cpuid check for 128-bit operation, AVX2 for 256-bit.
+ */
+FNIEMOP_DEF_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Lx, PCIEMOPBLENDOP, pImpl)
+{
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            uint8_t bOp4; IEM_OPCODE_GET_NEXT_U8(&bOp4);
+
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_BEGIN(4, 4);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc3);
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,  uDst,  0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1, 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2, 2);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc3, uSrc3, 3);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_FETCH_YREG_U256(uSrc1,   IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_FETCH_YREG_U256(uSrc2,   IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_FETCH_YREG_U256(uSrc3,   bOp4 >> 4); /** @todo Ignore MSB in 32-bit mode. */
+            IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU256, puDst, puSrc1, puSrc2, puSrc3);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            uint8_t bOp4; IEM_OPCODE_GET_NEXT_U8(&bOp4);
+
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_BEGIN(4, 0);
+            IEM_MC_ARG(PRTUINT128U,          puDst,  0);
+            IEM_MC_ARG(PCRTUINT128U,         puSrc1, 1);
+            IEM_MC_ARG(PCRTUINT128U,         puSrc2, 2);
+            IEM_MC_ARG(PCRTUINT128U,         puSrc3, 3);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+            IEM_MC_REF_XREG_U128(puDst,        IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc1, IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_REF_XREG_U128_CONST(puSrc2, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc3, bOp4 >> 4); /** @todo Ignore MSB in 32-bit mode. */
+            IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU128, puDst, puSrc1, puSrc2, puSrc3);
+            IEM_MC_CLEAR_YREG_128_UP(          IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.uVexLength)
+        {
+            IEM_MC_BEGIN(4, 5);
+            IEM_MC_LOCAL(RTUINT256U,            uDst);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc1);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc2);
+            IEM_MC_LOCAL(RTUINT256U,            uSrc3);
+            IEM_MC_LOCAL(RTGCPTR,               GCPtrEffSrc);
+            IEM_MC_ARG_LOCAL_REF(PRTUINT256U,   puDst,  uDst,  0);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc1, uSrc1, 1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc2, uSrc2, 2);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT256U,  puSrc3, uSrc3, 3);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            uint8_t bOp4; IEM_OPCODE_GET_NEXT_U8(&bOp4);
+
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx2);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U256_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_FETCH_YREG_U256(uSrc1,      IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_FETCH_YREG_U256(uSrc3,      IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_FETCH_YREG_U256(uSrc3,      bOp4 >> 4); /** @todo Ignore MSB in 32-bit mode. */
+            IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU256, puDst, puSrc1, puSrc2, puSrc3);
+            IEM_MC_STORE_YREG_U256_ZX_VLMAX(   IEM_GET_MODRM_REG(pVCpu, bRm), uDst);
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(4, 2);
+            IEM_MC_LOCAL(RTUINT128U,                uSrc2);
+            IEM_MC_LOCAL(RTGCPTR,                   GCPtrEffSrc);
+            IEM_MC_ARG(PRTUINT128U,                 puDst,         0);
+            IEM_MC_ARG(PCRTUINT128U,                puSrc1,        1);
+            IEM_MC_ARG_LOCAL_REF(PCRTUINT128U,      puSrc2, uSrc2, 2);
+            IEM_MC_ARG(PCRTUINT128U,                puSrc3,        3);
+
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 0);
+            uint8_t bOp4; IEM_OPCODE_GET_NEXT_U8(&bOp4);
+
+            IEMOP_HLP_DONE_VEX_DECODING_EX(fAvx);
+            IEM_MC_MAYBE_RAISE_AVX2_RELATED_XCPT();
+            IEM_MC_PREPARE_AVX_USAGE();
+
+            IEM_MC_FETCH_MEM_U128_NO_AC(uSrc2, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_XREG_U128(puDst,         IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_REF_XREG_U128_CONST(puSrc1,  IEM_GET_EFFECTIVE_VVVV(pVCpu));
+            IEM_MC_REF_XREG_U128_CONST(puSrc3, bOp4 >> 4); /** @todo Ignore MSB in 32-bit mode. */
+            IEM_MC_CALL_VOID_AIMPL_4(pImpl->pfnU128, puDst, puSrc1, puSrc2, puSrc3);
+            IEM_MC_CLEAR_YREG_128_UP(           IEM_GET_MODRM_REG(pVCpu, bRm));
+
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/** Opcode VEX.66.0F3A 0x4a (vex only). */
+FNIEMOP_DEF(iemOp_vblendvps_Vx_Hx_Wx_Lx)
+{
+    //IEMOP_MNEMONIC4(VEX_RVM, VBLENDVPS, vpblendvps, Vx, Hx, Wx, Lx, DISOPTYPE_HARMLESS, 0); @todo
+    IEMOPBLENDOP_INIT_VARS(vblendvps);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Lx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F3A 0x4b (vex only). */
+FNIEMOP_DEF(iemOp_vblendvpd_Vx_Hx_Wx_Lx)
+{
+    //IEMOP_MNEMONIC4(VEX_RVM, VPBLENDVPD, blendvpd, Vx, Hx, Wx, Lx, DISOPTYPE_HARMLESS, 0); @todo
+    IEMOPBLENDOP_INIT_VARS(vblendvpd);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Lx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/** Opcode VEX.66.0F3A 0x4c (vex only). */
+FNIEMOP_DEF(iemOp_vpblendvb_Vx_Hx_Wx_Lx)
+{
+    //IEMOP_MNEMONIC4(VEX_RVM, VPBLENDVB, vpblendvb, Vx, Hx, Wx, Lx, DISOPTYPE_HARMLESS, 0); @todo
+    IEMOPBLENDOP_INIT_VARS(vpblendvb);
+    return FNIEMOP_CALL_1(iemOpCommonAvxAvx2_Vx_Hx_Wx_Lx, IEM_SELECT_HOST_OR_FALLBACK(fAvx2, &s_Host, &s_Fallback));
+}
+
+
+/*  Opcode VEX.66.0F3A 0x4d - invalid */
+/*  Opcode VEX.66.0F3A 0x4e - invalid */
+/*  Opcode VEX.66.0F3A 0x4f - invalid */
+
+
+/*  Opcode VEX.66.0F3A 0x50 - invalid */
+/*  Opcode VEX.66.0F3A 0x51 - invalid */
+/*  Opcode VEX.66.0F3A 0x52 - invalid */
+/*  Opcode VEX.66.0F3A 0x53 - invalid */
+/*  Opcode VEX.66.0F3A 0x54 - invalid */
+/*  Opcode VEX.66.0F3A 0x55 - invalid */
+/*  Opcode VEX.66.0F3A 0x56 - invalid */
+/*  Opcode VEX.66.0F3A 0x57 - invalid */
+/*  Opcode VEX.66.0F3A 0x58 - invalid */
+/*  Opcode VEX.66.0F3A 0x59 - invalid */
+/*  Opcode VEX.66.0F3A 0x5a - invalid */
+/*  Opcode VEX.66.0F3A 0x5b - invalid */
+/** Opcode VEX.66.0F3A 0x5c (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddsubps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x5d (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddsubpd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x5e (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubaddps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x5f (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubaddpd_Vx_Lx_Wx_Hx);
+
+
+/** Opcode VEX.66.0F3A 0x60. */
+FNIEMOP_STUB(iemOp_vpcmpestrm_Vdq_Wdq_Ib);
+/** Opcode VEX.66.0F3A 0x61, */
+FNIEMOP_STUB(iemOp_vpcmpestri_Vdq_Wdq_Ib);
+/** Opcode VEX.66.0F3A 0x62. */
+FNIEMOP_STUB(iemOp_vpcmpistrm_Vdq_Wdq_Ib);
+/** Opcode VEX.66.0F3A 0x63*/
+FNIEMOP_STUB(iemOp_vpcmpistri_Vdq_Wdq_Ib);
+/*  Opcode VEX.66.0F3A 0x64 - invalid */
+/*  Opcode VEX.66.0F3A 0x65 - invalid */
+/*  Opcode VEX.66.0F3A 0x66 - invalid */
+/*  Opcode VEX.66.0F3A 0x67 - invalid */
+/** Opcode VEX.66.0F3A 0x68 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x69 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddpd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6a (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddss_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6b (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmaddsd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6c (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6d (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubpd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6e (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubss_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x6f (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfmsubsd_Vx_Lx_Wx_Hx);
+
+/*  Opcode VEX.66.0F3A 0x70 - invalid */
+/*  Opcode VEX.66.0F3A 0x71 - invalid */
+/*  Opcode VEX.66.0F3A 0x72 - invalid */
+/*  Opcode VEX.66.0F3A 0x73 - invalid */
+/*  Opcode VEX.66.0F3A 0x74 - invalid */
+/*  Opcode VEX.66.0F3A 0x75 - invalid */
+/*  Opcode VEX.66.0F3A 0x76 - invalid */
+/*  Opcode VEX.66.0F3A 0x77 - invalid */
+/** Opcode VEX.66.0F3A 0x78 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmaddps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x79 (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmaddpd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7a (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmaddss_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7b (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmaddsd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7c (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmsubps_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7d (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmsubpd_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7e (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmsubss_Vx_Lx_Wx_Hx);
+/** Opcode VEX.66.0F3A 0x7f (AMD tables only). */
+FNIEMOP_STUB(iemOp_vfnmsubsd_Vx_Lx_Wx_Hx);
+
+/*  Opcodes 0x0f 0x80 thru 0x0f 0xb0 are unused.  */
+
+
+/*  Opcode      0x0f 0xc0 - invalid */
+/*  Opcode      0x0f 0xc1 - invalid */
+/*  Opcode      0x0f 0xc2 - invalid */
+/*  Opcode      0x0f 0xc3 - invalid */
+/*  Opcode      0x0f 0xc4 - invalid */
+/*  Opcode      0x0f 0xc5 - invalid */
+/*  Opcode      0x0f 0xc6 - invalid */
+/*  Opcode      0x0f 0xc7 - invalid */
+/*  Opcode      0x0f 0xc8 - invalid */
+/*  Opcode      0x0f 0xc9 - invalid */
+/*  Opcode      0x0f 0xca - invalid */
+/*  Opcode      0x0f 0xcb - invalid */
+/*  Opcode      0x0f 0xcc */
+FNIEMOP_STUB(iemOp_vsha1rnds4_Vdq_Wdq_Ib);
+/*  Opcode      0x0f 0xcd - invalid */
+/*  Opcode      0x0f 0xce - invalid */
+/*  Opcode      0x0f 0xcf - invalid */
+
+
+/*  Opcode VEX.66.0F3A 0xd0 - invalid */
+/*  Opcode VEX.66.0F3A 0xd1 - invalid */
+/*  Opcode VEX.66.0F3A 0xd2 - invalid */
+/*  Opcode VEX.66.0F3A 0xd3 - invalid */
+/*  Opcode VEX.66.0F3A 0xd4 - invalid */
+/*  Opcode VEX.66.0F3A 0xd5 - invalid */
+/*  Opcode VEX.66.0F3A 0xd6 - invalid */
+/*  Opcode VEX.66.0F3A 0xd7 - invalid */
+/*  Opcode VEX.66.0F3A 0xd8 - invalid */
+/*  Opcode VEX.66.0F3A 0xd9 - invalid */
+/*  Opcode VEX.66.0F3A 0xda - invalid */
+/*  Opcode VEX.66.0F3A 0xdb - invalid */
+/*  Opcode VEX.66.0F3A 0xdc - invalid */
+/*  Opcode VEX.66.0F3A 0xdd - invalid */
+/*  Opcode VEX.66.0F3A 0xde - invalid */
+/*  Opcode VEX.66.0F3A 0xdf - (aeskeygenassist). */
+FNIEMOP_STUB(iemOp_vaeskeygen_Vdq_Wdq_Ib);
+
+
+/** Opcode VEX.F2.0F3A (vex only) */
+FNIEMOP_DEF(iemOp_rorx_Gy_Ey_Ib)
+{
+    IEMOP_MNEMONIC3(VEX_RMI, RORX, rorx, Gy, Ey, Ib, DISOPTYPE_HARMLESS, IEMOPHINT_VEX_L_ZERO | IEMOPHINT_VEX_V_ZERO);
+    if (!IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fBmi2)
+        return iemOp_InvalidNeedRMImm8(pVCpu);
+    uint8_t bRm; IEM_OPCODE_GET_NEXT_U8(&bRm);
+    if (IEM_IS_MODRM_REG_MODE(bRm))
+    {
+        /*
+         * Register, register.
+         */
+        uint8_t bImm8; IEM_OPCODE_GET_NEXT_U8(&bImm8);
+        IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint64_t *,          pDst,    0);
+            IEM_MC_ARG(uint64_t,            uSrc1,   1);
+            IEM_MC_ARG_CONST(uint64_t,      uSrc2,   bImm8, 2);
+            IEM_MC_REF_GREG_U64(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_FETCH_GREG_U64(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_rorx_u64, pDst, uSrc1, uSrc2);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 0);
+            IEM_MC_ARG(uint32_t *,          pDst,    0);
+            IEM_MC_ARG(uint32_t,            uSrc1,   1);
+            IEM_MC_ARG_CONST(uint32_t,      uSrc2,   bImm8, 2);
+            IEM_MC_REF_GREG_U32(pDst,    IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_FETCH_GREG_U32(uSrc1, IEM_GET_MODRM_RM(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_rorx_u32, pDst, uSrc1, uSrc2);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+    else
+    {
+        /*
+         * Register, memory.
+         */
+        if (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_SIZE_REX_W)
+        {
+            IEM_MC_BEGIN(3, 1);
+            IEM_MC_ARG(uint64_t *,          pDst,    0);
+            IEM_MC_ARG(uint64_t,            uSrc1,   1);
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+            uint8_t bImm8; IEM_OPCODE_GET_NEXT_U8(&bImm8);
+            IEM_MC_ARG_CONST(uint64_t,      uSrc2,   bImm8, 2);
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_FETCH_MEM_U64(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_GREG_U64(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_rorx_u64, pDst, uSrc1, uSrc2);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+        else
+        {
+            IEM_MC_BEGIN(3, 1);
+            IEM_MC_ARG(uint32_t *,          pDst,    0);
+            IEM_MC_ARG(uint32_t,            uSrc1,   1);
+            IEM_MC_LOCAL(RTGCPTR,           GCPtrEffSrc);
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffSrc, bRm, 1);
+            uint8_t bImm8; IEM_OPCODE_GET_NEXT_U8(&bImm8);
+            IEM_MC_ARG_CONST(uint32_t,      uSrc2,   bImm8, 2);
+            IEMOP_HLP_DONE_VEX_DECODING_L0_AND_NO_VVVV();
+            IEM_MC_FETCH_MEM_U32(uSrc1, pVCpu->iem.s.iEffSeg, GCPtrEffSrc);
+            IEM_MC_REF_GREG_U32(pDst, IEM_GET_MODRM_REG(pVCpu, bRm));
+            IEM_MC_CALL_VOID_AIMPL_3(iemAImpl_rorx_u32, pDst, uSrc1, uSrc2);
+            IEM_MC_CLEAR_HIGH_GREG_U64_BY_REF(pDst);
+            IEM_MC_ADVANCE_RIP_AND_FINISH();
+            IEM_MC_END();
+        }
+    }
+}
+
+
+/**
+ * VEX opcode map \#3.
+ *
+ * @sa      g_apfnThreeByte0f3a
+ */
+IEM_STATIC const PFNIEMOP g_apfnVexMap3[] =
+{
+    /*          no prefix,                  066h prefix                 f3h prefix,                 f2h prefix */
+    /* 0x00 */  iemOp_InvalidNeedRMImm8,    iemOp_vpermq_Vqq_Wqq_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x01 */  iemOp_InvalidNeedRMImm8,    iemOp_vpermqd_Vqq_Wqq_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x02 */  iemOp_InvalidNeedRMImm8,    iemOp_vpblendd_Vx_Wx_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x03 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x04 */  iemOp_InvalidNeedRMImm8,    iemOp_vpermilps_Vx_Wx_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x05 */  iemOp_InvalidNeedRMImm8,    iemOp_vpermilpd_Vx_Wx_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x06 */  iemOp_InvalidNeedRMImm8,    iemOp_vperm2f128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x07 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x08 */  iemOp_InvalidNeedRMImm8,    iemOp_vroundps_Vx_Wx_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x09 */  iemOp_InvalidNeedRMImm8,    iemOp_vroundpd_Vx_Wx_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0a */  iemOp_InvalidNeedRMImm8,    iemOp_vroundss_Vss_Wss_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0b */  iemOp_InvalidNeedRMImm8,    iemOp_vroundsd_Vsd_Wsd_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0c */  iemOp_InvalidNeedRMImm8,    iemOp_vblendps_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0d */  iemOp_InvalidNeedRMImm8,    iemOp_vblendpd_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0e */  iemOp_InvalidNeedRMImm8,    iemOp_vpblendw_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x0f */  iemOp_InvalidNeedRMImm8,    iemOp_vpalignr_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+
+    /* 0x10 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x11 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x12 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x13 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x14 */  iemOp_InvalidNeedRMImm8,    iemOp_vpextrb_RdMb_Vdq_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x15 */  iemOp_InvalidNeedRMImm8,    iemOp_vpextrw_RdMw_Vdq_Ib,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x16 */  iemOp_InvalidNeedRMImm8,    iemOp_vpextrd_q_RdMw_Vdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x17 */  iemOp_InvalidNeedRMImm8,    iemOp_vextractps_Ed_Vdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x18 */  iemOp_InvalidNeedRMImm8,    iemOp_vinsertf128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x19 */  iemOp_InvalidNeedRMImm8,    iemOp_vextractf128_Wdq_Vqq_Ib, iemOp_InvalidNeedRMImm8,  iemOp_InvalidNeedRMImm8,
+    /* 0x1a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1d */  iemOp_InvalidNeedRMImm8,    iemOp_vcvtps2ph_Wx_Vx_Ib,   iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x1e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x1f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x20 */  iemOp_InvalidNeedRMImm8,    iemOp_vpinsrb_Vdq_Hdq_RyMb_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x21 */  iemOp_InvalidNeedRMImm8,    iemOp_vinsertps_Vdq_Hdq_UdqMd_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x22 */  iemOp_InvalidNeedRMImm8,    iemOp_vpinsrd_q_Vdq_Hdq_Ey_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x23 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x24 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x25 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x26 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x27 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x28 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x29 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x2f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x30 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x31 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x32 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x33 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x34 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x35 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x36 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x37 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x38 */  iemOp_InvalidNeedRMImm8,    iemOp_vinserti128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x39 */  iemOp_InvalidNeedRMImm8,    iemOp_vextracti128_Wdq_Vqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x3a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x3f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x40 */  iemOp_InvalidNeedRMImm8,    iemOp_vdpps_Vx_Hx_Wx_Ib,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x41 */  iemOp_InvalidNeedRMImm8,    iemOp_vdppd_Vdq_Hdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x42 */  iemOp_InvalidNeedRMImm8,    iemOp_vmpsadbw_Vx_Hx_Wx_Ib, iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0x43 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x44 */  iemOp_InvalidNeedRMImm8,    iemOp_vpclmulqdq_Vdq_Hdq_Wdq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x45 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x46 */  iemOp_InvalidNeedRMImm8,    iemOp_vperm2i128_Vqq_Hqq_Wqq_Ib, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x47 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x48 */  iemOp_InvalidNeedRMImm8,    iemOp_vperlmilzz2ps_Vx_Hx_Wp_Lx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x49 */  iemOp_InvalidNeedRMImm8,    iemOp_vperlmilzz2pd_Vx_Hx_Wp_Lx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x4a */  iemOp_InvalidNeedRMImm8,    iemOp_vblendvps_Vx_Hx_Wx_Lx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x4b */  iemOp_InvalidNeedRMImm8,    iemOp_vblendvpd_Vx_Hx_Wx_Lx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x4c */  iemOp_InvalidNeedRMImm8,    iemOp_vpblendvb_Vx_Hx_Wx_Lx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x4d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x4f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x50 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x51 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x52 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x53 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x54 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x55 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x56 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x57 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x58 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x59 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x5c */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddsubps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x5d */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddsubpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x5e */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubaddps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x5f */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubaddpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+
+    /* 0x60 */  iemOp_InvalidNeedRMImm8,    iemOp_vpcmpestrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x61 */  iemOp_InvalidNeedRMImm8,    iemOp_vpcmpestri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x62 */  iemOp_InvalidNeedRMImm8,    iemOp_vpcmpistrm_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x63 */  iemOp_InvalidNeedRMImm8,    iemOp_vpcmpistri_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,   iemOp_InvalidNeedRMImm8,
+    /* 0x64 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x65 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x66 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x67 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x68 */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x69 */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6a */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6b */  iemOp_InvalidNeedRMImm8,    iemOp_vfmaddsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6c */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6d */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6e */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x6f */  iemOp_InvalidNeedRMImm8,    iemOp_vfmsubsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+
+    /* 0x70 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x71 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x72 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x73 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x74 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x75 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x76 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x77 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x78 */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmaddps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x79 */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmaddpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7a */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmaddss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7b */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmaddsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7c */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmsubps_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7d */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmsubpd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7e */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmsubss_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+    /* 0x7f */  iemOp_InvalidNeedRMImm8,    iemOp_vfnmsubsd_Vx_Lx_Wx_Hx, iemOp_InvalidNeedRMImm8, iemOp_InvalidNeedRMImm8,
+
+    /* 0x80 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x81 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x82 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x83 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x84 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x85 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x86 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x87 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x88 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x89 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x8f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0x90 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x91 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x92 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x93 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x94 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x95 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x96 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x97 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x98 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x99 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9a */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9b */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9c */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9d */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9e */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0x9f */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xa0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xa9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xaa */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xab */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xac */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xad */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xae */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xaf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xb0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xb9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xba */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbe */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xbf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xc0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xc9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xca */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcc */  iemOp_vsha1rnds4_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+    /* 0xcd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xce */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xcf */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xd0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xd9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xda */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xde */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xdf */  iemOp_vaeskeygen_Vdq_Wdq_Ib, iemOp_InvalidNeedRMImm8,  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,
+
+    /* 0xe0 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xe9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xea */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xeb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xec */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xed */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xee */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xef */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+
+    /* 0xf0 */  iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,    iemOp_InvalidNeedRMImm8,    iemOp_rorx_Gy_Ey_Ib,
+    /* 0xf1 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf2 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf3 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf4 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf5 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf6 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf7 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf8 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xf9 */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfa */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfb */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfc */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfd */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xfe */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+    /* 0xff */  IEMOP_X4(iemOp_InvalidNeedRMImm8),
+};
+AssertCompile(RT_ELEMENTS(g_apfnVexMap3) == 1024);
+
+/** @} */
+
diff --git a/src/VBox/VMM/VMMAll/IOMAll.cpp b/src/VBox/VMM/VMMAll/IOMAll.cpp
new file mode 100644
index 00000000..522156b5
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IOMAll.cpp
@@ -0,0 +1,618 @@
+/* $Id: IOMAll.cpp $ */
+/** @file
+ * IOM - Input / Output Monitor - Any Context.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM_IOPORT
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/param.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include "IOMInline.h"
+
+
+/**
+ * Reads an I/O port register.
+ *
+ * @returns Strict VBox status code. Informational status codes other than the one documented
+ *          here are to be treated as internal failure. Use IOM_SUCCESS() to check for success.
+ * @retval  VINF_SUCCESS                Success.
+ * @retval  VINF_EM_FIRST-VINF_EM_LAST  Success with some exceptions (see IOM_SUCCESS()), the
+ *                                      status code must be passed on to EM.
+ * @retval  VINF_IOM_R3_IOPORT_READ     Defer the read to ring-3. (R0/RC only)
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   Port        The port to read.
+ * @param   pu32Value   Where to store the value read.
+ * @param   cbValue     The size of the register to read in bytes. 1, 2 or 4 bytes.
+ */
+VMMDECL(VBOXSTRICTRC) IOMIOPortRead(PVMCC pVM, PVMCPU pVCpu, RTIOPORT Port, uint32_t *pu32Value, size_t cbValue)
+{
+    STAM_COUNTER_INC(&pVM->iom.s.StatIoPortIn);
+    Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0);
+
+/** @todo should initialize *pu32Value here because it can happen that some
+ *        handle is buggy and doesn't handle all cases. */
+
+    /* For lookups we need to share lock IOM. */
+    int rc2 = IOM_LOCK_SHARED(pVM);
+    if (RT_SUCCESS(rc2))
+    { /* likely */ }
+#ifndef IN_RING3
+    else if (rc2 == VERR_SEM_BUSY)
+        return VINF_IOM_R3_IOPORT_READ;
+#endif
+    else
+        AssertMsgFailedReturn(("rc2=%Rrc\n", rc2), rc2);
+
+    /*
+     * Get the entry for the current context.
+     */
+    uint16_t offPort;
+    CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, Port, &offPort, &pVCpu->iom.s.idxIoPortLastRead);
+    if (pRegEntry)
+    {
+#ifdef VBOX_WITH_STATISTICS
+        PIOMIOPORTSTATSENTRY  pStats    = iomIoPortGetStats(pVM, pRegEntry, offPort);
+#endif
+
+        /*
+         * Found an entry, get the data so we can leave the IOM lock.
+         */
+        uint16_t const    fFlags        = pRegEntry->fFlags;
+        PFNIOMIOPORTNEWIN pfnInCallback = pRegEntry->pfnInCallback;
+        PPDMDEVINS        pDevIns       = pRegEntry->pDevIns;
+#ifndef IN_RING3
+        if (   pfnInCallback
+            && pDevIns
+            && pRegEntry->cPorts > 0)
+        { /* likely */ }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->InRZToR3);
+            IOM_UNLOCK_SHARED(pVM);
+            return VINF_IOM_R3_IOPORT_READ;
+        }
+#endif
+        void           *pvUser       = pRegEntry->pvUser;
+        IOM_UNLOCK_SHARED(pVM);
+        AssertPtr(pDevIns);
+        AssertPtr(pfnInCallback);
+
+        /*
+         * Call the device.
+         */
+        VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_READ);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfIn), a);
+            rcStrict = pfnInCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? Port : offPort, pu32Value, (unsigned)cbValue);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfIn), a);
+            PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
+
+#ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_IOPORT_READ)
+                STAM_COUNTER_INC(&pStats->InRZToR3);
+            else
+#endif
+            {
+                STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(In));
+                STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total);
+                if (rcStrict == VERR_IOM_IOPORT_UNUSED)
+                {
+                    /* make return value */
+                    rcStrict = VINF_SUCCESS;
+                    switch (cbValue)
+                    {
+                        case 1: *(uint8_t  *)pu32Value = 0xff; break;
+                        case 2: *(uint16_t *)pu32Value = 0xffff; break;
+                        case 4: *(uint32_t *)pu32Value = UINT32_C(0xffffffff); break;
+                        default:
+                            AssertMsgFailedReturn(("Invalid I/O port size %d. Port=%d\n", cbValue, Port), VERR_IOM_INVALID_IOPORT_SIZE);
+                    }
+                }
+           }
+            Log3(("IOMIOPortRead: Port=%RTiop *pu32=%08RX32 cb=%d rc=%Rrc\n", Port, *pu32Value, cbValue, VBOXSTRICTRC_VAL(rcStrict)));
+            STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total);
+        }
+        else
+            STAM_COUNTER_INC(&pStats->InRZToR3);
+        return rcStrict;
+    }
+
+    /*
+     * Ok, no handler for this port.
+     */
+    IOM_UNLOCK_SHARED(pVM);
+    switch (cbValue)
+    {
+        case 1: *(uint8_t  *)pu32Value = 0xff; break;
+        case 2: *(uint16_t *)pu32Value = 0xffff; break;
+        case 4: *(uint32_t *)pu32Value = UINT32_C(0xffffffff); break;
+        default:
+            AssertMsgFailedReturn(("Invalid I/O port size %d. Port=%d\n", cbValue, Port), VERR_IOM_INVALID_IOPORT_SIZE);
+    }
+    Log3(("IOMIOPortRead: Port=%RTiop *pu32=%08RX32 cb=%d rc=VINF_SUCCESS\n", Port, *pu32Value, cbValue));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Reads the string buffer of an I/O port register.
+ *
+ * @returns Strict VBox status code. Informational status codes other than the one documented
+ *          here are to be treated as internal failure. Use IOM_SUCCESS() to check for success.
+ * @retval  VINF_SUCCESS                Success or no string I/O callback in
+ *                                      this context.
+ * @retval  VINF_EM_FIRST-VINF_EM_LAST  Success with some exceptions (see IOM_SUCCESS()), the
+ *                                      status code must be passed on to EM.
+ * @retval  VINF_IOM_R3_IOPORT_READ     Defer the read to ring-3. (R0/RC only)
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uPort       The port to read.
+ * @param   pvDst       Pointer to the destination buffer.
+ * @param   pcTransfers Pointer to the number of transfer units to read, on return remaining transfer units.
+ * @param   cb          Size of the transfer unit (1, 2 or 4 bytes).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IOMIOPortReadString(PVMCC pVM, PVMCPU pVCpu, RTIOPORT uPort,
+                                               void *pvDst, uint32_t *pcTransfers, unsigned cb)
+{
+    STAM_COUNTER_INC(&pVM->iom.s.StatIoPortInS);
+    Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0);
+
+    /* For lookups we need to share lock IOM. */
+    int rc2 = IOM_LOCK_SHARED(pVM);
+    if (RT_SUCCESS(rc2))
+    { /* likely */ }
+#ifndef IN_RING3
+    else if (rc2 == VERR_SEM_BUSY)
+        return VINF_IOM_R3_IOPORT_READ;
+#endif
+    else
+        AssertMsgFailedReturn(("rc2=%Rrc\n", rc2), rc2);
+
+    const uint32_t cRequestedTransfers = *pcTransfers;
+    Assert(cRequestedTransfers > 0);
+
+    /*
+     * Get the entry for the current context.
+     */
+    uint16_t offPort;
+    CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, uPort, &offPort, &pVCpu->iom.s.idxIoPortLastReadStr);
+    if (pRegEntry)
+    {
+#ifdef VBOX_WITH_STATISTICS
+        PIOMIOPORTSTATSENTRY  pStats    = iomIoPortGetStats(pVM, pRegEntry, offPort);
+#endif
+
+        /*
+         * Found an entry, get the data so we can leave the IOM lock.
+         */
+        uint16_t const          fFlags           = pRegEntry->fFlags;
+        PFNIOMIOPORTNEWINSTRING pfnInStrCallback = pRegEntry->pfnInStrCallback;
+        PFNIOMIOPORTNEWIN       pfnInCallback    = pRegEntry->pfnInCallback;
+        PPDMDEVINS              pDevIns          = pRegEntry->pDevIns;
+#ifndef IN_RING3
+        if (   pfnInCallback
+            && pDevIns
+            && pRegEntry->cPorts > 0)
+        { /* likely */ }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->InRZToR3);
+            IOM_UNLOCK_SHARED(pVM);
+            return VINF_IOM_R3_IOPORT_READ;
+        }
+#endif
+        void           *pvUser       = pRegEntry->pvUser;
+        IOM_UNLOCK_SHARED(pVM);
+        AssertPtr(pDevIns);
+        AssertPtr(pfnInCallback);
+
+        /*
+         * Call the device.
+         */
+        VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_READ);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /*
+             * First using the string I/O callback.
+             */
+            if (pfnInStrCallback)
+            {
+                STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfIn), a);
+                rcStrict = pfnInStrCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort,
+                                            (uint8_t *)pvDst, pcTransfers, cb);
+                STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfIn), a);
+            }
+
+            /*
+             * Then doing the single I/O fallback.
+             */
+            if (   *pcTransfers > 0
+                && rcStrict == VINF_SUCCESS)
+            {
+                pvDst = (uint8_t *)pvDst + (cRequestedTransfers - *pcTransfers) * cb;
+                do
+                {
+                    uint32_t u32Value = 0;
+                    STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfIn), a);
+                    rcStrict = pfnInCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort, &u32Value, cb);
+                    STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfIn), a);
+                    if (rcStrict == VERR_IOM_IOPORT_UNUSED)
+                    {
+                        u32Value = UINT32_MAX;
+                        rcStrict = VINF_SUCCESS;
+                    }
+                    if (IOM_SUCCESS(rcStrict))
+                    {
+                        switch (cb)
+                        {
+                            case 4: *(uint32_t *)pvDst =           u32Value; pvDst = (uint8_t *)pvDst + 4; break;
+                            case 2: *(uint16_t *)pvDst = (uint16_t)u32Value; pvDst = (uint8_t *)pvDst + 2; break;
+                            case 1: *(uint8_t  *)pvDst = (uint8_t )u32Value; pvDst = (uint8_t *)pvDst + 1; break;
+                            default: AssertFailed();
+                        }
+                        *pcTransfers -= 1;
+                    }
+                } while (   *pcTransfers > 0
+                         && rcStrict == VINF_SUCCESS);
+            }
+            PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
+
+#ifdef VBOX_WITH_STATISTICS
+# ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_IOPORT_READ)
+                STAM_COUNTER_INC(&pStats->InRZToR3);
+            else
+# endif
+            {
+                STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(In));
+                STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total);
+            }
+#endif
+            Log3(("IOMIOPortReadStr: uPort=%RTiop pvDst=%p pcTransfer=%p:{%#x->%#x} cb=%d rc=%Rrc\n",
+                  uPort, pvDst, pcTransfers, cRequestedTransfers, *pcTransfers, cb, VBOXSTRICTRC_VAL(rcStrict)));
+        }
+#ifndef IN_RING3
+        else
+            STAM_COUNTER_INC(&pStats->InRZToR3);
+#endif
+        return rcStrict;
+    }
+
+    /*
+     * Ok, no handler for this port.
+     */
+    IOM_UNLOCK_SHARED(pVM);
+    *pcTransfers = 0;
+    memset(pvDst, 0xff, cRequestedTransfers * cb);
+    Log3(("IOMIOPortReadStr: uPort=%RTiop (unused) pvDst=%p pcTransfer=%p:{%#x->%#x} cb=%d rc=VINF_SUCCESS\n",
+          uPort, pvDst, pcTransfers, cRequestedTransfers, *pcTransfers, cb));
+    return VINF_SUCCESS;
+}
+
+
+#ifndef IN_RING3
+/**
+ * Defers a pending I/O port write to ring-3.
+ *
+ * @returns VINF_IOM_R3_IOPORT_COMMIT_WRITE
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   Port        The port to write to.
+ * @param   u32Value    The value to write.
+ * @param   cbValue     The size of the value (1, 2, 4).
+ */
+static VBOXSTRICTRC iomIOPortRing3WritePending(PVMCPU pVCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
+{
+    Log5(("iomIOPortRing3WritePending: %#x LB %u -> %RTiop\n", u32Value, cbValue, Port));
+    AssertReturn(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0, VERR_IOM_IOPORT_IPE_1);
+    pVCpu->iom.s.PendingIOPortWrite.IOPort   = Port;
+    pVCpu->iom.s.PendingIOPortWrite.u32Value = u32Value;
+    pVCpu->iom.s.PendingIOPortWrite.cbValue  = (uint32_t)cbValue;
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
+    return VINF_IOM_R3_IOPORT_COMMIT_WRITE;
+}
+#endif
+
+
+/**
+ * Writes to an I/O port register.
+ *
+ * @returns Strict VBox status code. Informational status codes other than the one documented
+ *          here are to be treated as internal failure. Use IOM_SUCCESS() to check for success.
+ * @retval  VINF_SUCCESS                Success.
+ * @retval  VINF_EM_FIRST-VINF_EM_LAST  Success with some exceptions (see IOM_SUCCESS()), the
+ *                                      status code must be passed on to EM.
+ * @retval  VINF_IOM_R3_IOPORT_WRITE    Defer the write to ring-3. (R0/RC only)
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   Port        The port to write to.
+ * @param   u32Value    The value to write.
+ * @param   cbValue     The size of the register to read in bytes. 1, 2 or 4 bytes.
+ */
+VMMDECL(VBOXSTRICTRC) IOMIOPortWrite(PVMCC pVM, PVMCPU pVCpu, RTIOPORT Port, uint32_t u32Value, size_t cbValue)
+{
+    STAM_COUNTER_INC(&pVM->iom.s.StatIoPortOut);
+#ifndef IN_RING3
+    Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0);
+#endif
+
+    /* For lookups we need to share lock IOM. */
+    int rc2 = IOM_LOCK_SHARED(pVM);
+    if (RT_SUCCESS(rc2))
+    { /* likely */ }
+#ifndef IN_RING3
+    else if (rc2 == VERR_SEM_BUSY)
+        return iomIOPortRing3WritePending(pVCpu, Port, u32Value, cbValue);
+#endif
+    else
+        AssertMsgFailedReturn(("rc2=%Rrc\n", rc2), rc2);
+
+    /*
+     * Get the entry for the current context.
+     */
+    uint16_t offPort;
+    CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, Port, &offPort, &pVCpu->iom.s.idxIoPortLastWrite);
+    if (pRegEntry)
+    {
+#ifdef VBOX_WITH_STATISTICS
+        PIOMIOPORTSTATSENTRY  pStats    = iomIoPortGetStats(pVM, pRegEntry, offPort);
+#endif
+
+        /*
+         * Found an entry, get the data so we can leave the IOM lock.
+         */
+        uint16_t const     fFlags           = pRegEntry->fFlags;
+        PFNIOMIOPORTNEWOUT pfnOutCallback   = pRegEntry->pfnOutCallback;
+        PPDMDEVINS         pDevIns          = pRegEntry->pDevIns;
+#ifndef IN_RING3
+        if (   pfnOutCallback
+            && pDevIns
+            && pRegEntry->cPorts > 0)
+        { /* likely */ }
+        else
+        {
+            IOM_UNLOCK_SHARED(pVM);
+            STAM_COUNTER_INC(&pStats->OutRZToR3);
+            return iomIOPortRing3WritePending(pVCpu, Port, u32Value, cbValue);
+        }
+#endif
+        void           *pvUser       = pRegEntry->pvUser;
+        IOM_UNLOCK_SHARED(pVM);
+        AssertPtr(pDevIns);
+        AssertPtr(pfnOutCallback);
+
+        /*
+         * Call the device.
+         */
+        VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_WRITE);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfOut), a);
+            rcStrict = pfnOutCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? Port : offPort, u32Value, (unsigned)cbValue);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfOut), a);
+
+            PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
+
+#ifdef VBOX_WITH_STATISTICS
+# ifndef IN_RING3
+            if (rcStrict != VINF_IOM_R3_IOPORT_WRITE)
+# endif
+            {
+                STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(Out));
+                STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total);
+            }
+#endif
+            Log3(("IOMIOPortWrite: Port=%RTiop u32=%08RX32 cb=%d rc=%Rrc\n", Port, u32Value, cbValue, VBOXSTRICTRC_VAL(rcStrict)));
+        }
+#ifndef IN_RING3
+        if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
+        {
+            STAM_COUNTER_INC(&pStats->OutRZToR3);
+            return iomIOPortRing3WritePending(pVCpu, Port, u32Value, cbValue);
+        }
+#endif
+        return rcStrict;
+    }
+
+    /*
+     * Ok, no handler for that port.
+     */
+    IOM_UNLOCK_SHARED(pVM);
+    Log3(("IOMIOPortWrite: Port=%RTiop u32=%08RX32 cb=%d nop\n", Port, u32Value, cbValue));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Writes the string buffer of an I/O port register.
+ *
+ * @returns Strict VBox status code. Informational status codes other than the one documented
+ *          here are to be treated as internal failure. Use IOM_SUCCESS() to check for success.
+ * @retval  VINF_SUCCESS                Success or no string I/O callback in
+ *                                      this context.
+ * @retval  VINF_EM_FIRST-VINF_EM_LAST  Success with some exceptions (see IOM_SUCCESS()), the
+ *                                      status code must be passed on to EM.
+ * @retval  VINF_IOM_R3_IOPORT_WRITE    Defer the write to ring-3. (R0/RC only)
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uPort       The port to write to.
+ * @param   pvSrc       The guest page to read from.
+ * @param   pcTransfers Pointer to the number of transfer units to write, on
+ *                      return remaining transfer units.
+ * @param   cb          Size of the transfer unit (1, 2 or 4 bytes).
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IOMIOPortWriteString(PVMCC pVM, PVMCPU pVCpu, RTIOPORT uPort, void const *pvSrc,
+                                                uint32_t *pcTransfers, unsigned cb)
+{
+    STAM_COUNTER_INC(&pVM->iom.s.StatIoPortOutS);
+    Assert(pVCpu->iom.s.PendingIOPortWrite.cbValue == 0);
+    Assert(cb == 1 || cb == 2 || cb == 4);
+
+    /* Take the IOM lock before performing any device I/O. */
+    int rc2 = IOM_LOCK_SHARED(pVM);
+    if (RT_SUCCESS(rc2))
+    { /* likely */ }
+#ifndef IN_RING3
+    else if (rc2 == VERR_SEM_BUSY)
+        return VINF_IOM_R3_IOPORT_WRITE;
+#endif
+    else
+        AssertMsgFailedReturn(("rc2=%Rrc\n", rc2), rc2);
+
+    const uint32_t cRequestedTransfers = *pcTransfers;
+    Assert(cRequestedTransfers > 0);
+
+    /*
+     * Get the entry for the current context.
+     */
+    uint16_t offPort;
+    CTX_SUFF(PIOMIOPORTENTRY) pRegEntry = iomIoPortGetEntry(pVM, uPort, &offPort, &pVCpu->iom.s.idxIoPortLastWriteStr);
+    if (pRegEntry)
+    {
+#ifdef VBOX_WITH_STATISTICS
+        PIOMIOPORTSTATSENTRY  pStats    = iomIoPortGetStats(pVM, pRegEntry, offPort);
+#endif
+
+        /*
+         * Found an entry, get the data so we can leave the IOM lock.
+         */
+        uint16_t const             fFlags            = pRegEntry->fFlags;
+        PFNIOMIOPORTNEWOUTSTRING   pfnOutStrCallback = pRegEntry->pfnOutStrCallback;
+        PFNIOMIOPORTNEWOUT         pfnOutCallback    = pRegEntry->pfnOutCallback;
+        PPDMDEVINS                 pDevIns           = pRegEntry->pDevIns;
+#ifndef IN_RING3
+        if (   pfnOutCallback
+            && pDevIns
+            && pRegEntry->cPorts > 0)
+        { /* likely */ }
+        else
+        {
+            IOM_UNLOCK_SHARED(pVM);
+            STAM_COUNTER_INC(&pStats->OutRZToR3);
+            return VINF_IOM_R3_IOPORT_WRITE;
+        }
+#endif
+        void           *pvUser       = pRegEntry->pvUser;
+        IOM_UNLOCK_SHARED(pVM);
+        AssertPtr(pDevIns);
+        AssertPtr(pfnOutCallback);
+
+        /*
+         * Call the device.
+         */
+        VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_IOPORT_WRITE);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /*
+             * First using string I/O if possible.
+             */
+            if (pfnOutStrCallback)
+            {
+                STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfOut), a);
+                rcStrict = pfnOutStrCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort,
+                                             (uint8_t const *)pvSrc, pcTransfers, cb);
+                STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfOut), a);
+            }
+
+            /*
+             * Then doing the single I/O fallback.
+             */
+            if (   *pcTransfers > 0
+                && rcStrict == VINF_SUCCESS)
+            {
+                pvSrc = (uint8_t *)pvSrc + (cRequestedTransfers - *pcTransfers) * cb;
+                do
+                {
+                    uint32_t u32Value;
+                    switch (cb)
+                    {
+                        case 4: u32Value = *(uint32_t *)pvSrc; pvSrc = (uint8_t const *)pvSrc + 4; break;
+                        case 2: u32Value = *(uint16_t *)pvSrc; pvSrc = (uint8_t const *)pvSrc + 2; break;
+                        case 1: u32Value = *(uint8_t  *)pvSrc; pvSrc = (uint8_t const *)pvSrc + 1; break;
+                        default: AssertFailed(); u32Value = UINT32_MAX;
+                    }
+                    STAM_PROFILE_START(&pStats->CTX_SUFF_Z(ProfOut), a);
+                    rcStrict = pfnOutCallback(pDevIns, pvUser, fFlags & IOM_IOPORT_F_ABS ? uPort : offPort, u32Value, cb);
+                    STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfOut), a);
+                    if (IOM_SUCCESS(rcStrict))
+                        *pcTransfers -= 1;
+                } while (   *pcTransfers > 0
+                         && rcStrict == VINF_SUCCESS);
+            }
+
+            PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
+
+#ifdef VBOX_WITH_STATISTICS
+# ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
+                STAM_COUNTER_INC(&pStats->OutRZToR3);
+            else
+# endif
+            {
+                STAM_COUNTER_INC(&pStats->CTX_SUFF_Z(Out));
+                STAM_COUNTER_INC(&iomIoPortGetStats(pVM, pRegEntry, 0)->Total);
+            }
+#endif
+            Log3(("IOMIOPortWriteStr: uPort=%RTiop pvSrc=%p pcTransfer=%p:{%#x->%#x} cb=%d rcStrict=%Rrc\n",
+                  uPort, pvSrc, pcTransfers, cRequestedTransfers, *pcTransfers, cb, VBOXSTRICTRC_VAL(rcStrict)));
+        }
+#ifndef IN_RING3
+        else
+            STAM_COUNTER_INC(&pStats->OutRZToR3);
+#endif
+        return rcStrict;
+    }
+
+    /*
+     * Ok, no handler for this port.
+     */
+    IOM_UNLOCK_SHARED(pVM);
+    *pcTransfers = 0;
+    Log3(("IOMIOPortWriteStr: uPort=%RTiop (unused) pvSrc=%p pcTransfer=%p:{%#x->%#x} cb=%d rc=VINF_SUCCESS\n",
+          uPort, pvSrc, pcTransfers, cRequestedTransfers, *pcTransfers, cb));
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp b/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp
new file mode 100644
index 00000000..c737bf1f
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/IOMAllMmioNew.cpp
@@ -0,0 +1,1320 @@
+/* $Id: IOMAllMmioNew.cpp $ */
+/** @file
+ * IOM - Input / Output Monitor - Any Context, MMIO & String I/O.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_IOM_MMIO
+#define VMCPU_INCL_CPUM_GST_CTX
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/iem.h>
+#include "IOMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/hm.h>
+#include "IOMInline.h"
+
+#include <VBox/dis.h>
+#include <VBox/disopcode.h>
+#include <VBox/vmm/pdmdev.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** @def IOM_MMIO_STATS_COMMA_DECL
+ * Parameter list declaration for statistics entry pointer. */
+/** @def IOM_MMIO_STATS_COMMA_ARG
+ * Statistics entry pointer argument. */
+#if defined(VBOX_WITH_STATISTICS) || defined(DOXYGEN_RUNNING)
+# define IOM_MMIO_STATS_COMMA_DECL , PIOMMMIOSTATSENTRY pStats
+# define IOM_MMIO_STATS_COMMA_ARG  , pStats
+#else
+# define IOM_MMIO_STATS_COMMA_DECL
+# define IOM_MMIO_STATS_COMMA_ARG
+#endif
+
+
+
+#ifndef IN_RING3
+/**
+ * Defers a pending MMIO write to ring-3.
+ *
+ * @returns VINF_IOM_R3_MMIO_COMMIT_WRITE
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhys      The write address.
+ * @param   pvBuf       The bytes being written.
+ * @param   cbBuf       How many bytes.
+ * @param   idxRegEntry The MMIO registration index (handle) if available,
+ *                      otherwise UINT32_MAX.
+ */
+static VBOXSTRICTRC iomMmioRing3WritePending(PVMCPU pVCpu, RTGCPHYS GCPhys, void const *pvBuf, size_t cbBuf,
+                                             uint32_t idxRegEntry)
+{
+    Log5(("iomMmioRing3WritePending: %RGp LB %#x (idx=%#x)\n", GCPhys, cbBuf, idxRegEntry));
+    if (pVCpu->iom.s.PendingMmioWrite.cbValue == 0)
+    {
+        pVCpu->iom.s.PendingMmioWrite.GCPhys            = GCPhys;
+        AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
+        pVCpu->iom.s.PendingMmioWrite.cbValue           = (uint32_t)cbBuf;
+        pVCpu->iom.s.PendingMmioWrite.idxMmioRegionHint = idxRegEntry;
+        memcpy(pVCpu->iom.s.PendingMmioWrite.abValue, pvBuf, cbBuf);
+    }
+    else
+    {
+        /*
+         * Join with pending if adjecent.
+         *
+         * This may happen if the stack overflows into MMIO territory and RSP/ESP/SP
+         * isn't aligned. IEM will bounce buffer the access and do one write for each
+         * page.  We get here when the 2nd page part is written.
+         */
+        uint32_t const cbOldValue = pVCpu->iom.s.PendingMmioWrite.cbValue;
+        AssertMsgReturn(GCPhys == pVCpu->iom.s.PendingMmioWrite.GCPhys + cbOldValue,
+                        ("pending %RGp LB %#x; incoming %RGp LB %#x\n",
+                         pVCpu->iom.s.PendingMmioWrite.GCPhys, cbOldValue, GCPhys, cbBuf),
+                        VERR_IOM_MMIO_IPE_1);
+        AssertReturn(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue) - cbOldValue, VERR_IOM_MMIO_IPE_2);
+        pVCpu->iom.s.PendingMmioWrite.cbValue = cbOldValue + (uint32_t)cbBuf;
+        memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[cbOldValue], pvBuf, cbBuf);
+    }
+
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
+    return VINF_IOM_R3_MMIO_COMMIT_WRITE;
+}
+#endif
+
+
+/**
+ * Deals with complicated MMIO writes.
+ *
+ * Complicated means unaligned or non-dword/qword sized accesses depending on
+ * the MMIO region's access mode flags.
+ *
+ * @returns Strict VBox status code. Any EM scheduling status code,
+ *          VINF_IOM_R3_MMIO_WRITE, VINF_IOM_R3_MMIO_READ_WRITE or
+ *          VINF_IOM_R3_MMIO_READ may be returned.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pRegEntry   The MMIO entry for the current context.
+ * @param   GCPhys      The physical address to start writing.
+ * @param   offRegion   MMIO region offset corresponding to @a GCPhys.
+ * @param   pvValue     Where to store the value.
+ * @param   cbValue     The size of the value to write.
+ * @param   pStats      Pointer to the statistics (never NULL).
+ */
+static VBOXSTRICTRC iomMmioDoComplicatedWrite(PVM pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
+                                              RTGCPHYS GCPhys, RTGCPHYS offRegion,
+                                              void const *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    AssertReturn(   (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) != IOMMMIO_FLAGS_WRITE_PASSTHRU
+                 && (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) <= IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING,
+                 VERR_IOM_MMIO_IPE_1);
+    AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
+    RTGCPHYS const GCPhysStart  = GCPhys; NOREF(GCPhysStart);
+    bool const     fReadMissing = (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_READ_MISSING
+                               || (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_DWORD_QWORD_READ_MISSING;
+    RT_NOREF_PV(pVCpu); /* ring-3 */
+
+    /*
+     * Do debug stop if requested.
+     */
+    VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
+#ifdef VBOX_STRICT
+    if (!(pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_WRITE))
+    { /* likely */ }
+    else
+    {
+# ifdef IN_RING3
+        LogRel(("IOM: Complicated write %#x byte at %RGp to %s, initiating debugger intervention\n", cbValue, GCPhys,
+                R3STRING(pRegEntry->pszDesc)));
+        rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
+                            "Complicated write %#x byte at %RGp to %s\n", cbValue, GCPhys, pRegEntry->pszDesc);
+        if (rc == VERR_DBGF_NOT_ATTACHED)
+            rc = VINF_SUCCESS;
+# else
+        return VINF_IOM_R3_MMIO_WRITE;
+# endif
+    }
+#endif
+
+    STAM_COUNTER_INC(&pStats->ComplicatedWrites);
+
+    /*
+     * Check if we should ignore the write.
+     */
+    if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD)
+    {
+        Assert(cbValue != 4 || (GCPhys & 3));
+        return VINF_SUCCESS;
+    }
+    if ((pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_ONLY_DWORD_QWORD)
+    {
+        Assert((cbValue != 4 && cbValue != 8) || (GCPhys & (cbValue - 1)));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Split and conquer.
+     */
+    for (;;)
+    {
+        unsigned const  offAccess  = GCPhys & 3;
+        unsigned        cbThisPart = 4 - offAccess;
+        if (cbThisPart > cbValue)
+            cbThisPart = cbValue;
+
+        /*
+         * Get the missing bits (if any).
+         */
+        uint32_t u32MissingValue = 0;
+        if (fReadMissing && cbThisPart != 4)
+        {
+            VBOXSTRICTRC rc2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                          !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
+                                                          ? offRegion & ~(RTGCPHYS)3 : (GCPhys & ~(RTGCPHYS)3),
+                                                          &u32MissingValue, sizeof(u32MissingValue));
+            switch (VBOXSTRICTRC_VAL(rc2))
+            {
+                case VINF_SUCCESS:
+                    break;
+                case VINF_IOM_MMIO_UNUSED_FF:
+                    STAM_COUNTER_INC(&pStats->FFor00Reads);
+                    u32MissingValue = UINT32_C(0xffffffff);
+                    break;
+                case VINF_IOM_MMIO_UNUSED_00:
+                    STAM_COUNTER_INC(&pStats->FFor00Reads);
+                    u32MissingValue = 0;
+                    break;
+#ifndef IN_RING3
+                case VINF_IOM_R3_MMIO_READ:
+                case VINF_IOM_R3_MMIO_READ_WRITE:
+                case VINF_IOM_R3_MMIO_WRITE:
+                    LogFlow(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
+                    rc2 = iomMmioRing3WritePending(pVCpu, GCPhys, pvValue, cbValue, pRegEntry->idxSelf);
+                    if (rc == VINF_SUCCESS || rc2 < rc)
+                        rc = rc2;
+                    return rc;
+#endif
+                default:
+                    if (RT_FAILURE(rc2))
+                    {
+                        Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [read]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
+                        return rc2;
+                    }
+                    AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                    if (rc == VINF_SUCCESS || rc2 < rc)
+                        rc = rc2;
+                    break;
+            }
+        }
+
+        /*
+         * Merge missing and given bits.
+         */
+        uint32_t u32GivenMask;
+        uint32_t u32GivenValue;
+        switch (cbThisPart)
+        {
+            case 1:
+                u32GivenValue = *(uint8_t  const *)pvValue;
+                u32GivenMask  = UINT32_C(0x000000ff);
+                break;
+            case 2:
+                u32GivenValue = *(uint16_t const *)pvValue;
+                u32GivenMask  = UINT32_C(0x0000ffff);
+                break;
+            case 3:
+                u32GivenValue = RT_MAKE_U32_FROM_U8(((uint8_t const *)pvValue)[0], ((uint8_t const *)pvValue)[1],
+                                                    ((uint8_t const *)pvValue)[2], 0);
+                u32GivenMask  = UINT32_C(0x00ffffff);
+                break;
+            case 4:
+                u32GivenValue = *(uint32_t const *)pvValue;
+                u32GivenMask  = UINT32_C(0xffffffff);
+                break;
+            default:
+                AssertFailedReturn(VERR_IOM_MMIO_IPE_3);
+        }
+        if (offAccess)
+        {
+            u32GivenValue <<= offAccess * 8;
+            u32GivenMask  <<= offAccess * 8;
+        }
+
+        uint32_t u32Value = (u32MissingValue & ~u32GivenMask)
+                          | (u32GivenValue & u32GivenMask);
+
+        /*
+         * Do DWORD write to the device.
+         */
+        VBOXSTRICTRC rc2 = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                       !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
+                                                       ? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
+                                                       &u32Value, sizeof(u32Value));
+        switch (VBOXSTRICTRC_VAL(rc2))
+        {
+            case VINF_SUCCESS:
+                break;
+#ifndef IN_RING3
+            case VINF_IOM_R3_MMIO_READ:
+            case VINF_IOM_R3_MMIO_READ_WRITE:
+            case VINF_IOM_R3_MMIO_WRITE:
+                Log3(("iomMmioDoComplicatedWrite: deferring GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
+                AssertReturn(pVCpu->iom.s.PendingMmioWrite.cbValue == 0, VERR_IOM_MMIO_IPE_1);
+                AssertReturn(cbValue + (GCPhys & 3) <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue), VERR_IOM_MMIO_IPE_2);
+                pVCpu->iom.s.PendingMmioWrite.GCPhys  = GCPhys & ~(RTGCPHYS)3;
+                pVCpu->iom.s.PendingMmioWrite.cbValue = cbValue + (GCPhys & 3);
+                *(uint32_t *)pVCpu->iom.s.PendingMmioWrite.abValue = u32Value;
+                if (cbValue > cbThisPart)
+                    memcpy(&pVCpu->iom.s.PendingMmioWrite.abValue[4],
+                           (uint8_t const *)pvValue + cbThisPart, cbValue - cbThisPart);
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_IOM);
+                if (rc == VINF_SUCCESS)
+                    rc = VINF_IOM_R3_MMIO_COMMIT_WRITE;
+                return rc;
+#endif
+            default:
+                if (RT_FAILURE(rc2))
+                {
+                    Log(("iomMmioDoComplicatedWrite: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rc=%Rrc [write]\n", GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rc2)));
+                    return rc2;
+                }
+                AssertMsgReturn(rc2 >= VINF_EM_FIRST && rc2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rc2)), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                if (rc == VINF_SUCCESS || rc2 < rc)
+                    rc = rc2;
+                break;
+        }
+
+        /*
+         * Advance.
+         */
+        cbValue   -= cbThisPart;
+        if (!cbValue)
+            break;
+        GCPhys    += cbThisPart;
+        offRegion += cbThisPart;
+        pvValue    = (uint8_t const *)pvValue + cbThisPart;
+    }
+
+    return rc;
+}
+
+
+
+
+/**
+ * Wrapper which does the write.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioDoWrite(PVMCC pVM, PVMCPU pVCpu, CTX_SUFF(PIOMMMIOENTRY) pRegEntry,
+                                        RTGCPHYS GCPhys, RTGCPHYS offRegion,
+                                        const void *pvData, uint32_t cb IOM_MMIO_STATS_COMMA_DECL)
+{
+    VBOXSTRICTRC rcStrict;
+    if (RT_LIKELY(pRegEntry->pfnWriteCallback))
+    {
+        if (   (cb == 4 && !(GCPhys & 3))
+            || (pRegEntry->fFlags & IOMMMIO_FLAGS_WRITE_MODE) == IOMMMIO_FLAGS_WRITE_PASSTHRU
+            || (cb == 8 && !(GCPhys & 7) && IOMMMIO_DOES_WRITE_MODE_ALLOW_QWORD(pRegEntry->fFlags)) )
+            rcStrict = pRegEntry->pfnWriteCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                   !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvData, cb);
+        else
+            rcStrict = iomMmioDoComplicatedWrite(pVM, pVCpu, pRegEntry, GCPhys, offRegion, pvData, cb IOM_MMIO_STATS_COMMA_ARG);
+    }
+    else
+        rcStrict = VINF_SUCCESS;
+    return rcStrict;
+}
+
+
+#ifdef IN_RING3
+/**
+ * Helper for IOMR3ProcessForceFlag() that lives here to utilize iomMmioDoWrite et al.
+ */
+VBOXSTRICTRC iomR3MmioCommitWorker(PVM pVM, PVMCPU pVCpu, PIOMMMIOENTRYR3 pRegEntry, RTGCPHYS offRegion)
+{
+# ifdef VBOX_WITH_STATISTICS
+    STAM_PROFILE_START(UnusedMacroArg, Prf);
+    PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
+# endif
+    PPDMDEVINS const         pDevIns = pRegEntry->pDevIns;
+    int rc = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VERR_IGNORED);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, pVCpu->iom.s.PendingMmioWrite.GCPhys, offRegion,
+                                           pVCpu->iom.s.PendingMmioWrite.abValue, pVCpu->iom.s.PendingMmioWrite.cbValue
+                                           IOM_MMIO_STATS_COMMA_ARG);
+
+    PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
+    STAM_PROFILE_STOP(&pStats->ProfWriteR3, Prf);
+    return rcStrict;
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Deals with complicated MMIO reads.
+ *
+ * Complicated means unaligned or non-dword/qword sized accesses depending on
+ * the MMIO region's access mode flags.
+ *
+ * @returns Strict VBox status code. Any EM scheduling status code,
+ *          VINF_IOM_R3_MMIO_READ, VINF_IOM_R3_MMIO_READ_WRITE or
+ *          VINF_IOM_R3_MMIO_WRITE may be returned.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pRegEntry   The MMIO entry for the current context.
+ * @param   GCPhys      The physical address to start reading.
+ * @param   offRegion   MMIO region offset corresponding to @a GCPhys.
+ * @param   pvValue     Where to store the value.
+ * @param   cbValue     The size of the value to read.
+ * @param   pStats      Pointer to the statistics (never NULL).
+ */
+static VBOXSTRICTRC iomMMIODoComplicatedRead(PVM pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
+                                             void *pvValue, unsigned cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    AssertReturn(   (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD
+                 || (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD,
+                 VERR_IOM_MMIO_IPE_1);
+    AssertReturn(cbValue != 0 && cbValue <= 16, VERR_IOM_MMIO_IPE_2);
+#ifdef LOG_ENABLED
+    RTGCPHYS const GCPhysStart = GCPhys;
+#endif
+
+    /*
+     * Do debug stop if requested.
+     */
+    VBOXSTRICTRC rc = VINF_SUCCESS; NOREF(pVM);
+#ifdef VBOX_STRICT
+    if (pRegEntry->fFlags & IOMMMIO_FLAGS_DBGSTOP_ON_COMPLICATED_READ)
+    {
+# ifdef IN_RING3
+        rc = DBGFR3EventSrc(pVM, DBGFEVENT_DEV_STOP, RT_SRC_POS,
+                            "Complicated read %#x byte at %RGp to %s\n", cbValue, GCPhys, R3STRING(pRegEntry->pszDesc));
+        if (rc == VERR_DBGF_NOT_ATTACHED)
+            rc = VINF_SUCCESS;
+# else
+        return VINF_IOM_R3_MMIO_READ;
+# endif
+    }
+#endif
+
+    STAM_COUNTER_INC(&pStats->ComplicatedReads);
+
+    /*
+     * Split and conquer.
+     */
+    for (;;)
+    {
+        /*
+         * Do DWORD read from the device.
+         */
+        uint32_t u32Value;
+        VBOXSTRICTRC rcStrict2 = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                            !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS)
+                                                            ? offRegion & ~(RTGCPHYS)3 : GCPhys & ~(RTGCPHYS)3,
+                                                            &u32Value, sizeof(u32Value));
+        switch (VBOXSTRICTRC_VAL(rcStrict2))
+        {
+            case VINF_SUCCESS:
+                break;
+            case VINF_IOM_MMIO_UNUSED_FF:
+                STAM_COUNTER_INC(&pStats->FFor00Reads);
+                u32Value = UINT32_C(0xffffffff);
+                break;
+            case VINF_IOM_MMIO_UNUSED_00:
+                STAM_COUNTER_INC(&pStats->FFor00Reads);
+                u32Value = 0;
+                break;
+            case VINF_IOM_R3_MMIO_READ:
+            case VINF_IOM_R3_MMIO_READ_WRITE:
+            case VINF_IOM_R3_MMIO_WRITE:
+                /** @todo What if we've split a transfer and already read
+                 * something?  Since reads can have sideeffects we could be
+                 * kind of screwed here... */
+                LogFlow(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
+                         GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
+                return rcStrict2;
+            default:
+                if (RT_FAILURE(rcStrict2))
+                {
+                    Log(("iomMMIODoComplicatedRead: GCPhys=%RGp GCPhysStart=%RGp cbValue=%u rcStrict2=%Rrc\n",
+                         GCPhys, GCPhysStart, cbValue, VBOXSTRICTRC_VAL(rcStrict2)));
+                    return rcStrict2;
+                }
+                AssertMsgReturn(rcStrict2 >= VINF_EM_FIRST && rcStrict2 <= VINF_EM_LAST, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)),
+                                VERR_IPE_UNEXPECTED_INFO_STATUS);
+                if (rc == VINF_SUCCESS || rcStrict2 < rc)
+                    rc = rcStrict2;
+                break;
+        }
+        u32Value >>= (GCPhys & 3) * 8;
+
+        /*
+         * Write what we've read.
+         */
+        unsigned cbThisPart = 4 - (GCPhys & 3);
+        if (cbThisPart > cbValue)
+            cbThisPart = cbValue;
+
+        switch (cbThisPart)
+        {
+            case 1:
+                *(uint8_t *)pvValue = (uint8_t)u32Value;
+                break;
+            case 2:
+                *(uint16_t *)pvValue = (uint16_t)u32Value;
+                break;
+            case 3:
+                ((uint8_t *)pvValue)[0] = RT_BYTE1(u32Value);
+                ((uint8_t *)pvValue)[1] = RT_BYTE2(u32Value);
+                ((uint8_t *)pvValue)[2] = RT_BYTE3(u32Value);
+                break;
+            case 4:
+                *(uint32_t *)pvValue = u32Value;
+                break;
+        }
+
+        /*
+         * Advance.
+         */
+        cbValue   -= cbThisPart;
+        if (!cbValue)
+            break;
+        GCPhys    += cbThisPart;
+        offRegion += cbThisPart;
+        pvValue    = (uint8_t *)pvValue + cbThisPart;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Implements VINF_IOM_MMIO_UNUSED_FF.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pvValue     Where to store the zeros.
+ * @param   cbValue     How many bytes to read.
+ * @param   pStats      Pointer to the statistics (never NULL).
+ */
+static int iomMMIODoReadFFs(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    switch (cbValue)
+    {
+        case 1: *(uint8_t  *)pvValue = UINT8_C(0xff); break;
+        case 2: *(uint16_t *)pvValue = UINT16_C(0xffff); break;
+        case 4: *(uint32_t *)pvValue = UINT32_C(0xffffffff); break;
+        case 8: *(uint64_t *)pvValue = UINT64_C(0xffffffffffffffff); break;
+        default:
+        {
+            uint8_t *pb = (uint8_t *)pvValue;
+            while (cbValue--)
+                *pb++ = UINT8_C(0xff);
+            break;
+        }
+    }
+    STAM_COUNTER_INC(&pStats->FFor00Reads);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Implements VINF_IOM_MMIO_UNUSED_00.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pvValue     Where to store the zeros.
+ * @param   cbValue     How many bytes to read.
+ * @param   pStats      Pointer to the statistics (never NULL).
+ */
+static int iomMMIODoRead00s(void *pvValue, size_t cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    switch (cbValue)
+    {
+        case 1: *(uint8_t  *)pvValue = UINT8_C(0x00); break;
+        case 2: *(uint16_t *)pvValue = UINT16_C(0x0000); break;
+        case 4: *(uint32_t *)pvValue = UINT32_C(0x00000000); break;
+        case 8: *(uint64_t *)pvValue = UINT64_C(0x0000000000000000); break;
+        default:
+        {
+            uint8_t *pb = (uint8_t *)pvValue;
+            while (cbValue--)
+                *pb++ = UINT8_C(0x00);
+            break;
+        }
+    }
+    STAM_COUNTER_INC(&pStats->FFor00Reads);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Wrapper which does the read.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioDoRead(PVMCC pVM, CTX_SUFF(PIOMMMIOENTRY) pRegEntry, RTGCPHYS GCPhys, RTGCPHYS offRegion,
+                                       void *pvValue, uint32_t cbValue IOM_MMIO_STATS_COMMA_DECL)
+{
+    VBOXSTRICTRC rcStrict;
+    if (RT_LIKELY(pRegEntry->pfnReadCallback))
+    {
+        if (   (   cbValue == 4
+                && !(GCPhys & 3))
+            || (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_PASSTHRU
+            || (    cbValue == 8
+                && !(GCPhys & 7)
+                && (pRegEntry->fFlags & IOMMMIO_FLAGS_READ_MODE) == IOMMMIO_FLAGS_READ_DWORD_QWORD ) )
+            rcStrict = pRegEntry->pfnReadCallback(pRegEntry->pDevIns, pRegEntry->pvUser,
+                                                  !(pRegEntry->fFlags & IOMMMIO_FLAGS_ABS) ? offRegion : GCPhys, pvValue, cbValue);
+        else
+            rcStrict = iomMMIODoComplicatedRead(pVM, pRegEntry, GCPhys, offRegion, pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG);
+    }
+    else
+        rcStrict = VINF_IOM_MMIO_UNUSED_FF;
+    if (rcStrict != VINF_SUCCESS)
+    {
+        switch (VBOXSTRICTRC_VAL(rcStrict))
+        {
+            case VINF_IOM_MMIO_UNUSED_FF: rcStrict = iomMMIODoReadFFs(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
+            case VINF_IOM_MMIO_UNUSED_00: rcStrict = iomMMIODoRead00s(pvValue, cbValue IOM_MMIO_STATS_COMMA_ARG); break;
+        }
+    }
+    return rcStrict;
+}
+
+#ifndef IN_RING3
+
+/**
+ * Checks if we can handle an MMIO \#PF in R0/RC.
+ */
+DECLINLINE(bool) iomMmioCanHandlePfInRZ(PVMCC pVM, uint32_t uErrorCode, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
+    if (pRegEntry->cbRegion > 0)
+    {
+        if (   pRegEntry->pfnWriteCallback
+            && pRegEntry->pfnReadCallback)
+            return true;
+
+        PIOMMMIOENTRYR3 const pRegEntryR3 = &pVM->iomr0.s.paMmioRing3Regs[pRegEntry->idxSelf];
+        if (  uErrorCode == UINT32_MAX
+            ? pRegEntryR3->pfnWriteCallback || pRegEntryR3->pfnReadCallback
+            : uErrorCode & X86_TRAP_PF_RW
+              ? !pRegEntry->pfnWriteCallback && pRegEntryR3->pfnWriteCallback
+              : !pRegEntry->pfnReadCallback  && pRegEntryR3->pfnReadCallback)
+            return false;
+
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Common worker for the \#PF handler and IOMMMIOPhysHandler (APIC+VT-x).
+ *
+ * @returns VBox status code (appropriate for GC return).
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uErrorCode  CPU Error code.  This is UINT32_MAX when we don't have
+ *                      any error code (the EPT misconfig hack).
+ * @param   GCPhysFault The GC physical address corresponding to pvFault.
+ * @param   pRegEntry   The MMIO entry for the current context.
+ */
+DECLINLINE(VBOXSTRICTRC) iomMmioCommonPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode,
+                                                   RTGCPHYS GCPhysFault, CTX_SUFF(PIOMMMIOENTRY) pRegEntry)
+{
+    Log(("iomMmioCommonPfHandler: GCPhysFault=%RGp uErr=%#x rip=%RGv\n", GCPhysFault, uErrorCode, CPUMGetGuestRIP(pVCpu) ));
+    RT_NOREF(GCPhysFault, uErrorCode);
+
+    VBOXSTRICTRC rcStrict;
+
+#ifndef IN_RING3
+    /*
+     * Should we defer the request right away?  This isn't usually the case, so
+     * do the simple test first and the try deal with uErrorCode being N/A.
+     */
+    PPDMDEVINS const pDevIns = pRegEntry->pDevIns;
+    if (RT_LIKELY(   pDevIns
+                  && iomMmioCanHandlePfInRZ(pVM, uErrorCode, pRegEntry)))
+    {
+        /*
+         * Enter the device critsect prior to engaging IOM in case of lock contention.
+         * Note! Perhaps not a good move?
+         */
+        rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), VINF_IOM_R3_MMIO_READ_WRITE);
+        if (rcStrict == VINF_SUCCESS)
+        {
+#endif /* !IN_RING3 */
+
+            /*
+             * Let IEM call us back via iomMmioHandler.
+             */
+            rcStrict = IEMExecOne(pVCpu);
+
+#ifndef IN_RING3
+            PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
+#endif
+            if (RT_SUCCESS(rcStrict))
+            { /* likely */ }
+            else if (   rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
+                     || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED)
+            {
+                Log(("IOM: Hit unsupported IEM feature!\n"));
+                rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+            }
+#ifndef IN_RING3
+            return rcStrict;
+        }
+        STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
+    }
+    else
+        rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
+
+# ifdef VBOX_WITH_STATISTICS
+    if (rcStrict == VINF_IOM_R3_MMIO_READ_WRITE)
+    {
+        PIOMMMIOSTATSENTRY const pStats = iomMmioGetStats(pVM, pRegEntry);
+        if (uErrorCode & X86_TRAP_PF_RW)
+        {
+            STAM_COUNTER_INC(&pStats->WriteRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
+        }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->ReadRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
+        }
+    }
+# endif
+#else  /* IN_RING3 */
+    RT_NOREF(pVM, pRegEntry);
+#endif /* IN_RING3 */
+    return rcStrict;
+}
+
+
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      \#PF access handler callback for MMIO pages.}
+ *
+ * @remarks The @a uUser argument is the MMIO handle.
+ */
+DECLCALLBACK(VBOXSTRICTRC) iomMmioPfHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
+                                               RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
+{
+    STAM_PROFILE_START(&pVM->iom.s.StatMmioPfHandler, Prf);
+    LogFlow(("iomMmioPfHandlerNew: GCPhys=%RGp uErr=%#x pvFault=%RGv rip=%RGv\n",
+             GCPhysFault, (uint32_t)uErrorCode, pvFault, (RTGCPTR)pCtx->rip));
+    RT_NOREF(pvFault, pCtx);
+
+    /* Translate the MMIO handle to a registration entry for the current context. */
+    AssertReturn(uUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+# ifdef IN_RING0
+    AssertReturn(uUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iomr0.s.paMmioRegs[uUser];
+# else
+    CTX_SUFF(PIOMMMIOENTRY) pRegEntry = &pVM->iom.s.paMmioRegs[uUser];
+# endif
+
+    VBOXSTRICTRC rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
+
+    STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPfHandler, Prf);
+    return rcStrict;
+}
+
+#endif /* !IN_RING3 */
+
+#ifdef IN_RING0
+/**
+ * Physical access handler for MMIO ranges.
+ *
+ * This is actually only used by VT-x for APIC page accesses.
+ *
+ * @returns VBox status code (appropriate for GC return).
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   uErrorCode  CPU Error code.
+ * @param   GCPhysFault The GC physical address.
+ */
+VMM_INT_DECL(VBOXSTRICTRC) IOMR0MmioPhysHandler(PVMCC pVM, PVMCPUCC pVCpu, uint32_t uErrorCode, RTGCPHYS GCPhysFault)
+{
+    STAM_PROFILE_START(&pVM->iom.s.StatMmioPhysHandler, Prf);
+
+    /*
+     * We don't have a range here, so look it up before calling the common function.
+     */
+    VBOXSTRICTRC rcStrict = IOM_LOCK_SHARED(pVM);
+    if (RT_SUCCESS(rcStrict))
+    {
+        RTGCPHYS offRegion;
+        CTX_SUFF(PIOMMMIOENTRY) pRegEntry = iomMmioGetEntry(pVM, GCPhysFault, &offRegion, &pVCpu->iom.s.idxMmioLastPhysHandler);
+        IOM_UNLOCK_SHARED(pVM);
+        if (RT_LIKELY(pRegEntry))
+            rcStrict = iomMmioCommonPfHandlerNew(pVM, pVCpu, (uint32_t)uErrorCode, GCPhysFault, pRegEntry);
+        else
+            rcStrict = VERR_IOM_MMIO_RANGE_NOT_FOUND;
+    }
+    else if (rcStrict == VERR_SEM_BUSY)
+        rcStrict = VINF_IOM_R3_MMIO_READ_WRITE;
+
+    STAM_PROFILE_STOP(&pVM->iom.s.StatMmioPhysHandler, Prf);
+    return rcStrict;
+}
+#endif /* IN_RING0 */
+
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER, MMIO page accesses}
+ *
+ * @remarks The @a uUser argument is the MMIO handle.
+ */
+DECLCALLBACK(VBOXSTRICTRC) iomMmioHandlerNew(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysFault, void *pvPhys, void *pvBuf,
+                                             size_t cbBuf, PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
+{
+    STAM_PROFILE_START(UnusedMacroArg, Prf);
+    STAM_COUNTER_INC(&pVM->iom.s.CTX_SUFF(StatMmioHandler));
+    Log4(("iomMmioHandlerNew: GCPhysFault=%RGp cbBuf=%#x enmAccessType=%d enmOrigin=%d uUser=%p\n", GCPhysFault, cbBuf, enmAccessType, enmOrigin, uUser));
+
+    Assert(enmAccessType == PGMACCESSTYPE_READ || enmAccessType == PGMACCESSTYPE_WRITE);
+    AssertMsg(cbBuf >= 1, ("%zu\n", cbBuf));
+    NOREF(pvPhys); NOREF(enmOrigin);
+
+#ifdef IN_RING3
+    int const rcToRing3 = VERR_IOM_MMIO_IPE_3;
+#else
+    int const rcToRing3 = enmAccessType == PGMACCESSTYPE_READ ? VINF_IOM_R3_MMIO_READ : VINF_IOM_R3_MMIO_WRITE;
+#endif
+
+    /*
+     * Translate uUser to an MMIO registration table entry.  We can do this
+     * without any locking as the data is static after VM creation.
+     */
+    AssertReturn(uUser < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn(uUser < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    CTX_SUFF(PIOMMMIOENTRY) const pRegEntry    = &pVM->iomr0.s.paMmioRegs[uUser];
+    PIOMMMIOENTRYR3 const         pRegEntryR3  = &pVM->iomr0.s.paMmioRing3Regs[uUser];
+#else
+    CTX_SUFF(PIOMMMIOENTRY) const pRegEntry    = &pVM->iom.s.paMmioRegs[uUser];
+#endif
+#ifdef VBOX_WITH_STATISTICS
+    PIOMMMIOSTATSENTRY const      pStats       = iomMmioGetStats(pVM, pRegEntry);  /* (Works even without ring-0 device setup.) */
+#endif
+    PPDMDEVINS const              pDevIns      = pRegEntry->pDevIns;
+
+#ifdef VBOX_STRICT
+    /*
+     * Assert the right entry in strict builds.  This may yield a false positive
+     * for SMP VMs if we're unlucky and the guest isn't well behaved.
+     */
+# ifdef IN_RING0
+    Assert(pRegEntry && (GCPhysFault - pRegEntryR3->GCPhysMapping < pRegEntryR3->cbRegion || !pRegEntryR3->fMapped));
+# else
+    Assert(pRegEntry && (GCPhysFault - pRegEntry->GCPhysMapping   < pRegEntry->cbRegion   || !pRegEntry->fMapped));
+# endif
+#endif
+
+#ifndef IN_RING3
+    /*
+     * If someone is doing FXSAVE, FXRSTOR, XSAVE, XRSTOR or other stuff dealing with
+     * large amounts of data, just go to ring-3 where we don't need to deal with partial
+     * successes.  No chance any of these will be problematic read-modify-write stuff.
+     *
+     * Also drop back if the ring-0 registration entry isn't actually used.
+     */
+    if (   RT_LIKELY(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue))
+        && pRegEntry->cbRegion != 0
+        && (  enmAccessType == PGMACCESSTYPE_READ
+            ? pRegEntry->pfnReadCallback  != NULL || pVM->iomr0.s.paMmioRing3Regs[uUser].pfnReadCallback == NULL
+            : pRegEntry->pfnWriteCallback != NULL || pVM->iomr0.s.paMmioRing3Regs[uUser].pfnWriteCallback == NULL)
+        && pDevIns )
+    { /* likely */ }
+    else
+    {
+        Log4(("iomMmioHandlerNew: to ring-3: to-big=%RTbool zero-size=%RTbool no-callback=%RTbool pDevIns=%p hRegion=%#RX64\n",
+              !(cbBuf <= sizeof(pVCpu->iom.s.PendingMmioWrite.abValue)), !(pRegEntry->cbRegion != 0),
+              !(  enmAccessType == PGMACCESSTYPE_READ
+                ? pRegEntry->pfnReadCallback  != NULL || pVM->iomr0.s.paMmioRing3Regs[uUser].pfnReadCallback == NULL
+                : pRegEntry->pfnWriteCallback != NULL || pVM->iomr0.s.paMmioRing3Regs[uUser].pfnWriteCallback == NULL),
+              pDevIns, uUser));
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
+        return rcToRing3;
+    }
+#endif /* !IN_RING3 */
+
+    /*
+     * If we've got an offset that's outside the region, defer to ring-3 if we
+     * can, or pretend there is nothing there.  This shouldn't happen, but can
+     * if we're unlucky with an SMP VM and the guest isn't behaving very well.
+     */
+#ifdef IN_RING0
+    RTGCPHYS const GCPhysMapping = pRegEntryR3->GCPhysMapping;
+#else
+    RTGCPHYS const GCPhysMapping = pRegEntry->GCPhysMapping;
+#endif
+    RTGCPHYS const offRegion     = GCPhysFault - GCPhysMapping;
+    if (RT_LIKELY(offRegion < pRegEntry->cbRegion && GCPhysMapping != NIL_RTGCPHYS))
+    { /* likely */ }
+    else
+    {
+        STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioStaleMappings);
+        LogRelMax(64, ("iomMmioHandlerNew: Stale access at %#RGp to range #%#x currently residing at %RGp LB %RGp\n",
+                       GCPhysFault, pRegEntry->idxSelf, GCPhysMapping, pRegEntry->cbRegion));
+#ifdef IN_RING3
+        if (enmAccessType == PGMACCESSTYPE_READ)
+            iomMMIODoReadFFs(pvBuf, cbBuf IOM_MMIO_STATS_COMMA_ARG);
+        return VINF_SUCCESS;
+#else
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pStats->ReadRZToR3 : &pStats->WriteRZToR3);
+        STAM_COUNTER_INC(enmAccessType == PGMACCESSTYPE_READ ? &pVM->iom.s.StatMmioReadsR0ToR3 : &pVM->iom.s.StatMmioWritesR0ToR3);
+        return rcToRing3;
+#endif
+    }
+
+    /*
+     * Guard against device configurations causing recursive MMIO accesses
+     * (see @bugref{10315}).
+     */
+    uint8_t const idxDepth = pVCpu->iom.s.cMmioRecursionDepth;
+    if (RT_LIKELY(idxDepth < RT_ELEMENTS(pVCpu->iom.s.apMmioRecursionStack)))
+    {
+        pVCpu->iom.s.cMmioRecursionDepth  = idxDepth + 1;
+        /** @todo Add iomr0 with a apMmioRecursionStack for ring-0. */
+#ifdef IN_RING3
+        pVCpu->iom.s.apMmioRecursionStack[idxDepth] = pDevIns;
+#endif
+    }
+    else
+    {
+        STAM_REL_COUNTER_INC(&pVM->iom.s.StatMmioTooDeepRecursion);
+#ifdef IN_RING3
+        AssertCompile(RT_ELEMENTS(pVCpu->iom.s.apMmioRecursionStack) == 2);
+        LogRelMax(64, ("iomMmioHandlerNew: Too deep recursion %RGp LB %#zx: %p (%s); %p (%s); %p (%s)\n",
+                       GCPhysFault, cbBuf, pDevIns, pDevIns->pReg->szName,
+                       pVCpu->iom.s.apMmioRecursionStack[1], pVCpu->iom.s.apMmioRecursionStack[1]->pReg->szName,
+                       pVCpu->iom.s.apMmioRecursionStack[0], pVCpu->iom.s.apMmioRecursionStack[0]->pReg->szName));
+#else
+        LogRelMax(64, ("iomMmioHandlerNew: Too deep recursion %RGp LB %#zx!: %p (%s)\n",
+                       GCPhysFault, cbBuf, pDevIns, pDevIns->pReg->szName));
+#endif
+        return VINF_PGM_HANDLER_DO_DEFAULT;
+    }
+
+
+    /*
+     * Perform locking and the access.
+     *
+     * Writes requiring a return to ring-3 are buffered by IOM so IEM can
+     * commit the instruction.
+     *
+     * Note! We may end up locking the device even when the relevant callback is
+     *       NULL.  This is supposed to be an unlikely case, so not optimized yet.
+     *
+     * Note! All returns goes thru the one return statement at the end of the
+     *       function in order to correctly maintaint the recursion counter.
+     */
+    VBOXSTRICTRC rcStrict = PDMCritSectEnter(pVM, pDevIns->CTX_SUFF(pCritSectRo), rcToRing3);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (enmAccessType == PGMACCESSTYPE_READ)
+        {
+            /*
+             * Read.
+             */
+            rcStrict = iomMmioDoRead(pVM, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
+
+            PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
+#ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_MMIO_READ)
+            {
+                STAM_COUNTER_INC(&pStats->ReadRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatMmioReadsR0ToR3);
+            }
+            else
+#endif
+                STAM_COUNTER_INC(&pStats->Reads);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfRead), Prf);
+        }
+        else
+        {
+            /*
+             * Write.
+             */
+            rcStrict = iomMmioDoWrite(pVM, pVCpu, pRegEntry, GCPhysFault, offRegion, pvBuf, (uint32_t)cbBuf IOM_MMIO_STATS_COMMA_ARG);
+            PDMCritSectLeave(pVM, pDevIns->CTX_SUFF(pCritSectRo));
+#ifndef IN_RING3
+            if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+                rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
+            if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+            {
+                STAM_COUNTER_INC(&pStats->WriteRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
+            }
+            else if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
+            {
+                STAM_COUNTER_INC(&pStats->CommitRZToR3);
+                STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
+            }
+            else
+#endif
+                STAM_COUNTER_INC(&pStats->Writes);
+            STAM_PROFILE_STOP(&pStats->CTX_SUFF_Z(ProfWrite), Prf);
+        }
+
+        /*
+         * Check the return code.
+         */
+#ifdef IN_RING3
+        AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc -  Access type %d - %RGp - %s\n",
+                                             VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pRegEntry->pszDesc));
+#else
+        AssertMsg(   rcStrict == VINF_SUCCESS
+                  || rcStrict == rcToRing3
+                  || (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE && enmAccessType == PGMACCESSTYPE_WRITE)
+                  || rcStrict == VINF_EM_DBG_STOP
+                  || rcStrict == VINF_EM_DBG_EVENT
+                  || rcStrict == VINF_EM_DBG_BREAKPOINT
+                  || rcStrict == VINF_EM_OFF
+                  || rcStrict == VINF_EM_SUSPEND
+                  || rcStrict == VINF_EM_RESET
+                  //|| rcStrict == VINF_EM_HALT       /* ?? */
+                  //|| rcStrict == VINF_EM_NO_MEMORY  /* ?? */
+                  , ("%Rrc - Access type %d - %RGp - %s #%u\n",
+                     VBOXSTRICTRC_VAL(rcStrict), enmAccessType, GCPhysFault, pDevIns->pReg->szName, pDevIns->iInstance));
+#endif
+    }
+    /*
+     * Deal with enter-critsect failures.
+     */
+#ifndef IN_RING3
+    else if (rcStrict == VINF_IOM_R3_MMIO_WRITE)
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+        rcStrict = iomMmioRing3WritePending(pVCpu, GCPhysFault, pvBuf, cbBuf, pRegEntry->idxSelf);
+        if (rcStrict == VINF_IOM_R3_MMIO_COMMIT_WRITE)
+        {
+            STAM_COUNTER_INC(&pStats->CommitRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatMmioCommitsR0ToR3);
+        }
+        else
+        {
+            STAM_COUNTER_INC(&pStats->WriteRZToR3);
+            STAM_COUNTER_INC(&pVM->iom.s.StatMmioWritesR0ToR3);
+        }
+        STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
+    }
+    else if (rcStrict == VINF_IOM_R3_MMIO_READ)
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_READ);
+        STAM_COUNTER_INC(&pStats->ReadRZToR3);
+        STAM_COUNTER_INC(&pVM->iom.s.StatMmioDevLockContentionR0);
+    }
+#endif
+    else
+        AssertMsg(RT_FAILURE_NP(rcStrict), ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    pVCpu->iom.s.cMmioRecursionDepth = idxDepth;
+    return rcStrict;
+}
+
+
+/**
+ * Mapping an MMIO2 page in place of an MMIO page for direct access.
+ *
+ * This is a special optimization used by the VGA device.  Call
+ * IOMMmioResetRegion() to undo the mapping.
+ *
+ * @returns VBox status code.  This API may return VINF_SUCCESS even if no
+ *          remapping is made.
+ * @retval  VERR_SEM_BUSY in ring-0 if we cannot get the IOM lock.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance @a hRegion and @a hMmio2 are
+ *                          associated with.
+ * @param   hRegion         The handle to the MMIO region.
+ * @param   offRegion       The offset into @a hRegion of the page to be
+ *                          remapped.
+ * @param   hMmio2          The MMIO2 handle.
+ * @param   offMmio2        Offset into @a hMmio2 of the page to be use for the
+ *                          mapping.
+ * @param   fPageFlags      Page flags to set. Must be (X86_PTE_RW | X86_PTE_P)
+ *                          for the time being.
+ */
+VMMDECL(int) IOMMmioMapMmio2Page(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion, RTGCPHYS offRegion,
+                                 uint64_t hMmio2, RTGCPHYS offMmio2, uint64_t fPageFlags)
+{
+    /* Currently only called from the VGA device during MMIO. */
+    Log(("IOMMmioMapMmio2Page %#RX64/%RGp -> %#RX64/%RGp flags=%RX64\n", hRegion, offRegion, hMmio2, offMmio2, fPageFlags));
+    AssertReturn(fPageFlags == (X86_PTE_RW | X86_PTE_P), VERR_INVALID_PARAMETER);
+    AssertReturn(pDevIns, VERR_INVALID_POINTER);
+
+/** @todo Why is this restricted to protected mode???  Try it in all modes! */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+
+    /* This currently only works in real mode, protected mode without paging or with nested paging. */
+    /** @todo NEM: MMIO page aliasing. */
+    if (    !HMIsEnabled(pVM)       /* useless without VT-x/AMD-V */
+        ||  (   CPUMIsGuestInPagedProtectedMode(pVCpu)
+             && !HMIsNestedPagingActive(pVM)))
+        return VINF_SUCCESS;    /* ignore */ /** @todo return some indicator if we fail here */
+
+    /*
+     * Translate the handle into an entry and check the region offset.
+     */
+    AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(offRegion < pVM->iomr0.s.paMmioRegs[hRegion].cbRegion, VERR_OUT_OF_RANGE);
+    AssertReturn(   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
+                 || (   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
+                     && pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
+#else
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
+#endif
+    AssertReturn(offRegion < pRegEntry->cbRegion, VERR_OUT_OF_RANGE);
+    Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0);
+
+    /*
+     * When getting and using the mapping address, we must sit on the IOM lock
+     * to prevent remapping.  Shared suffices as we change nothing.
+     */
+    int rc = IOM_LOCK_SHARED(pVM);
+    if (rc == VINF_SUCCESS)
+    {
+        RTGCPHYS const GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
+        if (GCPhys != NIL_RTGCPHYS)
+        {
+            Assert(!(GCPhys & GUEST_PAGE_OFFSET_MASK));
+
+            /*
+             * Do the aliasing; page align the addresses since PGM is picky.
+             */
+            rc = PGMHandlerPhysicalPageAliasMmio2(pVM, GCPhys, GCPhys + (offRegion & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK),
+                                                  pDevIns, hMmio2, offMmio2);
+        }
+        else
+            AssertFailedStmt(rc = VERR_IOM_MMIO_REGION_NOT_MAPPED);
+
+        IOM_UNLOCK_SHARED(pVM);
+    }
+
+/** @todo either ditch this or replace it with something that works in the
+ *        nested case, since we really only care about nested paging! */
+#if 0
+    /*
+     * Modify the shadow page table. Since it's an MMIO page it won't be present and we
+     * can simply prefetch it.
+     *
+     * Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
+     */
+# if 0 /* The assertion is wrong for the PGM_SYNC_CLEAR_PGM_POOL and VINF_PGM_HANDLER_ALREADY_ALIASED cases. */
+#  ifdef VBOX_STRICT
+    uint64_t fFlags;
+    RTHCPHYS HCPhys;
+    rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
+    Assert(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+#  endif
+# endif
+    rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
+    Assert(rc == VINF_SUCCESS || rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+#endif
+    return rc;
+}
+
+
+#ifdef IN_RING0 /* VT-x ring-0 only, move to IOMR0Mmio.cpp later.  */
+/**
+ * Mapping a HC page in place of an MMIO page for direct access.
+ *
+ * This is a special optimization used by the APIC in the VT-x case.  This VT-x
+ * code uses PGMHandlerPhysicalReset rather than IOMMmioResetRegion() to undo
+ * the effects here.
+ *
+ * @todo Make VT-x usage more consistent.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhys          The address of the MMIO page to be changed.
+ * @param   HCPhys          The address of the host physical page.
+ * @param   fPageFlags      Page flags to set. Must be (X86_PTE_RW | X86_PTE_P)
+ *                          for the time being.
+ */
+VMMR0_INT_DECL(int) IOMR0MmioMapMmioHCPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint64_t fPageFlags)
+{
+    /* Currently only called from VT-x code during a page fault. */
+    Log(("IOMR0MmioMapMmioHCPage %RGp -> %RGp flags=%RX64\n", GCPhys, HCPhys, fPageFlags));
+
+    AssertReturn(fPageFlags == (X86_PTE_RW | X86_PTE_P), VERR_INVALID_PARAMETER);
+    /** @todo NEM: MMIO page aliasing?? */
+    Assert(HMIsEnabled(pVM));
+
+# ifdef VBOX_STRICT
+    /*
+     * Check input address (it's HM calling, not the device, so no region handle).
+     */
+    int rcSem = IOM_LOCK_SHARED(pVM);
+    if (rcSem == VINF_SUCCESS)
+    {
+        RTGCPHYS offIgn;
+        uint16_t idxIgn = UINT16_MAX;
+        PIOMMMIOENTRYR0 pRegEntry = iomMmioGetEntry(pVM, GCPhys, &offIgn, &idxIgn);
+        IOM_UNLOCK_SHARED(pVM);
+        Assert(pRegEntry);
+        Assert(pRegEntry && !(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK));
+    }
+# endif
+
+    /*
+     * Do the aliasing; page align the addresses since PGM is picky.
+     */
+    GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    HCPhys &= ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK;
+
+    int rc = PGMHandlerPhysicalPageAliasHC(pVM, GCPhys, GCPhys, HCPhys);
+    AssertRCReturn(rc, rc);
+
+/** @todo either ditch this or replace it with something that works in the
+ *        nested case, since we really only care about nested paging! */
+
+    /*
+     * Modify the shadow page table. Since it's an MMIO page it won't be present and we
+     * can simply prefetch it.
+     *
+     * Note: This is a NOP in the EPT case; we'll just let it fault again to resync the page.
+     */
+    rc = PGMPrefetchPage(pVCpu, (RTGCPTR)GCPhys);
+    Assert(rc == VINF_SUCCESS || rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+    return VINF_SUCCESS;
+}
+#endif
+
+
+/**
+ * Reset a previously modified MMIO region; restore the access flags.
+ *
+ * This undoes the effects of IOMMmioMapMmio2Page() and is currently only
+ * intended for some ancient VGA hack.  However, it would be great to extend it
+ * beyond VT-x and/or nested-paging.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device instance @a hRegion is associated with.
+ * @param   hRegion         The handle to the MMIO region.
+ */
+VMMDECL(int) IOMMmioResetRegion(PVMCC pVM, PPDMDEVINS pDevIns, IOMMMIOHANDLE hRegion)
+{
+    Log(("IOMMMIOResetRegion %#RX64\n", hRegion));
+    AssertReturn(pDevIns, VERR_INVALID_POINTER);
+
+/** @todo Get rid of this this real/protected or nested paging restriction,
+ *        it probably shouldn't be here and would be nasty when the CPU
+ *        changes mode while we have the hack enabled... */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+
+    /* This currently only works in real mode, protected mode without paging or with nested paging. */
+    /** @todo NEM: MMIO page aliasing. */
+    if (   !HMIsEnabled(pVM)       /* useless without VT-x/AMD-V */
+        || (   CPUMIsGuestInPagedProtectedMode(pVCpu)
+            && !HMIsNestedPagingActive(pVM)))
+        return VINF_SUCCESS;    /* ignore */
+
+    /*
+     * Translate the handle into an entry and mapping address for PGM.
+     * We have to take the lock to safely access the mapping address here.
+     */
+    AssertReturn(hRegion < RT_MIN(pVM->iom.s.cMmioRegs, pVM->iom.s.cMmioAlloc), VERR_IOM_INVALID_MMIO_HANDLE);
+#ifdef IN_RING0
+    AssertReturn(hRegion < pVM->iomr0.s.cMmioAlloc, VERR_IOM_INVALID_MMIO_HANDLE);
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iomr0.s.paMmioRing3Regs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == pDevIns
+                 || (   pVM->iomr0.s.paMmioRegs[hRegion].pDevIns == NULL
+                     && pRegEntry->pDevIns == pDevIns->pDevInsForR3), VERR_ACCESS_DENIED);
+#else
+    PIOMMMIOENTRYR3 const pRegEntry = &pVM->iom.s.paMmioRegs[hRegion];
+    AssertReturn(pRegEntry->cbRegion > 0, VERR_IOM_INVALID_MMIO_HANDLE);
+    AssertReturn(pRegEntry->pDevIns == pDevIns, VERR_ACCESS_DENIED);
+#endif
+    Assert((pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK) == 0);
+
+    int rcSem = IOM_LOCK_SHARED(pVM);
+    RTGCPHYS GCPhys = pRegEntry->fMapped ? pRegEntry->GCPhysMapping : NIL_RTGCPHYS;
+    if (rcSem == VINF_SUCCESS)
+        IOM_UNLOCK_SHARED(pVM);
+
+    Assert(!(GCPhys              & GUEST_PAGE_OFFSET_MASK));
+    Assert(!(pRegEntry->cbRegion & GUEST_PAGE_OFFSET_MASK));
+
+    /*
+     * Call PGM to do the job work.
+     *
+     * After the call, all the pages should be non-present, unless there is
+     * a page pool flush pending (unlikely).
+     */
+    int rc = PGMHandlerPhysicalReset(pVM, GCPhys);
+    AssertRC(rc);
+
+# ifdef VBOX_STRICT
+    if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
+    {
+        RTGCPHYS cb = pRegEntry->cbRegion;
+        while (cb)
+        {
+            uint64_t fFlags;
+            RTHCPHYS HCPhys;
+            rc = PGMShwGetPage(pVCpu, (RTGCPTR)GCPhys, &fFlags, &HCPhys);
+            Assert(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT);
+            cb     -= RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE);
+            GCPhys += RT_MIN(GUEST_PAGE_SIZE, HOST_PAGE_SIZE);
+        }
+    }
+# endif
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMAll/MMAll.cpp b/src/VBox/VMM/VMMAll/MMAll.cpp
new file mode 100644
index 00000000..856beb00
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/MMAll.cpp
@@ -0,0 +1,160 @@
+/* $Id: MMAll.cpp $ */
+/** @file
+ * MM - Memory Manager - Any Context.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_MM_HYPER
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmm.h>
+#include "MMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/**
+ * Gets the string name of a memory tag.
+ *
+ * @returns name of enmTag.
+ * @param   enmTag      The tag.
+ */
+const char *mmGetTagName(MMTAG enmTag)
+{
+    switch (enmTag)
+    {
+        #define TAG2STR(tag) case MM_TAG_##tag: return #tag
+
+        TAG2STR(CFGM);
+        TAG2STR(CFGM_BYTES);
+        TAG2STR(CFGM_STRING);
+        TAG2STR(CFGM_USER);
+
+        TAG2STR(CPUM_CTX);
+        TAG2STR(CPUM_CPUID);
+        TAG2STR(CPUM_MSRS);
+
+        TAG2STR(CSAM);
+        TAG2STR(CSAM_PATCH);
+
+        TAG2STR(DBGF);
+        TAG2STR(DBGF_AS);
+        TAG2STR(DBGF_FLOWTRACE);
+        TAG2STR(DBGF_INFO);
+        TAG2STR(DBGF_LINE);
+        TAG2STR(DBGF_LINE_DUP);
+        TAG2STR(DBGF_MODULE);
+        TAG2STR(DBGF_OS);
+        TAG2STR(DBGF_REG);
+        TAG2STR(DBGF_STACK);
+        TAG2STR(DBGF_SYMBOL);
+        TAG2STR(DBGF_SYMBOL_DUP);
+        TAG2STR(DBGF_TYPE);
+        TAG2STR(DBGF_TRACER);
+
+        TAG2STR(EM);
+
+        TAG2STR(IEM);
+
+        TAG2STR(IOM);
+        TAG2STR(IOM_STATS);
+
+        TAG2STR(MM);
+        TAG2STR(MM_LOOKUP_GUEST);
+        TAG2STR(MM_LOOKUP_PHYS);
+        TAG2STR(MM_LOOKUP_VIRT);
+        TAG2STR(MM_PAGE);
+
+        TAG2STR(PARAV);
+
+        TAG2STR(PATM);
+        TAG2STR(PATM_PATCH);
+
+        TAG2STR(PDM);
+        TAG2STR(PDM_DEVICE);
+        TAG2STR(PDM_DEVICE_DESC);
+        TAG2STR(PDM_DEVICE_USER);
+        TAG2STR(PDM_DRIVER);
+        TAG2STR(PDM_DRIVER_DESC);
+        TAG2STR(PDM_DRIVER_USER);
+        TAG2STR(PDM_USB);
+        TAG2STR(PDM_USB_DESC);
+        TAG2STR(PDM_USB_USER);
+        TAG2STR(PDM_LUN);
+        TAG2STR(PDM_QUEUE);
+        TAG2STR(PDM_THREAD);
+        TAG2STR(PDM_ASYNC_COMPLETION);
+#ifdef VBOX_WITH_NETSHAPER
+        TAG2STR(PDM_NET_SHAPER);
+#endif /* VBOX_WITH_NETSHAPER */
+
+        TAG2STR(PGM);
+        TAG2STR(PGM_CHUNK_MAPPING);
+        TAG2STR(PGM_HANDLERS);
+        TAG2STR(PGM_HANDLER_TYPES);
+        TAG2STR(PGM_MAPPINGS);
+        TAG2STR(PGM_PHYS);
+        TAG2STR(PGM_POOL);
+
+        TAG2STR(REM);
+
+        TAG2STR(SELM);
+
+        TAG2STR(SSM);
+
+        TAG2STR(STAM);
+
+        TAG2STR(TM);
+
+        TAG2STR(TRPM);
+
+        TAG2STR(VM);
+        TAG2STR(VM_REQ);
+
+        TAG2STR(VMM);
+
+        TAG2STR(HM);
+
+        #undef TAG2STR
+
+        default:
+        {
+            AssertMsgFailed(("Unknown tag %d! forgot to add it to the switch?\n", enmTag));
+#ifdef IN_RING3
+            static char sz[48];
+            RTStrPrintf(sz, sizeof(sz), "%d", enmTag);
+            return sz;
+#else
+            return "unknown tag!";
+#endif
+        }
+    }
+}
+
diff --git a/src/VBox/VMM/VMMAll/Makefile.kup b/src/VBox/VMM/VMMAll/Makefile.kup
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/Makefile.kup
diff --git a/src/VBox/VMM/VMMAll/NEMAll.cpp b/src/VBox/VMM/VMMAll/NEMAll.cpp
new file mode 100644
index 00000000..1a44e573
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/NEMAll.cpp
@@ -0,0 +1,136 @@
+/* $Id: NEMAll.cpp $ */
+/** @file
+ * NEM - Native execution manager, R0 and R3 context code.
+ */
+
+/*
+ * Copyright (C) 2018-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_NEM
+#include <VBox/vmm/nem.h>
+#include "NEMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+
+
+/**
+ * Checks if this VM is in NEM mode and is long-mode capable.
+ *
+ * Use VMR3IsLongModeAllowed() instead of this, when possible.
+ *
+ * @returns true if long mode is allowed, false otherwise.
+ * @param   pVM         The cross context VM structure.
+ * @sa      VMR3IsLongModeAllowed, HMIsLongModeAllowed
+ */
+VMM_INT_DECL(bool) NEMHCIsLongModeAllowed(PVMCC pVM)
+{
+    return pVM->nem.s.fAllow64BitGuests && VM_IS_NEM_ENABLED(pVM);
+}
+
+
+/**
+ * Physical access handler registration notification.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   enmKind     The kind of access handler.
+ * @param   GCPhys      Start of the access handling range.
+ * @param   cb          Length of the access handling range.
+ *
+ * @note    Called while holding down the PGM lock.
+ */
+VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (VM_IS_NEM_ENABLED(pVM))
+        nemHCNativeNotifyHandlerPhysicalRegister(pVM, enmKind, GCPhys, cb);
+#else
+    RT_NOREF(pVM, enmKind, GCPhys, cb);
+#endif
+}
+
+
+VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
+                                                    RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    if (VM_IS_NEM_ENABLED(pVM))
+        nemHCNativeNotifyHandlerPhysicalModify(pVM, enmKind, GCPhysOld, GCPhysNew, cb, fRestoreAsRAM);
+#else
+    RT_NOREF(pVM, enmKind, GCPhysOld, GCPhysNew, cb, fRestoreAsRAM);
+#endif
+}
+
+
+VMM_INT_DECL(int)  NEMHCNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
+                                                PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    Assert(VM_IS_NEM_ENABLED(pVM));
+#ifdef VBOX_WITH_NATIVE_NEM
+    return nemHCNativeNotifyPhysPageAllocated(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State);
+#else
+    RT_NOREF(pVM, GCPhys, HCPhys, fPageProt, enmType, pu2State);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+#ifndef VBOX_WITH_NATIVE_NEM
+VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
+{
+    RT_NOREF(pVM);
+    return 0;
+}
+#endif
+
+
+#ifndef VBOX_WITH_NATIVE_NEM
+VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
+{
+    RT_NOREF(pVCpu, fWhat);
+    return VERR_NEM_IPE_9;
+}
+#endif
+
+
+#ifndef VBOX_WITH_NATIVE_NEM
+VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t *pcTicks, uint32_t *puAux)
+{
+    RT_NOREF(pVCpu, pcTicks, puAux);
+    AssertFailed();
+    return VERR_NEM_IPE_9;
+}
+#endif
+
+
+#ifndef VBOX_WITH_NATIVE_NEM
+VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
+{
+    RT_NOREF(pVM, pVCpu, uPausedTscValue);
+    AssertFailed();
+    return VERR_NEM_IPE_9;
+}
+#endif
+
diff --git a/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h b/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h
new file mode 100644
index 00000000..f3493079
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/NEMAllNativeTemplate-win.cpp.h
@@ -0,0 +1,3038 @@
+/* $Id: NEMAllNativeTemplate-win.cpp.h $ */
+/** @file
+ * NEM - Native execution manager, Windows code template ring-0/3.
+ */
+
+/*
+ * Copyright (C) 2018-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#ifndef IN_RING3
+# error "This is ring-3 only now"
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** Copy back a segment from hyper-V. */
+#define NEM_WIN_COPY_BACK_SEG(a_Dst, a_Src) \
+            do { \
+                (a_Dst).u64Base  = (a_Src).Base; \
+                (a_Dst).u32Limit = (a_Src).Limit; \
+                (a_Dst).ValidSel = (a_Dst).Sel = (a_Src).Selector; \
+                (a_Dst).Attr.u   = (a_Src).Attributes; \
+                (a_Dst).fFlags   = CPUMSELREG_FLAGS_VALID; \
+            } while (0)
+
+/** @def NEMWIN_ASSERT_MSG_REG_VAL
+ * Asserts the correctness of a register value in a message/context.
+ */
+#if 0
+# define NEMWIN_NEED_GET_REGISTER
+# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) \
+    do { \
+        WHV_REGISTER_VALUE TmpVal; \
+        nemR3WinGetRegister(a_pVCpu, a_enmReg, &TmpVal); \
+        AssertMsg(a_Expr, a_Msg); \
+    } while (0)
+#else
+# define NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, a_Expr, a_Msg) do { } while (0)
+#endif
+
+/** @def NEMWIN_ASSERT_MSG_REG_VAL
+ * Asserts the correctness of a 64-bit register value in a message/context.
+ */
+#define NEMWIN_ASSERT_MSG_REG_VAL64(a_pVCpu, a_enmReg, a_u64Val) \
+    NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, (a_u64Val) == TmpVal.Reg64, \
+                              (#a_u64Val "=%#RX64, expected %#RX64\n", (a_u64Val), TmpVal.Reg64))
+/** @def NEMWIN_ASSERT_MSG_REG_VAL
+ * Asserts the correctness of a segment register value in a message/context.
+ */
+#define NEMWIN_ASSERT_MSG_REG_SEG(a_pVCpu, a_enmReg, a_SReg) \
+    NEMWIN_ASSERT_MSG_REG_VAL(a_pVCpu, a_enmReg, \
+                                 (a_SReg).Base       == TmpVal.Segment.Base \
+                              && (a_SReg).Limit      == TmpVal.Segment.Limit \
+                              && (a_SReg).Selector   == TmpVal.Segment.Selector \
+                              && (a_SReg).Attributes == TmpVal.Segment.Attributes, \
+                              ( #a_SReg "=%#RX16 {%#RX64 LB %#RX32,%#RX16} expected %#RX16 {%#RX64 LB %#RX32,%#RX16}\n", \
+                               (a_SReg).Selector, (a_SReg).Base, (a_SReg).Limit, (a_SReg).Attributes, \
+                               TmpVal.Segment.Selector, TmpVal.Segment.Base, TmpVal.Segment.Limit, TmpVal.Segment.Attributes))
+
+
+#ifndef NTDDI_WIN10_19H1
+# define NTDDI_WIN10_19H1           0x0a000007
+#endif
+
+/** WHvRegisterPendingEvent0 was renamed to WHvRegisterPendingEvent between
+ *  SDK 17134 and 18362. */
+#if WDK_NTDDI_VERSION < NTDDI_WIN10_19H1
+# define WHvRegisterPendingEvent    WHvRegisterPendingEvent0
+#endif
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** NEM_WIN_PAGE_STATE_XXX names. */
+NEM_TMPL_STATIC const char * const g_apszPageStates[4] = { "not-set", "unmapped", "readable", "writable" };
+
+/** HV_INTERCEPT_ACCESS_TYPE names. */
+static const char * const g_apszHvInterceptAccessTypes[4] = { "read", "write", "exec", "!undefined!" };
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
+                                           uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged);
+
+
+
+NEM_TMPL_STATIC int nemHCWinCopyStateToHyperV(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    /*
+     * The following is very similar to what nemR0WinExportState() does.
+     */
+    WHV_REGISTER_NAME  aenmNames[128];
+    WHV_REGISTER_VALUE aValues[128];
+
+    uint64_t const fWhat = ~pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK);
+    if (   !fWhat
+        && pVCpu->nem.s.fCurrentInterruptWindows == pVCpu->nem.s.fDesiredInterruptWindows)
+        return VINF_SUCCESS;
+    uintptr_t iReg = 0;
+
+#define ADD_REG64(a_enmName, a_uValue) do { \
+            aenmNames[iReg]      = (a_enmName); \
+            aValues[iReg].Reg128.High64 = 0; \
+            aValues[iReg].Reg64  = (a_uValue); \
+            iReg++; \
+        } while (0)
+#define ADD_REG128(a_enmName, a_uValueLo, a_uValueHi) do { \
+            aenmNames[iReg] = (a_enmName); \
+            aValues[iReg].Reg128.Low64  = (a_uValueLo); \
+            aValues[iReg].Reg128.High64 = (a_uValueHi); \
+            iReg++; \
+        } while (0)
+
+    /* GPRs */
+    if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_RAX)
+            ADD_REG64(WHvX64RegisterRax, pVCpu->cpum.GstCtx.rax);
+        if (fWhat & CPUMCTX_EXTRN_RCX)
+            ADD_REG64(WHvX64RegisterRcx, pVCpu->cpum.GstCtx.rcx);
+        if (fWhat & CPUMCTX_EXTRN_RDX)
+            ADD_REG64(WHvX64RegisterRdx, pVCpu->cpum.GstCtx.rdx);
+        if (fWhat & CPUMCTX_EXTRN_RBX)
+            ADD_REG64(WHvX64RegisterRbx, pVCpu->cpum.GstCtx.rbx);
+        if (fWhat & CPUMCTX_EXTRN_RSP)
+            ADD_REG64(WHvX64RegisterRsp, pVCpu->cpum.GstCtx.rsp);
+        if (fWhat & CPUMCTX_EXTRN_RBP)
+            ADD_REG64(WHvX64RegisterRbp, pVCpu->cpum.GstCtx.rbp);
+        if (fWhat & CPUMCTX_EXTRN_RSI)
+            ADD_REG64(WHvX64RegisterRsi, pVCpu->cpum.GstCtx.rsi);
+        if (fWhat & CPUMCTX_EXTRN_RDI)
+            ADD_REG64(WHvX64RegisterRdi, pVCpu->cpum.GstCtx.rdi);
+        if (fWhat & CPUMCTX_EXTRN_R8_R15)
+        {
+            ADD_REG64(WHvX64RegisterR8, pVCpu->cpum.GstCtx.r8);
+            ADD_REG64(WHvX64RegisterR9, pVCpu->cpum.GstCtx.r9);
+            ADD_REG64(WHvX64RegisterR10, pVCpu->cpum.GstCtx.r10);
+            ADD_REG64(WHvX64RegisterR11, pVCpu->cpum.GstCtx.r11);
+            ADD_REG64(WHvX64RegisterR12, pVCpu->cpum.GstCtx.r12);
+            ADD_REG64(WHvX64RegisterR13, pVCpu->cpum.GstCtx.r13);
+            ADD_REG64(WHvX64RegisterR14, pVCpu->cpum.GstCtx.r14);
+            ADD_REG64(WHvX64RegisterR15, pVCpu->cpum.GstCtx.r15);
+        }
+    }
+
+    /* RIP & Flags */
+    if (fWhat & CPUMCTX_EXTRN_RIP)
+        ADD_REG64(WHvX64RegisterRip, pVCpu->cpum.GstCtx.rip);
+    if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+        ADD_REG64(WHvX64RegisterRflags, pVCpu->cpum.GstCtx.rflags.u);
+
+    /* Segments */
+#define ADD_SEG(a_enmName, a_SReg) \
+        do { \
+            aenmNames[iReg]                  = a_enmName; \
+            aValues[iReg].Segment.Base       = (a_SReg).u64Base; \
+            aValues[iReg].Segment.Limit      = (a_SReg).u32Limit; \
+            aValues[iReg].Segment.Selector   = (a_SReg).Sel; \
+            aValues[iReg].Segment.Attributes = (a_SReg).Attr.u; \
+            iReg++; \
+        } while (0)
+    if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_ES)
+            ADD_SEG(WHvX64RegisterEs,   pVCpu->cpum.GstCtx.es);
+        if (fWhat & CPUMCTX_EXTRN_CS)
+            ADD_SEG(WHvX64RegisterCs,   pVCpu->cpum.GstCtx.cs);
+        if (fWhat & CPUMCTX_EXTRN_SS)
+            ADD_SEG(WHvX64RegisterSs,   pVCpu->cpum.GstCtx.ss);
+        if (fWhat & CPUMCTX_EXTRN_DS)
+            ADD_SEG(WHvX64RegisterDs,   pVCpu->cpum.GstCtx.ds);
+        if (fWhat & CPUMCTX_EXTRN_FS)
+            ADD_SEG(WHvX64RegisterFs,   pVCpu->cpum.GstCtx.fs);
+        if (fWhat & CPUMCTX_EXTRN_GS)
+            ADD_SEG(WHvX64RegisterGs,   pVCpu->cpum.GstCtx.gs);
+    }
+
+    /* Descriptor tables & task segment. */
+    if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_LDTR)
+            ADD_SEG(WHvX64RegisterLdtr, pVCpu->cpum.GstCtx.ldtr);
+        if (fWhat & CPUMCTX_EXTRN_TR)
+            ADD_SEG(WHvX64RegisterTr,   pVCpu->cpum.GstCtx.tr);
+        if (fWhat & CPUMCTX_EXTRN_IDTR)
+        {
+            aenmNames[iReg] = WHvX64RegisterIdtr;
+            aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.idtr.cbIdt;
+            aValues[iReg].Table.Base  = pVCpu->cpum.GstCtx.idtr.pIdt;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_GDTR)
+        {
+            aenmNames[iReg] = WHvX64RegisterGdtr;
+            aValues[iReg].Table.Limit = pVCpu->cpum.GstCtx.gdtr.cbGdt;
+            aValues[iReg].Table.Base  = pVCpu->cpum.GstCtx.gdtr.pGdt;
+            iReg++;
+        }
+    }
+
+    /* Control registers. */
+    if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CR0)
+            ADD_REG64(WHvX64RegisterCr0, pVCpu->cpum.GstCtx.cr0);
+        if (fWhat & CPUMCTX_EXTRN_CR2)
+            ADD_REG64(WHvX64RegisterCr2, pVCpu->cpum.GstCtx.cr2);
+        if (fWhat & CPUMCTX_EXTRN_CR3)
+            ADD_REG64(WHvX64RegisterCr3, pVCpu->cpum.GstCtx.cr3);
+        if (fWhat & CPUMCTX_EXTRN_CR4)
+            ADD_REG64(WHvX64RegisterCr4, pVCpu->cpum.GstCtx.cr4);
+    }
+    if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
+        ADD_REG64(WHvX64RegisterCr8, CPUMGetGuestCR8(pVCpu));
+
+    /* Debug registers. */
+/** @todo fixme. Figure out what the hyper-v version of KVM_SET_GUEST_DEBUG would be. */
+    if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
+    {
+        ADD_REG64(WHvX64RegisterDr0, pVCpu->cpum.GstCtx.dr[0]); // CPUMGetHyperDR0(pVCpu));
+        ADD_REG64(WHvX64RegisterDr1, pVCpu->cpum.GstCtx.dr[1]); // CPUMGetHyperDR1(pVCpu));
+        ADD_REG64(WHvX64RegisterDr2, pVCpu->cpum.GstCtx.dr[2]); // CPUMGetHyperDR2(pVCpu));
+        ADD_REG64(WHvX64RegisterDr3, pVCpu->cpum.GstCtx.dr[3]); // CPUMGetHyperDR3(pVCpu));
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR6)
+        ADD_REG64(WHvX64RegisterDr6, pVCpu->cpum.GstCtx.dr[6]); // CPUMGetHyperDR6(pVCpu));
+    if (fWhat & CPUMCTX_EXTRN_DR7)
+        ADD_REG64(WHvX64RegisterDr7, pVCpu->cpum.GstCtx.dr[7]); // CPUMGetHyperDR7(pVCpu));
+
+    /* Floating point state. */
+    if (fWhat & CPUMCTX_EXTRN_X87)
+    {
+        ADD_REG128(WHvX64RegisterFpMmx0, pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx1, pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx2, pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx3, pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx4, pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx5, pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx6, pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[1]);
+        ADD_REG128(WHvX64RegisterFpMmx7, pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[1]);
+
+        aenmNames[iReg] = WHvX64RegisterFpControlStatus;
+        aValues[iReg].FpControlStatus.FpControl = pVCpu->cpum.GstCtx.XState.x87.FCW;
+        aValues[iReg].FpControlStatus.FpStatus  = pVCpu->cpum.GstCtx.XState.x87.FSW;
+        aValues[iReg].FpControlStatus.FpTag     = pVCpu->cpum.GstCtx.XState.x87.FTW;
+        aValues[iReg].FpControlStatus.Reserved  = pVCpu->cpum.GstCtx.XState.x87.FTW >> 8;
+        aValues[iReg].FpControlStatus.LastFpOp  = pVCpu->cpum.GstCtx.XState.x87.FOP;
+        aValues[iReg].FpControlStatus.LastFpRip = (pVCpu->cpum.GstCtx.XState.x87.FPUIP)
+                                                | ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.CS << 32)
+                                                | ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.Rsrvd1 << 48);
+        iReg++;
+
+        aenmNames[iReg] = WHvX64RegisterXmmControlStatus;
+        aValues[iReg].XmmControlStatus.LastFpRdp            = (pVCpu->cpum.GstCtx.XState.x87.FPUDP)
+                                                            | ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.DS << 32)
+                                                            | ((uint64_t)pVCpu->cpum.GstCtx.XState.x87.Rsrvd2 << 48);
+        aValues[iReg].XmmControlStatus.XmmStatusControl     = pVCpu->cpum.GstCtx.XState.x87.MXCSR;
+        aValues[iReg].XmmControlStatus.XmmStatusControlMask = pVCpu->cpum.GstCtx.XState.x87.MXCSR_MASK; /** @todo ??? (Isn't this an output field?) */
+        iReg++;
+    }
+
+    /* Vector state. */
+    if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
+    {
+        ADD_REG128(WHvX64RegisterXmm0,  pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm1,  pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm2,  pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm3,  pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm4,  pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm5,  pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm6,  pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm7,  pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm8,  pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm9,  pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm10, pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm11, pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm12, pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm13, pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm14, pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Hi);
+        ADD_REG128(WHvX64RegisterXmm15, pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Hi);
+    }
+
+    /* MSRs */
+    // WHvX64RegisterTsc - don't touch
+    if (fWhat & CPUMCTX_EXTRN_EFER)
+        ADD_REG64(WHvX64RegisterEfer, pVCpu->cpum.GstCtx.msrEFER);
+    if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+        ADD_REG64(WHvX64RegisterKernelGsBase, pVCpu->cpum.GstCtx.msrKERNELGSBASE);
+    if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+    {
+        ADD_REG64(WHvX64RegisterSysenterCs, pVCpu->cpum.GstCtx.SysEnter.cs);
+        ADD_REG64(WHvX64RegisterSysenterEip, pVCpu->cpum.GstCtx.SysEnter.eip);
+        ADD_REG64(WHvX64RegisterSysenterEsp, pVCpu->cpum.GstCtx.SysEnter.esp);
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+    {
+        ADD_REG64(WHvX64RegisterStar, pVCpu->cpum.GstCtx.msrSTAR);
+        ADD_REG64(WHvX64RegisterLstar, pVCpu->cpum.GstCtx.msrLSTAR);
+        ADD_REG64(WHvX64RegisterCstar, pVCpu->cpum.GstCtx.msrCSTAR);
+        ADD_REG64(WHvX64RegisterSfmask, pVCpu->cpum.GstCtx.msrSFMASK);
+    }
+    if (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
+    {
+        PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
+        if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
+            ADD_REG64(WHvX64RegisterTscAux, pCtxMsrs->msr.TscAux);
+        if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
+        {
+            ADD_REG64(WHvX64RegisterApicBase, APICGetBaseMsrNoCheck(pVCpu));
+            ADD_REG64(WHvX64RegisterPat, pVCpu->cpum.GstCtx.msrPAT);
+#if 0 /** @todo check if WHvX64RegisterMsrMtrrCap works here... */
+            ADD_REG64(WHvX64RegisterMsrMtrrCap, CPUMGetGuestIa32MtrrCap(pVCpu));
+#endif
+            ADD_REG64(WHvX64RegisterMsrMtrrDefType, pCtxMsrs->msr.MtrrDefType);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix64k00000, pCtxMsrs->msr.MtrrFix64K_00000);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix16k80000, pCtxMsrs->msr.MtrrFix16K_80000);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix16kA0000, pCtxMsrs->msr.MtrrFix16K_A0000);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix4kC0000,  pCtxMsrs->msr.MtrrFix4K_C0000);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix4kC8000,  pCtxMsrs->msr.MtrrFix4K_C8000);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix4kD0000,  pCtxMsrs->msr.MtrrFix4K_D0000);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix4kD8000,  pCtxMsrs->msr.MtrrFix4K_D8000);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix4kE0000,  pCtxMsrs->msr.MtrrFix4K_E0000);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix4kE8000,  pCtxMsrs->msr.MtrrFix4K_E8000);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix4kF0000,  pCtxMsrs->msr.MtrrFix4K_F0000);
+            ADD_REG64(WHvX64RegisterMsrMtrrFix4kF8000,  pCtxMsrs->msr.MtrrFix4K_F8000);
+#if 0 /** @todo these registers aren't available? Might explain something.. .*/
+            const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
+            if (enmCpuVendor != CPUMCPUVENDOR_AMD)
+            {
+                ADD_REG64(HvX64RegisterIa32MiscEnable, pCtxMsrs->msr.MiscEnable);
+                ADD_REG64(HvX64RegisterIa32FeatureControl, CPUMGetGuestIa32FeatureControl(pVCpu));
+            }
+#endif
+        }
+    }
+
+    /* event injection (clear it). */
+    if (fWhat & CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)
+        ADD_REG64(WHvRegisterPendingInterruption, 0);
+
+    /* Interruptibility state.  This can get a little complicated since we get
+       half of the state via HV_X64_VP_EXECUTION_STATE. */
+    if (   (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
+        ==          (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI) )
+    {
+        ADD_REG64(WHvRegisterInterruptState, 0);
+        if (CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
+            aValues[iReg - 1].InterruptState.InterruptShadow = 1;
+        aValues[iReg - 1].InterruptState.NmiMasked = CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx);
+    }
+    else if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
+    {
+        if (   pVCpu->nem.s.fLastInterruptShadow
+            || CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
+        {
+            ADD_REG64(WHvRegisterInterruptState, 0);
+            if (CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx))
+                aValues[iReg - 1].InterruptState.InterruptShadow = 1;
+            /** @todo Retrieve NMI state, currently assuming it's zero. (yes this may happen on I/O) */
+            //if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
+            //    aValues[iReg - 1].InterruptState.NmiMasked = 1;
+        }
+    }
+    else
+        Assert(!(fWhat & CPUMCTX_EXTRN_INHIBIT_NMI));
+
+    /* Interrupt windows. Always set if active as Hyper-V seems to be forgetful. */
+    uint8_t const fDesiredIntWin = pVCpu->nem.s.fDesiredInterruptWindows;
+    if (   fDesiredIntWin
+        || pVCpu->nem.s.fCurrentInterruptWindows != fDesiredIntWin)
+    {
+        pVCpu->nem.s.fCurrentInterruptWindows = pVCpu->nem.s.fDesiredInterruptWindows;
+        Log8(("Setting WHvX64RegisterDeliverabilityNotifications, fDesiredIntWin=%X\n", fDesiredIntWin));
+        ADD_REG64(WHvX64RegisterDeliverabilityNotifications, fDesiredIntWin);
+        Assert(aValues[iReg - 1].DeliverabilityNotifications.NmiNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_NMI));
+        Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptNotification == RT_BOOL(fDesiredIntWin & NEM_WIN_INTW_F_REGULAR));
+        Assert(aValues[iReg - 1].DeliverabilityNotifications.InterruptPriority == (unsigned)((fDesiredIntWin & NEM_WIN_INTW_F_PRIO_MASK) >> NEM_WIN_INTW_F_PRIO_SHIFT));
+    }
+
+    /// @todo WHvRegisterPendingEvent
+
+#undef ADD_REG64
+#undef ADD_REG128
+#undef ADD_SEG
+
+    /*
+     * Set the registers.
+     */
+    Assert(iReg < RT_ELEMENTS(aValues));
+    Assert(iReg < RT_ELEMENTS(aenmNames));
+#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+    Log12(("Calling WHvSetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
+           pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues));
+#endif
+    HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, iReg, aValues);
+    if (SUCCEEDED(hrc))
+    {
+        pVCpu->cpum.GstCtx.fExtrn |= CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK | CPUMCTX_EXTRN_KEEPER_NEM;
+        return VINF_SUCCESS;
+    }
+    AssertLogRelMsgFailed(("WHvSetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
+                           pVM->nem.s.hPartition, pVCpu->idCpu, iReg,
+                           hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+    return VERR_INTERNAL_ERROR;
+}
+
+
+NEM_TMPL_STATIC int nemHCWinCopyStateFromHyperV(PVMCC pVM, PVMCPUCC pVCpu, uint64_t fWhat)
+{
+    WHV_REGISTER_NAME  aenmNames[128];
+
+    fWhat &= pVCpu->cpum.GstCtx.fExtrn;
+    uintptr_t iReg = 0;
+
+    /* GPRs */
+    if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_RAX)
+            aenmNames[iReg++] = WHvX64RegisterRax;
+        if (fWhat & CPUMCTX_EXTRN_RCX)
+            aenmNames[iReg++] = WHvX64RegisterRcx;
+        if (fWhat & CPUMCTX_EXTRN_RDX)
+            aenmNames[iReg++] = WHvX64RegisterRdx;
+        if (fWhat & CPUMCTX_EXTRN_RBX)
+            aenmNames[iReg++] = WHvX64RegisterRbx;
+        if (fWhat & CPUMCTX_EXTRN_RSP)
+            aenmNames[iReg++] = WHvX64RegisterRsp;
+        if (fWhat & CPUMCTX_EXTRN_RBP)
+            aenmNames[iReg++] = WHvX64RegisterRbp;
+        if (fWhat & CPUMCTX_EXTRN_RSI)
+            aenmNames[iReg++] = WHvX64RegisterRsi;
+        if (fWhat & CPUMCTX_EXTRN_RDI)
+            aenmNames[iReg++] = WHvX64RegisterRdi;
+        if (fWhat & CPUMCTX_EXTRN_R8_R15)
+        {
+            aenmNames[iReg++] = WHvX64RegisterR8;
+            aenmNames[iReg++] = WHvX64RegisterR9;
+            aenmNames[iReg++] = WHvX64RegisterR10;
+            aenmNames[iReg++] = WHvX64RegisterR11;
+            aenmNames[iReg++] = WHvX64RegisterR12;
+            aenmNames[iReg++] = WHvX64RegisterR13;
+            aenmNames[iReg++] = WHvX64RegisterR14;
+            aenmNames[iReg++] = WHvX64RegisterR15;
+        }
+    }
+
+    /* RIP & Flags */
+    if (fWhat & CPUMCTX_EXTRN_RIP)
+        aenmNames[iReg++] = WHvX64RegisterRip;
+    if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+        aenmNames[iReg++] = WHvX64RegisterRflags;
+
+    /* Segments */
+    if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_ES)
+            aenmNames[iReg++] = WHvX64RegisterEs;
+        if (fWhat & CPUMCTX_EXTRN_CS)
+            aenmNames[iReg++] = WHvX64RegisterCs;
+        if (fWhat & CPUMCTX_EXTRN_SS)
+            aenmNames[iReg++] = WHvX64RegisterSs;
+        if (fWhat & CPUMCTX_EXTRN_DS)
+            aenmNames[iReg++] = WHvX64RegisterDs;
+        if (fWhat & CPUMCTX_EXTRN_FS)
+            aenmNames[iReg++] = WHvX64RegisterFs;
+        if (fWhat & CPUMCTX_EXTRN_GS)
+            aenmNames[iReg++] = WHvX64RegisterGs;
+    }
+
+    /* Descriptor tables. */
+    if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_LDTR)
+            aenmNames[iReg++] = WHvX64RegisterLdtr;
+        if (fWhat & CPUMCTX_EXTRN_TR)
+            aenmNames[iReg++] = WHvX64RegisterTr;
+        if (fWhat & CPUMCTX_EXTRN_IDTR)
+            aenmNames[iReg++] = WHvX64RegisterIdtr;
+        if (fWhat & CPUMCTX_EXTRN_GDTR)
+            aenmNames[iReg++] = WHvX64RegisterGdtr;
+    }
+
+    /* Control registers. */
+    if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CR0)
+            aenmNames[iReg++] = WHvX64RegisterCr0;
+        if (fWhat & CPUMCTX_EXTRN_CR2)
+            aenmNames[iReg++] = WHvX64RegisterCr2;
+        if (fWhat & CPUMCTX_EXTRN_CR3)
+            aenmNames[iReg++] = WHvX64RegisterCr3;
+        if (fWhat & CPUMCTX_EXTRN_CR4)
+            aenmNames[iReg++] = WHvX64RegisterCr4;
+    }
+    if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
+        aenmNames[iReg++] = WHvX64RegisterCr8;
+
+    /* Debug registers. */
+    if (fWhat & CPUMCTX_EXTRN_DR7)
+        aenmNames[iReg++] = WHvX64RegisterDr7;
+    if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
+    {
+        if (!(fWhat & CPUMCTX_EXTRN_DR7) && (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_DR7))
+        {
+            fWhat |= CPUMCTX_EXTRN_DR7;
+            aenmNames[iReg++] = WHvX64RegisterDr7;
+        }
+        aenmNames[iReg++] = WHvX64RegisterDr0;
+        aenmNames[iReg++] = WHvX64RegisterDr1;
+        aenmNames[iReg++] = WHvX64RegisterDr2;
+        aenmNames[iReg++] = WHvX64RegisterDr3;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR6)
+        aenmNames[iReg++] = WHvX64RegisterDr6;
+
+    /* Floating point state. */
+    if (fWhat & CPUMCTX_EXTRN_X87)
+    {
+        aenmNames[iReg++] = WHvX64RegisterFpMmx0;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx1;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx2;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx3;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx4;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx5;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx6;
+        aenmNames[iReg++] = WHvX64RegisterFpMmx7;
+        aenmNames[iReg++] = WHvX64RegisterFpControlStatus;
+    }
+    if (fWhat & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX))
+        aenmNames[iReg++] = WHvX64RegisterXmmControlStatus;
+
+    /* Vector state. */
+    if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
+    {
+        aenmNames[iReg++] = WHvX64RegisterXmm0;
+        aenmNames[iReg++] = WHvX64RegisterXmm1;
+        aenmNames[iReg++] = WHvX64RegisterXmm2;
+        aenmNames[iReg++] = WHvX64RegisterXmm3;
+        aenmNames[iReg++] = WHvX64RegisterXmm4;
+        aenmNames[iReg++] = WHvX64RegisterXmm5;
+        aenmNames[iReg++] = WHvX64RegisterXmm6;
+        aenmNames[iReg++] = WHvX64RegisterXmm7;
+        aenmNames[iReg++] = WHvX64RegisterXmm8;
+        aenmNames[iReg++] = WHvX64RegisterXmm9;
+        aenmNames[iReg++] = WHvX64RegisterXmm10;
+        aenmNames[iReg++] = WHvX64RegisterXmm11;
+        aenmNames[iReg++] = WHvX64RegisterXmm12;
+        aenmNames[iReg++] = WHvX64RegisterXmm13;
+        aenmNames[iReg++] = WHvX64RegisterXmm14;
+        aenmNames[iReg++] = WHvX64RegisterXmm15;
+    }
+
+    /* MSRs */
+    // WHvX64RegisterTsc - don't touch
+    if (fWhat & CPUMCTX_EXTRN_EFER)
+        aenmNames[iReg++] = WHvX64RegisterEfer;
+    if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+        aenmNames[iReg++] = WHvX64RegisterKernelGsBase;
+    if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+    {
+        aenmNames[iReg++] = WHvX64RegisterSysenterCs;
+        aenmNames[iReg++] = WHvX64RegisterSysenterEip;
+        aenmNames[iReg++] = WHvX64RegisterSysenterEsp;
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+    {
+        aenmNames[iReg++] = WHvX64RegisterStar;
+        aenmNames[iReg++] = WHvX64RegisterLstar;
+        aenmNames[iReg++] = WHvX64RegisterCstar;
+        aenmNames[iReg++] = WHvX64RegisterSfmask;
+    }
+
+//#ifdef LOG_ENABLED
+//    const CPUMCPUVENDOR enmCpuVendor = CPUMGetHostCpuVendor(pVM);
+//#endif
+    if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
+        aenmNames[iReg++] = WHvX64RegisterTscAux;
+    if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
+    {
+        aenmNames[iReg++] = WHvX64RegisterApicBase; /// @todo APIC BASE
+        aenmNames[iReg++] = WHvX64RegisterPat;
+#if 0 /*def LOG_ENABLED*/ /** @todo Check if WHvX64RegisterMsrMtrrCap works... */
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrCap;
+#endif
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrDefType;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix64k00000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16k80000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix16kA0000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC0000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kC8000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD0000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kD8000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE0000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kE8000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF0000;
+        aenmNames[iReg++] = WHvX64RegisterMsrMtrrFix4kF8000;
+        /** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
+//#ifdef LOG_ENABLED
+//        if (enmCpuVendor != CPUMCPUVENDOR_AMD)
+//            aenmNames[iReg++] = HvX64RegisterIa32FeatureControl;
+//#endif
+    }
+
+    /* Interruptibility. */
+    if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
+    {
+        aenmNames[iReg++] = WHvRegisterInterruptState;
+        aenmNames[iReg++] = WHvX64RegisterRip;
+    }
+
+    /* event injection */
+    aenmNames[iReg++] = WHvRegisterPendingInterruption;
+    aenmNames[iReg++] = WHvRegisterPendingEvent;
+
+    size_t const cRegs = iReg;
+    Assert(cRegs < RT_ELEMENTS(aenmNames));
+
+    /*
+     * Get the registers.
+     */
+    WHV_REGISTER_VALUE aValues[128];
+    RT_ZERO(aValues);
+    Assert(RT_ELEMENTS(aValues) >= cRegs);
+    Assert(RT_ELEMENTS(aenmNames) >= cRegs);
+#ifdef NEM_WIN_INTERCEPT_NT_IO_CTLS
+    Log12(("Calling WHvGetVirtualProcessorRegisters(%p, %u, %p, %u, %p)\n",
+          pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, cRegs, aValues));
+#endif
+    HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, (uint32_t)cRegs, aValues);
+    AssertLogRelMsgReturn(SUCCEEDED(hrc),
+                          ("WHvGetVirtualProcessorRegisters(%p, %u,,%u,) -> %Rhrc (Last=%#x/%u)\n",
+                           pVM->nem.s.hPartition, pVCpu->idCpu, cRegs, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
+                          , VERR_NEM_GET_REGISTERS_FAILED);
+
+    iReg = 0;
+#define GET_REG64(a_DstVar, a_enmName) do { \
+            Assert(aenmNames[iReg] == (a_enmName)); \
+            (a_DstVar) = aValues[iReg].Reg64; \
+            iReg++; \
+        } while (0)
+#define GET_REG64_LOG7(a_DstVar, a_enmName, a_szLogName) do { \
+            Assert(aenmNames[iReg] == (a_enmName)); \
+            if ((a_DstVar) != aValues[iReg].Reg64) \
+                Log7(("NEM/%u: " a_szLogName " changed %RX64 -> %RX64\n", pVCpu->idCpu, (a_DstVar), aValues[iReg].Reg64)); \
+            (a_DstVar) = aValues[iReg].Reg64; \
+            iReg++; \
+        } while (0)
+#define GET_REG128(a_DstVarLo, a_DstVarHi, a_enmName) do { \
+            Assert(aenmNames[iReg] == a_enmName); \
+            (a_DstVarLo) = aValues[iReg].Reg128.Low64; \
+            (a_DstVarHi) = aValues[iReg].Reg128.High64; \
+            iReg++; \
+        } while (0)
+#define GET_SEG(a_SReg, a_enmName) do { \
+            Assert(aenmNames[iReg] == (a_enmName)); \
+            NEM_WIN_COPY_BACK_SEG(a_SReg, aValues[iReg].Segment); \
+            iReg++; \
+        } while (0)
+
+    /* GPRs */
+    if (fWhat & CPUMCTX_EXTRN_GPRS_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_RAX)
+            GET_REG64(pVCpu->cpum.GstCtx.rax, WHvX64RegisterRax);
+        if (fWhat & CPUMCTX_EXTRN_RCX)
+            GET_REG64(pVCpu->cpum.GstCtx.rcx, WHvX64RegisterRcx);
+        if (fWhat & CPUMCTX_EXTRN_RDX)
+            GET_REG64(pVCpu->cpum.GstCtx.rdx, WHvX64RegisterRdx);
+        if (fWhat & CPUMCTX_EXTRN_RBX)
+            GET_REG64(pVCpu->cpum.GstCtx.rbx, WHvX64RegisterRbx);
+        if (fWhat & CPUMCTX_EXTRN_RSP)
+            GET_REG64(pVCpu->cpum.GstCtx.rsp, WHvX64RegisterRsp);
+        if (fWhat & CPUMCTX_EXTRN_RBP)
+            GET_REG64(pVCpu->cpum.GstCtx.rbp, WHvX64RegisterRbp);
+        if (fWhat & CPUMCTX_EXTRN_RSI)
+            GET_REG64(pVCpu->cpum.GstCtx.rsi, WHvX64RegisterRsi);
+        if (fWhat & CPUMCTX_EXTRN_RDI)
+            GET_REG64(pVCpu->cpum.GstCtx.rdi, WHvX64RegisterRdi);
+        if (fWhat & CPUMCTX_EXTRN_R8_R15)
+        {
+            GET_REG64(pVCpu->cpum.GstCtx.r8, WHvX64RegisterR8);
+            GET_REG64(pVCpu->cpum.GstCtx.r9, WHvX64RegisterR9);
+            GET_REG64(pVCpu->cpum.GstCtx.r10, WHvX64RegisterR10);
+            GET_REG64(pVCpu->cpum.GstCtx.r11, WHvX64RegisterR11);
+            GET_REG64(pVCpu->cpum.GstCtx.r12, WHvX64RegisterR12);
+            GET_REG64(pVCpu->cpum.GstCtx.r13, WHvX64RegisterR13);
+            GET_REG64(pVCpu->cpum.GstCtx.r14, WHvX64RegisterR14);
+            GET_REG64(pVCpu->cpum.GstCtx.r15, WHvX64RegisterR15);
+        }
+    }
+
+    /* RIP & Flags */
+    if (fWhat & CPUMCTX_EXTRN_RIP)
+        GET_REG64(pVCpu->cpum.GstCtx.rip, WHvX64RegisterRip);
+    if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+        GET_REG64(pVCpu->cpum.GstCtx.rflags.u, WHvX64RegisterRflags);
+
+    /* Segments */
+    if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_ES)
+            GET_SEG(pVCpu->cpum.GstCtx.es, WHvX64RegisterEs);
+        if (fWhat & CPUMCTX_EXTRN_CS)
+            GET_SEG(pVCpu->cpum.GstCtx.cs, WHvX64RegisterCs);
+        if (fWhat & CPUMCTX_EXTRN_SS)
+            GET_SEG(pVCpu->cpum.GstCtx.ss, WHvX64RegisterSs);
+        if (fWhat & CPUMCTX_EXTRN_DS)
+            GET_SEG(pVCpu->cpum.GstCtx.ds, WHvX64RegisterDs);
+        if (fWhat & CPUMCTX_EXTRN_FS)
+            GET_SEG(pVCpu->cpum.GstCtx.fs, WHvX64RegisterFs);
+        if (fWhat & CPUMCTX_EXTRN_GS)
+            GET_SEG(pVCpu->cpum.GstCtx.gs, WHvX64RegisterGs);
+    }
+
+    /* Descriptor tables and the task segment. */
+    if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_LDTR)
+            GET_SEG(pVCpu->cpum.GstCtx.ldtr, WHvX64RegisterLdtr);
+
+        if (fWhat & CPUMCTX_EXTRN_TR)
+        {
+            /* AMD-V likes loading TR with in AVAIL state, whereas intel insists on BUSY.  So,
+               avoid to trigger sanity assertions around the code, always fix this. */
+            GET_SEG(pVCpu->cpum.GstCtx.tr, WHvX64RegisterTr);
+            switch (pVCpu->cpum.GstCtx.tr.Attr.n.u4Type)
+            {
+                case X86_SEL_TYPE_SYS_386_TSS_BUSY:
+                case X86_SEL_TYPE_SYS_286_TSS_BUSY:
+                    break;
+                case X86_SEL_TYPE_SYS_386_TSS_AVAIL:
+                    pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
+                    break;
+                case X86_SEL_TYPE_SYS_286_TSS_AVAIL:
+                    pVCpu->cpum.GstCtx.tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
+                    break;
+            }
+        }
+        if (fWhat & CPUMCTX_EXTRN_IDTR)
+        {
+            Assert(aenmNames[iReg] == WHvX64RegisterIdtr);
+            pVCpu->cpum.GstCtx.idtr.cbIdt = aValues[iReg].Table.Limit;
+            pVCpu->cpum.GstCtx.idtr.pIdt  = aValues[iReg].Table.Base;
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_GDTR)
+        {
+            Assert(aenmNames[iReg] == WHvX64RegisterGdtr);
+            pVCpu->cpum.GstCtx.gdtr.cbGdt = aValues[iReg].Table.Limit;
+            pVCpu->cpum.GstCtx.gdtr.pGdt  = aValues[iReg].Table.Base;
+            iReg++;
+        }
+    }
+
+    /* Control registers. */
+    bool fMaybeChangedMode = false;
+    bool fUpdateCr3        = false;
+    if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+    {
+        if (fWhat & CPUMCTX_EXTRN_CR0)
+        {
+            Assert(aenmNames[iReg] == WHvX64RegisterCr0);
+            if (pVCpu->cpum.GstCtx.cr0 != aValues[iReg].Reg64)
+            {
+                CPUMSetGuestCR0(pVCpu, aValues[iReg].Reg64);
+                fMaybeChangedMode = true;
+            }
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_CR2)
+            GET_REG64(pVCpu->cpum.GstCtx.cr2, WHvX64RegisterCr2);
+        if (fWhat & CPUMCTX_EXTRN_CR3)
+        {
+            if (pVCpu->cpum.GstCtx.cr3 != aValues[iReg].Reg64)
+            {
+                CPUMSetGuestCR3(pVCpu, aValues[iReg].Reg64);
+                fUpdateCr3 = true;
+            }
+            iReg++;
+        }
+        if (fWhat & CPUMCTX_EXTRN_CR4)
+        {
+            if (pVCpu->cpum.GstCtx.cr4 != aValues[iReg].Reg64)
+            {
+                CPUMSetGuestCR4(pVCpu, aValues[iReg].Reg64);
+                fMaybeChangedMode = true;
+            }
+            iReg++;
+        }
+    }
+    if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterCr8);
+        APICSetTpr(pVCpu, (uint8_t)aValues[iReg].Reg64 << 4);
+        iReg++;
+    }
+
+    /* Debug registers. */
+    if (fWhat & CPUMCTX_EXTRN_DR7)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterDr7);
+        if (pVCpu->cpum.GstCtx.dr[7] != aValues[iReg].Reg64)
+            CPUMSetGuestDR7(pVCpu, aValues[iReg].Reg64);
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_DR7; /* Hack alert! Avoids asserting when processing CPUMCTX_EXTRN_DR0_DR3. */
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterDr0);
+        Assert(aenmNames[iReg+3] == WHvX64RegisterDr3);
+        if (pVCpu->cpum.GstCtx.dr[0] != aValues[iReg].Reg64)
+            CPUMSetGuestDR0(pVCpu, aValues[iReg].Reg64);
+        iReg++;
+        if (pVCpu->cpum.GstCtx.dr[1] != aValues[iReg].Reg64)
+            CPUMSetGuestDR1(pVCpu, aValues[iReg].Reg64);
+        iReg++;
+        if (pVCpu->cpum.GstCtx.dr[2] != aValues[iReg].Reg64)
+            CPUMSetGuestDR2(pVCpu, aValues[iReg].Reg64);
+        iReg++;
+        if (pVCpu->cpum.GstCtx.dr[3] != aValues[iReg].Reg64)
+            CPUMSetGuestDR3(pVCpu, aValues[iReg].Reg64);
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_DR6)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterDr6);
+        if (pVCpu->cpum.GstCtx.dr[6] != aValues[iReg].Reg64)
+            CPUMSetGuestDR6(pVCpu, aValues[iReg].Reg64);
+        iReg++;
+    }
+
+    /* Floating point state. */
+    if (fWhat & CPUMCTX_EXTRN_X87)
+    {
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[0].au64[1], WHvX64RegisterFpMmx0);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[1].au64[1], WHvX64RegisterFpMmx1);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[2].au64[1], WHvX64RegisterFpMmx2);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[3].au64[1], WHvX64RegisterFpMmx3);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[4].au64[1], WHvX64RegisterFpMmx4);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[5].au64[1], WHvX64RegisterFpMmx5);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[6].au64[1], WHvX64RegisterFpMmx6);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[0], pVCpu->cpum.GstCtx.XState.x87.aRegs[7].au64[1], WHvX64RegisterFpMmx7);
+
+        Assert(aenmNames[iReg] == WHvX64RegisterFpControlStatus);
+        pVCpu->cpum.GstCtx.XState.x87.FCW        = aValues[iReg].FpControlStatus.FpControl;
+        pVCpu->cpum.GstCtx.XState.x87.FSW        = aValues[iReg].FpControlStatus.FpStatus;
+        pVCpu->cpum.GstCtx.XState.x87.FTW        = aValues[iReg].FpControlStatus.FpTag
+                                        /*| (aValues[iReg].FpControlStatus.Reserved << 8)*/;
+        pVCpu->cpum.GstCtx.XState.x87.FOP        = aValues[iReg].FpControlStatus.LastFpOp;
+        pVCpu->cpum.GstCtx.XState.x87.FPUIP      = (uint32_t)aValues[iReg].FpControlStatus.LastFpRip;
+        pVCpu->cpum.GstCtx.XState.x87.CS         = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 32);
+        pVCpu->cpum.GstCtx.XState.x87.Rsrvd1     = (uint16_t)(aValues[iReg].FpControlStatus.LastFpRip >> 48);
+        iReg++;
+    }
+
+    if (fWhat & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX))
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterXmmControlStatus);
+        if (fWhat & CPUMCTX_EXTRN_X87)
+        {
+            pVCpu->cpum.GstCtx.XState.x87.FPUDP  = (uint32_t)aValues[iReg].XmmControlStatus.LastFpRdp;
+            pVCpu->cpum.GstCtx.XState.x87.DS     = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 32);
+            pVCpu->cpum.GstCtx.XState.x87.Rsrvd2 = (uint16_t)(aValues[iReg].XmmControlStatus.LastFpRdp >> 48);
+        }
+        pVCpu->cpum.GstCtx.XState.x87.MXCSR      = aValues[iReg].XmmControlStatus.XmmStatusControl;
+        pVCpu->cpum.GstCtx.XState.x87.MXCSR_MASK = aValues[iReg].XmmControlStatus.XmmStatusControlMask; /** @todo ??? (Isn't this an output field?) */
+        iReg++;
+    }
+
+    /* Vector state. */
+    if (fWhat & CPUMCTX_EXTRN_SSE_AVX)
+    {
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 0].uXmm.s.Hi, WHvX64RegisterXmm0);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 1].uXmm.s.Hi, WHvX64RegisterXmm1);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 2].uXmm.s.Hi, WHvX64RegisterXmm2);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 3].uXmm.s.Hi, WHvX64RegisterXmm3);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 4].uXmm.s.Hi, WHvX64RegisterXmm4);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 5].uXmm.s.Hi, WHvX64RegisterXmm5);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 6].uXmm.s.Hi, WHvX64RegisterXmm6);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 7].uXmm.s.Hi, WHvX64RegisterXmm7);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 8].uXmm.s.Hi, WHvX64RegisterXmm8);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[ 9].uXmm.s.Hi, WHvX64RegisterXmm9);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[10].uXmm.s.Hi, WHvX64RegisterXmm10);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[11].uXmm.s.Hi, WHvX64RegisterXmm11);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[12].uXmm.s.Hi, WHvX64RegisterXmm12);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[13].uXmm.s.Hi, WHvX64RegisterXmm13);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[14].uXmm.s.Hi, WHvX64RegisterXmm14);
+        GET_REG128(pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Lo, pVCpu->cpum.GstCtx.XState.x87.aXMM[15].uXmm.s.Hi, WHvX64RegisterXmm15);
+    }
+
+    /* MSRs */
+    // WHvX64RegisterTsc - don't touch
+    if (fWhat & CPUMCTX_EXTRN_EFER)
+    {
+        Assert(aenmNames[iReg] == WHvX64RegisterEfer);
+        if (aValues[iReg].Reg64 != pVCpu->cpum.GstCtx.msrEFER)
+        {
+            Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, aValues[iReg].Reg64));
+            if ((aValues[iReg].Reg64 ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
+                PGMNotifyNxeChanged(pVCpu, RT_BOOL(aValues[iReg].Reg64 & MSR_K6_EFER_NXE));
+            pVCpu->cpum.GstCtx.msrEFER = aValues[iReg].Reg64;
+            fMaybeChangedMode = true;
+        }
+        iReg++;
+    }
+    if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrKERNELGSBASE, WHvX64RegisterKernelGsBase, "MSR KERNEL_GS_BASE");
+    if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+    {
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.cs,  WHvX64RegisterSysenterCs,  "MSR SYSENTER.CS");
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.eip, WHvX64RegisterSysenterEip, "MSR SYSENTER.EIP");
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.SysEnter.esp, WHvX64RegisterSysenterEsp, "MSR SYSENTER.ESP");
+    }
+    if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+    {
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSTAR,   WHvX64RegisterStar,   "MSR STAR");
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrLSTAR,  WHvX64RegisterLstar,  "MSR LSTAR");
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrCSTAR,  WHvX64RegisterCstar,  "MSR CSTAR");
+        GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrSFMASK, WHvX64RegisterSfmask, "MSR SFMASK");
+    }
+    if (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
+    {
+        PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
+        if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
+            GET_REG64_LOG7(pCtxMsrs->msr.TscAux, WHvX64RegisterTscAux, "MSR TSC_AUX");
+        if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
+        {
+            Assert(aenmNames[iReg] == WHvX64RegisterApicBase);
+            const uint64_t uOldBase = APICGetBaseMsrNoCheck(pVCpu);
+            if (aValues[iReg].Reg64 != uOldBase)
+            {
+                Log7(("NEM/%u: MSR APICBase changed %RX64 -> %RX64 (%RX64)\n",
+                      pVCpu->idCpu, uOldBase, aValues[iReg].Reg64, aValues[iReg].Reg64 ^ uOldBase));
+                int rc2 = APICSetBaseMsr(pVCpu, aValues[iReg].Reg64);
+                AssertLogRelMsg(rc2 == VINF_SUCCESS, ("%Rrc %RX64\n", rc2, aValues[iReg].Reg64));
+            }
+            iReg++;
+
+            GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterPat, "MSR PAT");
+#if 0 /*def LOG_ENABLED*/ /** @todo something's wrong with HvX64RegisterMtrrCap? (AMD) */
+            GET_REG64_LOG7(pVCpu->cpum.GstCtx.msrPAT, WHvX64RegisterMsrMtrrCap);
+#endif
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrDefType,      WHvX64RegisterMsrMtrrDefType,     "MSR MTRR_DEF_TYPE");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix64K_00000, WHvX64RegisterMsrMtrrFix64k00000, "MSR MTRR_FIX_64K_00000");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_80000, WHvX64RegisterMsrMtrrFix16k80000, "MSR MTRR_FIX_16K_80000");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix16K_A0000, WHvX64RegisterMsrMtrrFix16kA0000, "MSR MTRR_FIX_16K_A0000");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C0000,  WHvX64RegisterMsrMtrrFix4kC0000,  "MSR MTRR_FIX_4K_C0000");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_C8000,  WHvX64RegisterMsrMtrrFix4kC8000,  "MSR MTRR_FIX_4K_C8000");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D0000,  WHvX64RegisterMsrMtrrFix4kD0000,  "MSR MTRR_FIX_4K_D0000");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_D8000,  WHvX64RegisterMsrMtrrFix4kD8000,  "MSR MTRR_FIX_4K_D8000");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E0000,  WHvX64RegisterMsrMtrrFix4kE0000,  "MSR MTRR_FIX_4K_E0000");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_E8000,  WHvX64RegisterMsrMtrrFix4kE8000,  "MSR MTRR_FIX_4K_E8000");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F0000,  WHvX64RegisterMsrMtrrFix4kF0000,  "MSR MTRR_FIX_4K_F0000");
+            GET_REG64_LOG7(pCtxMsrs->msr.MtrrFix4K_F8000,  WHvX64RegisterMsrMtrrFix4kF8000,  "MSR MTRR_FIX_4K_F8000");
+            /** @todo look for HvX64RegisterIa32MiscEnable and HvX64RegisterIa32FeatureControl? */
+        }
+    }
+
+    /* Interruptibility. */
+    if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
+    {
+        Assert(aenmNames[iReg] == WHvRegisterInterruptState);
+        Assert(aenmNames[iReg + 1] == WHvX64RegisterRip);
+
+        if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_INT))
+            pVCpu->nem.s.fLastInterruptShadow = CPUMUpdateInterruptShadowEx(&pVCpu->cpum.GstCtx,
+                                                                            aValues[iReg].InterruptState.InterruptShadow,
+                                                                            aValues[iReg + 1].Reg64);
+
+        if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_NMI))
+            CPUMUpdateInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx, aValues[iReg].InterruptState.NmiMasked);
+
+        fWhat |= CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI;
+        iReg += 2;
+    }
+
+    /* Event injection. */
+    /// @todo WHvRegisterPendingInterruption
+    Assert(aenmNames[iReg] == WHvRegisterPendingInterruption);
+    if (aValues[iReg].PendingInterruption.InterruptionPending)
+    {
+        Log7(("PendingInterruption: type=%u vector=%#x errcd=%RTbool/%#x instr-len=%u nested=%u\n",
+              aValues[iReg].PendingInterruption.InterruptionType, aValues[iReg].PendingInterruption.InterruptionVector,
+              aValues[iReg].PendingInterruption.DeliverErrorCode, aValues[iReg].PendingInterruption.ErrorCode,
+              aValues[iReg].PendingInterruption.InstructionLength, aValues[iReg].PendingInterruption.NestedEvent));
+        AssertMsg((aValues[iReg].PendingInterruption.AsUINT64 & UINT64_C(0xfc00)) == 0,
+                  ("%#RX64\n", aValues[iReg].PendingInterruption.AsUINT64));
+    }
+
+    /// @todo WHvRegisterPendingEvent
+
+    /* Almost done, just update extrn flags and maybe change PGM mode. */
+    pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
+    if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
+        pVCpu->cpum.GstCtx.fExtrn = 0;
+
+    /* Typical. */
+    if (!fMaybeChangedMode && !fUpdateCr3)
+        return VINF_SUCCESS;
+
+    /*
+     * Slow.
+     */
+    if (fMaybeChangedMode)
+    {
+        int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4, pVCpu->cpum.GstCtx.msrEFER,
+                               false /* fForce */);
+        AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
+    }
+
+    if (fUpdateCr3)
+    {
+        int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3);
+        if (rc == VINF_SUCCESS)
+        { /* likely */ }
+        else
+            AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Interface for importing state on demand (used by IEM).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context CPU structure.
+ * @param   fWhat       What to import, CPUMCTX_EXTRN_XXX.
+ */
+VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
+{
+    STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
+    return nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
+}
+
+
+/**
+ * Query the CPU tick counter and optionally the TSC_AUX MSR value.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context CPU structure.
+ * @param   pcTicks     Where to return the CPU tick count.
+ * @param   puAux       Where to return the TSC_AUX register value.
+ */
+VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t *pcTicks, uint32_t *puAux)
+{
+    STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
+    AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
+
+    /* Call the offical API. */
+    WHV_REGISTER_NAME  aenmNames[2] = { WHvX64RegisterTsc, WHvX64RegisterTscAux };
+    WHV_REGISTER_VALUE aValues[2]   = { { {0, 0} }, { {0, 0} } };
+    Assert(RT_ELEMENTS(aenmNames) == RT_ELEMENTS(aValues));
+    HRESULT hrc = WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, aenmNames, 2, aValues);
+    AssertLogRelMsgReturn(SUCCEEDED(hrc),
+                          ("WHvGetVirtualProcessorRegisters(%p, %u,{tsc,tsc_aux},2,) -> %Rhrc (Last=%#x/%u)\n",
+                           pVM->nem.s.hPartition, pVCpu->idCpu, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
+                          , VERR_NEM_GET_REGISTERS_FAILED);
+    *pcTicks = aValues[0].Reg64;
+    if (puAux)
+        *puAux = pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_TSC_AUX ? aValues[1].Reg64 : CPUMGetGuestTscAux(pVCpu);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Resumes CPU clock (TSC) on all virtual CPUs.
+ *
+ * This is called by TM when the VM is started, restored, resumed or similar.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context CPU structure of the calling EMT.
+ * @param   uPausedTscValue The TSC value at the time of pausing.
+ */
+VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
+{
+    VMCPU_ASSERT_EMT_RETURN(pVCpu, VERR_VM_THREAD_NOT_EMT);
+    AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_NEM_IPE_9);
+
+    /*
+     * Call the offical API to do the job.
+     */
+    if (pVM->cCpus > 1)
+        RTThreadYield(); /* Try decrease the chance that we get rescheduled in the middle. */
+
+    /* Start with the first CPU. */
+    WHV_REGISTER_NAME  enmName   = WHvX64RegisterTsc;
+    WHV_REGISTER_VALUE Value     = { {0, 0} };
+    Value.Reg64 = uPausedTscValue;
+    uint64_t const     uFirstTsc = ASMReadTSC();
+    HRESULT hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, 0 /*iCpu*/, &enmName, 1, &Value);
+    AssertLogRelMsgReturn(SUCCEEDED(hrc),
+                          ("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64) -> %Rhrc (Last=%#x/%u)\n",
+                           pVM->nem.s.hPartition, uPausedTscValue, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
+                          , VERR_NEM_SET_TSC);
+
+    /* Do the other CPUs, adjusting for elapsed TSC and keeping finger crossed
+       that we don't introduce too much drift here. */
+    for (VMCPUID iCpu = 1; iCpu < pVM->cCpus; iCpu++)
+    {
+        Assert(enmName == WHvX64RegisterTsc);
+        const uint64_t offDelta = (ASMReadTSC() - uFirstTsc);
+        Value.Reg64 = uPausedTscValue + offDelta;
+        hrc = WHvSetVirtualProcessorRegisters(pVM->nem.s.hPartition, iCpu, &enmName, 1, &Value);
+        AssertLogRelMsgReturn(SUCCEEDED(hrc),
+                              ("WHvSetVirtualProcessorRegisters(%p, 0,{tsc},2,%#RX64 + %#RX64) -> %Rhrc (Last=%#x/%u)\n",
+                               pVM->nem.s.hPartition, iCpu, uPausedTscValue, offDelta, hrc, RTNtLastStatusValue(), RTNtLastErrorValue())
+                              , VERR_NEM_SET_TSC);
+    }
+
+    return VINF_SUCCESS;
+}
+
+#ifdef LOG_ENABLED
+
+/**
+ * Get the virtual processor running status.
+ */
+DECLINLINE(VID_PROCESSOR_STATUS) nemHCWinCpuGetRunningStatus(PVMCPUCC pVCpu)
+{
+    RTERRVARS Saved;
+    RTErrVarsSave(&Saved);
+
+    /*
+     * This API is disabled in release builds, it seems.  On build 17101 it requires
+     * the following patch to be enabled (windbg): eb vid+12180 0f 84 98 00 00 00
+     */
+    VID_PROCESSOR_STATUS enmCpuStatus = VidProcessorStatusUndefined;
+    NTSTATUS rcNt = g_pfnVidGetVirtualProcessorRunningStatus(pVCpu->pVMR3->nem.s.hPartitionDevice, pVCpu->idCpu, &enmCpuStatus);
+    AssertRC(rcNt);
+
+    RTErrVarsRestore(&Saved);
+    return enmCpuStatus;
+}
+
+
+/**
+ * Logs the current CPU state.
+ */
+NEM_TMPL_STATIC void nemHCWinLogState(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (LogIs3Enabled())
+    {
+# if 0 // def IN_RING3 - causes lazy state import assertions all over CPUM.
+        char szRegs[4096];
+        DBGFR3RegPrintf(pVM->pUVM, pVCpu->idCpu, &szRegs[0], sizeof(szRegs),
+                        "rax=%016VR{rax} rbx=%016VR{rbx} rcx=%016VR{rcx} rdx=%016VR{rdx}\n"
+                        "rsi=%016VR{rsi} rdi=%016VR{rdi} r8 =%016VR{r8} r9 =%016VR{r9}\n"
+                        "r10=%016VR{r10} r11=%016VR{r11} r12=%016VR{r12} r13=%016VR{r13}\n"
+                        "r14=%016VR{r14} r15=%016VR{r15} %VRF{rflags}\n"
+                        "rip=%016VR{rip} rsp=%016VR{rsp} rbp=%016VR{rbp}\n"
+                        "cs={%04VR{cs} base=%016VR{cs_base} limit=%08VR{cs_lim} flags=%04VR{cs_attr}} cr0=%016VR{cr0}\n"
+                        "ds={%04VR{ds} base=%016VR{ds_base} limit=%08VR{ds_lim} flags=%04VR{ds_attr}} cr2=%016VR{cr2}\n"
+                        "es={%04VR{es} base=%016VR{es_base} limit=%08VR{es_lim} flags=%04VR{es_attr}} cr3=%016VR{cr3}\n"
+                        "fs={%04VR{fs} base=%016VR{fs_base} limit=%08VR{fs_lim} flags=%04VR{fs_attr}} cr4=%016VR{cr4}\n"
+                        "gs={%04VR{gs} base=%016VR{gs_base} limit=%08VR{gs_lim} flags=%04VR{gs_attr}} cr8=%016VR{cr8}\n"
+                        "ss={%04VR{ss} base=%016VR{ss_base} limit=%08VR{ss_lim} flags=%04VR{ss_attr}}\n"
+                        "dr0=%016VR{dr0} dr1=%016VR{dr1} dr2=%016VR{dr2} dr3=%016VR{dr3}\n"
+                        "dr6=%016VR{dr6} dr7=%016VR{dr7}\n"
+                        "gdtr=%016VR{gdtr_base}:%04VR{gdtr_lim}  idtr=%016VR{idtr_base}:%04VR{idtr_lim}  rflags=%08VR{rflags}\n"
+                        "ldtr={%04VR{ldtr} base=%016VR{ldtr_base} limit=%08VR{ldtr_lim} flags=%08VR{ldtr_attr}}\n"
+                        "tr  ={%04VR{tr} base=%016VR{tr_base} limit=%08VR{tr_lim} flags=%08VR{tr_attr}}\n"
+                        "    sysenter={cs=%04VR{sysenter_cs} eip=%08VR{sysenter_eip} esp=%08VR{sysenter_esp}}\n"
+                        "        efer=%016VR{efer}\n"
+                        "         pat=%016VR{pat}\n"
+                        "     sf_mask=%016VR{sf_mask}\n"
+                        "krnl_gs_base=%016VR{krnl_gs_base}\n"
+                        "       lstar=%016VR{lstar}\n"
+                        "        star=%016VR{star} cstar=%016VR{cstar}\n"
+                        "fcw=%04VR{fcw} fsw=%04VR{fsw} ftw=%04VR{ftw} mxcsr=%04VR{mxcsr} mxcsr_mask=%04VR{mxcsr_mask}\n"
+                        );
+
+        char szInstr[256];
+        DBGFR3DisasInstrEx(pVM->pUVM, pVCpu->idCpu, 0, 0,
+                           DBGF_DISAS_FLAGS_CURRENT_GUEST | DBGF_DISAS_FLAGS_DEFAULT_MODE,
+                           szInstr, sizeof(szInstr), NULL);
+        Log3(("%s%s\n", szRegs, szInstr));
+# else
+        /** @todo stat logging in ring-0 */
+        RT_NOREF(pVM, pVCpu);
+# endif
+    }
+}
+
+#endif /* LOG_ENABLED */
+
+/**
+ * Translates the execution stat bitfield into a short log string, WinHv version.
+ *
+ * @returns Read-only log string.
+ * @param   pExitCtx        The exit context which state to summarize.
+ */
+static const char *nemR3WinExecStateToLogStr(WHV_VP_EXIT_CONTEXT const *pExitCtx)
+{
+    unsigned u = (unsigned)pExitCtx->ExecutionState.InterruptionPending
+               | ((unsigned)pExitCtx->ExecutionState.DebugActive << 1)
+               | ((unsigned)pExitCtx->ExecutionState.InterruptShadow << 2);
+#define SWITCH_IT(a_szPrefix) \
+        do \
+            switch (u)\
+            { \
+                case 0x00: return a_szPrefix ""; \
+                case 0x01: return a_szPrefix ",Pnd"; \
+                case 0x02: return a_szPrefix ",Dbg"; \
+                case 0x03: return a_szPrefix ",Pnd,Dbg"; \
+                case 0x04: return a_szPrefix ",Shw"; \
+                case 0x05: return a_szPrefix ",Pnd,Shw"; \
+                case 0x06: return a_szPrefix ",Shw,Dbg"; \
+                case 0x07: return a_szPrefix ",Pnd,Shw,Dbg"; \
+                default: AssertFailedReturn("WTF?"); \
+            } \
+        while (0)
+    if (pExitCtx->ExecutionState.EferLma)
+        SWITCH_IT("LM");
+    else if (pExitCtx->ExecutionState.Cr0Pe)
+        SWITCH_IT("PM");
+    else
+        SWITCH_IT("RM");
+#undef SWITCH_IT
+}
+
+
+/**
+ * Advances the guest RIP and clear EFLAGS.RF, WinHv version.
+ *
+ * This may clear VMCPU_FF_INHIBIT_INTERRUPTS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pExitCtx        The exit context.
+ * @param   cbMinInstr      The minimum instruction length, or 1 if not unknown.
+ */
+DECLINLINE(void) nemR3WinAdvanceGuestRipAndClearRF(PVMCPUCC pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx, uint8_t cbMinInstr)
+{
+    Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)));
+
+    /* Advance the RIP. */
+    Assert(pExitCtx->InstructionLength >= cbMinInstr); RT_NOREF_PV(cbMinInstr);
+    pVCpu->cpum.GstCtx.rip += pExitCtx->InstructionLength;
+    pVCpu->cpum.GstCtx.rflags.Bits.u1RF = 0;
+    CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
+}
+
+
+/**
+ * State to pass between nemHCWinHandleMemoryAccess / nemR3WinWHvHandleMemoryAccess
+ * and nemHCWinHandleMemoryAccessPageCheckerCallback.
+ */
+typedef struct NEMHCWINHMACPCCSTATE
+{
+    /** Input: Write access. */
+    bool    fWriteAccess;
+    /** Output: Set if we did something. */
+    bool    fDidSomething;
+    /** Output: Set it we should resume. */
+    bool    fCanResume;
+} NEMHCWINHMACPCCSTATE;
+
+/**
+ * @callback_method_impl{FNPGMPHYSNEMCHECKPAGE,
+ *      Worker for nemR3WinHandleMemoryAccess; pvUser points to a
+ *      NEMHCWINHMACPCCSTATE structure. }
+ */
+NEM_TMPL_STATIC DECLCALLBACK(int)
+nemHCWinHandleMemoryAccessPageCheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
+{
+    NEMHCWINHMACPCCSTATE *pState = (NEMHCWINHMACPCCSTATE *)pvUser;
+    pState->fDidSomething = false;
+    pState->fCanResume    = false;
+
+    /* If A20 is disabled, we may need to make another query on the masked
+       page to get the correct protection information. */
+    uint8_t  u2State = pInfo->u2NemState;
+    RTGCPHYS GCPhysSrc;
+#ifdef NEM_WIN_WITH_A20
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
+#endif
+        GCPhysSrc = GCPhys;
+#ifdef NEM_WIN_WITH_A20
+    else
+    {
+        GCPhysSrc = GCPhys & ~(RTGCPHYS)RT_BIT_32(20);
+        PGMPHYSNEMPAGEINFO Info2;
+        int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhysSrc, pState->fWriteAccess, &Info2, NULL, NULL);
+        AssertRCReturn(rc, rc);
+
+        *pInfo = Info2;
+        pInfo->u2NemState = u2State;
+    }
+#endif
+
+    /*
+     * Consolidate current page state with actual page protection and access type.
+     * We don't really consider downgrades here, as they shouldn't happen.
+     */
+    /** @todo Someone at microsoft please explain:
+     * I'm not sure WTF was going on, but I ended up in a loop if I remapped a
+     * readonly page as writable (unmap, then map again).  Specifically, this was an
+     * issue with the big VRAM mapping at 0xe0000000 when booing DSL 4.4.1.  So, in
+     * a hope to work around that we no longer pre-map anything, just unmap stuff
+     * and do it lazily here.  And here we will first unmap, restart, and then remap
+     * with new protection or backing.
+     */
+    int rc;
+    switch (u2State)
+    {
+        case NEM_WIN_PAGE_STATE_UNMAPPED:
+        case NEM_WIN_PAGE_STATE_NOT_SET:
+            if (pInfo->fNemProt == NEM_PAGE_PROT_NONE)
+            {
+                Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1\n", GCPhys));
+                return VINF_SUCCESS;
+            }
+
+            /* Don't bother remapping it if it's a write request to a non-writable page. */
+            if (   pState->fWriteAccess
+                && !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE))
+            {
+                Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #1w\n", GCPhys));
+                return VINF_SUCCESS;
+            }
+
+            /* Map the page. */
+            rc = nemHCNativeSetPhysPage(pVM,
+                                        pVCpu,
+                                        GCPhysSrc & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
+                                        GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK,
+                                        pInfo->fNemProt,
+                                        &u2State,
+                                        true /*fBackingState*/);
+            pInfo->u2NemState = u2State;
+            Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - synced => %s + %Rrc\n",
+                  GCPhys, g_apszPageStates[u2State], rc));
+            pState->fDidSomething = true;
+            pState->fCanResume    = true;
+            return rc;
+
+        case NEM_WIN_PAGE_STATE_READABLE:
+            if (   !(pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
+                && (pInfo->fNemProt & (NEM_PAGE_PROT_READ | NEM_PAGE_PROT_EXECUTE)))
+            {
+                Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #2\n", GCPhys));
+                return VINF_SUCCESS;
+            }
+
+            break;
+
+        case NEM_WIN_PAGE_STATE_WRITABLE:
+            if (pInfo->fNemProt & NEM_PAGE_PROT_WRITE)
+            {
+                if (pInfo->u2OldNemState == NEM_WIN_PAGE_STATE_WRITABLE)
+                    Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3a\n", GCPhys));
+                else
+                {
+                    pState->fCanResume = true;
+                    Log4(("nemHCWinHandleMemoryAccessPageCheckerCallback: %RGp - #3b (%s -> %s)\n",
+                          GCPhys, g_apszPageStates[pInfo->u2OldNemState], g_apszPageStates[u2State]));
+                }
+                return VINF_SUCCESS;
+            }
+            break;
+
+        default:
+            AssertLogRelMsgFailedReturn(("u2State=%#x\n", u2State), VERR_NEM_IPE_4);
+    }
+
+    /*
+     * Unmap and restart the instruction.
+     * If this fails, which it does every so often, just unmap everything for now.
+     */
+    /** @todo figure out whether we mess up the state or if it's WHv.   */
+    STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
+    HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
+    STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
+    if (SUCCEEDED(hrc))
+    {
+        pState->fDidSomething = true;
+        pState->fCanResume    = true;
+        pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
+        STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
+        uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+        Log5(("NEM GPA unmapped/exit: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[u2State], cMappedPages));
+        return VINF_SUCCESS;
+    }
+    STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
+    LogRel(("nemHCWinHandleMemoryAccessPageCheckerCallback/unmap: GCPhysDst=%RGp %s hrc=%Rhrc (%#x)\n",
+            GCPhys, g_apszPageStates[u2State], hrc, hrc));
+    return VERR_NEM_UNMAP_PAGES_FAILED;
+}
+
+
+/**
+ * Wrapper around nemHCWinCopyStateFromHyperV.
+ *
+ * Unlike the wrapped APIs, this checks whether it's necessary.
+ *
+ * @returns VBox strict status code.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   fWhat           What to import.
+ * @param   pszCaller       Who is doing the importing.
+ */
+DECLINLINE(VBOXSTRICTRC) nemHCWinImportStateIfNeededStrict(PVMCPUCC pVCpu, uint64_t fWhat, const char *pszCaller)
+{
+    if (pVCpu->cpum.GstCtx.fExtrn & fWhat)
+    {
+        RT_NOREF(pszCaller);
+        int rc = nemHCWinCopyStateFromHyperV(pVCpu->pVMR3, pVCpu, fWhat);
+        AssertRCReturn(rc, rc);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Copies register state from the (common) exit context.
+ *
+ * ASSUMES no state copied yet.
+ *
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExitCtx        The common exit context.
+ * @sa      nemHCWinCopyStateFromX64Header
+ */
+DECLINLINE(void) nemR3WinCopyStateFromX64Header(PVMCPUCC pVCpu, WHV_VP_EXIT_CONTEXT const *pExitCtx)
+{
+    Assert(   (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_INHIBIT_INT))
+           ==                              (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_INHIBIT_INT));
+
+    NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.cs, pExitCtx->Cs);
+    pVCpu->cpum.GstCtx.rip      = pExitCtx->Rip;
+    pVCpu->cpum.GstCtx.rflags.u = pExitCtx->Rflags;
+    pVCpu->nem.s.fLastInterruptShadow = CPUMUpdateInterruptShadowEx(&pVCpu->cpum.GstCtx,
+                                                                    pExitCtx->ExecutionState.InterruptShadow,
+                                                                    pExitCtx->Rip);
+    APICSetTpr(pVCpu, pExitCtx->Cr8 << 4);
+
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_APIC_TPR);
+}
+
+
+/**
+ * Deals with memory access exits (WHvRunVpExitReasonMemoryAccess).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageMemory
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC
+nemR3WinHandleExitMemory(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    uint64_t const uHostTsc = ASMReadTSC();
+    Assert(pExit->MemoryAccess.AccessInfo.AccessType != 3);
+
+    /*
+     * Whatever we do, we must clear pending event injection upon resume.
+     */
+    if (pExit->VpContext.ExecutionState.InterruptionPending)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
+
+    /*
+     * Ask PGM for information about the given GCPhys.  We need to check if we're
+     * out of sync first.
+     */
+    NEMHCWINHMACPCCSTATE State = { pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite, false, false };
+    PGMPHYSNEMPAGEINFO   Info;
+    int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pExit->MemoryAccess.Gpa, State.fWriteAccess, &Info,
+                                       nemHCWinHandleMemoryAccessPageCheckerCallback, &State);
+    if (RT_SUCCESS(rc))
+    {
+        if (Info.fNemProt & (  pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
+                             ? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
+        {
+            if (State.fCanResume)
+            {
+                Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting (%s)\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                      pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
+                      Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
+                      State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
+                EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
+                                 pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc);
+                return VINF_SUCCESS;
+            }
+        }
+        Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating (%s)\n",
+              pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+              pExit->MemoryAccess.Gpa, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
+              Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
+              State.fDidSomething ? "" : " no-change", g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
+    }
+    else
+        Log4(("MemExit/%u: %04x:%08RX64/%s: %RGp rc=%Rrc%s; emulating (%s)\n",
+              pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+              pExit->MemoryAccess.Gpa, rc, State.fDidSomething ? " modified-backing" : "",
+              g_apszHvInterceptAccessTypes[pExit->MemoryAccess.AccessInfo.AccessType]));
+
+    /*
+     * Emulate the memory access, either access handler or special memory.
+     */
+    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
+                                              pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
+                                            ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
+                                            : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
+                                            pExit->VpContext.Rip + pExit->VpContext.Cs.Base, uHostTsc);
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_DS | CPUMCTX_EXTRN_ES);
+    AssertRCReturn(rc, rc);
+    if (pExit->VpContext.ExecutionState.Reserved0 || pExit->VpContext.ExecutionState.Reserved1)
+        Log(("MemExit/Hdr/State: Reserved0=%#x Reserved1=%#x\n", pExit->VpContext.ExecutionState.Reserved0, pExit->VpContext.ExecutionState.Reserved1));
+
+    VBOXSTRICTRC rcStrict;
+    if (!pExitRec)
+    {
+        //if (pMsg->InstructionByteCount > 0)
+        //    Log4(("InstructionByteCount=%#x %.16Rhxs\n", pMsg->InstructionByteCount, pMsg->InstructionBytes));
+        if (pExit->MemoryAccess.InstructionByteCount > 0)
+            rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, pExit->VpContext.Rip, pExit->MemoryAccess.InstructionBytes, pExit->MemoryAccess.InstructionByteCount);
+        else
+            rcStrict = IEMExecOne(pVCpu);
+        /** @todo do we need to do anything wrt debugging here?   */
+    }
+    else
+    {
+        /* Frequent access or probing. */
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        Log4(("MemExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Deals with I/O port access exits (WHvRunVpExitReasonX64IoPortAccess).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageIoPort
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitIoPort(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    Assert(   pExit->IoPortAccess.AccessInfo.AccessSize == 1
+           || pExit->IoPortAccess.AccessInfo.AccessSize == 2
+           || pExit->IoPortAccess.AccessInfo.AccessSize == 4);
+
+    /*
+     * Whatever we do, we must clear pending event injection upon resume.
+     */
+    if (pExit->VpContext.ExecutionState.InterruptionPending)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
+
+    /*
+     * Add history first to avoid two paths doing EMHistoryExec calls.
+     */
+    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
+                                            !pExit->IoPortAccess.AccessInfo.StringOp
+                                            ? (  pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
+                                               ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE)
+                                               : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ))
+                                            : (  pExit->MemoryAccess.AccessInfo.AccessType == WHvMemoryAccessWrite
+                                               ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)
+                                               : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)),
+                                            pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+    if (!pExitRec)
+    {
+        VBOXSTRICTRC rcStrict;
+        if (!pExit->IoPortAccess.AccessInfo.StringOp)
+        {
+            /*
+             * Simple port I/O.
+             */
+            static uint32_t const s_fAndMask[8] =
+            {   UINT32_MAX, UINT32_C(0xff), UINT32_C(0xffff), UINT32_MAX,   UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX   };
+            uint32_t const        fAndMask      = s_fAndMask[pExit->IoPortAccess.AccessInfo.AccessSize];
+            if (pExit->IoPortAccess.AccessInfo.IsWrite)
+            {
+                rcStrict = IOMIOPortWrite(pVM, pVCpu, pExit->IoPortAccess.PortNumber,
+                                          (uint32_t)pExit->IoPortAccess.Rax & fAndMask,
+                                          pExit->IoPortAccess.AccessInfo.AccessSize);
+                Log4(("IOExit/%u: %04x:%08RX64/%s: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                      pExit->IoPortAccess.PortNumber, (uint32_t)pExit->IoPortAccess.Rax & fAndMask,
+                      pExit->IoPortAccess.AccessInfo.AccessSize, VBOXSTRICTRC_VAL(rcStrict) ));
+                if (IOM_SUCCESS(rcStrict))
+                {
+                    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+                    nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1);
+                }
+            }
+            else
+            {
+                uint32_t uValue = 0;
+                rcStrict = IOMIOPortRead(pVM, pVCpu, pExit->IoPortAccess.PortNumber, &uValue,
+                                         pExit->IoPortAccess.AccessInfo.AccessSize);
+                Log4(("IOExit/%u: %04x:%08RX64/%s: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                      pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
+                if (IOM_SUCCESS(rcStrict))
+                {
+                    if (pExit->IoPortAccess.AccessInfo.AccessSize != 4)
+                        pVCpu->cpum.GstCtx.rax = (pExit->IoPortAccess.Rax & ~(uint64_t)fAndMask) | (uValue & fAndMask);
+                    else
+                        pVCpu->cpum.GstCtx.rax = uValue;
+                    pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
+                    Log4(("IOExit/%u: RAX %#RX64 -> %#RX64\n", pVCpu->idCpu, pExit->IoPortAccess.Rax, pVCpu->cpum.GstCtx.rax));
+                    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+                    nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 1);
+                }
+            }
+        }
+        else
+        {
+            /*
+             * String port I/O.
+             */
+            /** @todo Someone at Microsoft please explain how we can get the address mode
+             * from the IoPortAccess.VpContext.  CS.Attributes is only sufficient for
+             * getting the default mode, it can always be overridden by a prefix.   This
+             * forces us to interpret the instruction from opcodes, which is suboptimal.
+             * Both AMD-V and VT-x includes the address size in the exit info, at least on
+             * CPUs that are reasonably new.
+             *
+             * Of course, it's possible this is an undocumented and we just need to do some
+             * experiments to figure out how it's communicated.  Alternatively, we can scan
+             * the opcode bytes for possible evil prefixes.
+             */
+            nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+            pVCpu->cpum.GstCtx.fExtrn &= ~(  CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDI | CPUMCTX_EXTRN_RSI
+                                           | CPUMCTX_EXTRN_DS  | CPUMCTX_EXTRN_ES);
+            NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds);
+            NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es);
+            pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax;
+            pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx;
+            pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi;
+            pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi;
+            int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
+            AssertRCReturn(rc, rc);
+
+            Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s %#x LB %u (emulating)\n",
+                  pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                  pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "",
+                  pExit->IoPortAccess.AccessInfo.IsWrite ? "OUTS" : "INS",
+                  pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize ));
+            rcStrict = IEMExecOne(pVCpu);
+        }
+        if (IOM_SUCCESS(rcStrict))
+        {
+            /*
+             * Do debug checks.
+             */
+            if (   pExit->VpContext.ExecutionState.DebugActive /** @todo Microsoft: Does DebugActive this only reflect DR7? */
+                || (pExit->VpContext.Rflags & X86_EFL_TF)
+                || DBGFBpIsHwIoArmed(pVM) )
+            {
+                /** @todo Debugging. */
+            }
+        }
+        return rcStrict;
+    }
+
+    /*
+     * Frequent exit or something needing probing.
+     * Get state and call EMHistoryExec.
+     */
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    if (!pExit->IoPortAccess.AccessInfo.StringOp)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RAX;
+    else
+    {
+        pVCpu->cpum.GstCtx.fExtrn &= ~(  CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDI | CPUMCTX_EXTRN_RSI
+                          | CPUMCTX_EXTRN_DS  | CPUMCTX_EXTRN_ES);
+        NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.ds, pExit->IoPortAccess.Ds);
+        NEM_WIN_COPY_BACK_SEG(pVCpu->cpum.GstCtx.es, pExit->IoPortAccess.Es);
+        pVCpu->cpum.GstCtx.rcx = pExit->IoPortAccess.Rcx;
+        pVCpu->cpum.GstCtx.rdi = pExit->IoPortAccess.Rdi;
+        pVCpu->cpum.GstCtx.rsi = pExit->IoPortAccess.Rsi;
+    }
+    pVCpu->cpum.GstCtx.rax = pExit->IoPortAccess.Rax;
+    int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
+    AssertRCReturn(rc, rc);
+    Log4(("IOExit/%u: %04x:%08RX64/%s: %s%s%s %#x LB %u -> EMHistoryExec\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+          pExit->IoPortAccess.AccessInfo.RepPrefix ? "REP " : "",
+          pExit->IoPortAccess.AccessInfo.IsWrite ? "OUT" : "IN",
+          pExit->IoPortAccess.AccessInfo.StringOp ? "S" : "",
+          pExit->IoPortAccess.PortNumber, pExit->IoPortAccess.AccessInfo.AccessSize));
+    VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+    Log4(("IOExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+          VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    return rcStrict;
+}
+
+
+/**
+ * Deals with interrupt window exits (WHvRunVpExitReasonX64InterruptWindow).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageInterruptWindow
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitInterruptWindow(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    /*
+     * Assert message sanity.
+     */
+    AssertMsg(   pExit->InterruptWindow.DeliverableType == WHvX64PendingInterrupt
+              || pExit->InterruptWindow.DeliverableType == WHvX64PendingNmi,
+              ("%#x\n", pExit->InterruptWindow.DeliverableType));
+
+    /*
+     * Just copy the state we've got and handle it in the loop for now.
+     */
+    EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
+                     pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    Log4(("IntWinExit/%u: %04x:%08RX64/%s: %u IF=%d InterruptShadow=%d CR8=%#x\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,  nemR3WinExecStateToLogStr(&pExit->VpContext),
+          pExit->InterruptWindow.DeliverableType, RT_BOOL(pExit->VpContext.Rflags & X86_EFL_IF),
+          pExit->VpContext.ExecutionState.InterruptShadow, pExit->VpContext.Cr8));
+
+    /** @todo call nemHCWinHandleInterruptFF   */
+    RT_NOREF(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deals with CPUID exits (WHvRunVpExitReasonX64Cpuid).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageCpuId
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC
+nemR3WinHandleExitCpuId(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_CPUID),
+                                            pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+    if (!pExitRec)
+    {
+        /*
+         * Soak up state and execute the instruction.
+         */
+        nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+        VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu,
+                                                                    IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
+                                                                  | CPUMCTX_EXTRN_CR3, /* May call PGMChangeMode() requiring cr3 (due to cr0 being imported). */
+                                                                  "CPUID");
+        if (rcStrict == VINF_SUCCESS)
+        {
+            /* Copy in the low register values (top is always cleared). */
+            pVCpu->cpum.GstCtx.rax = (uint32_t)pExit->CpuidAccess.Rax;
+            pVCpu->cpum.GstCtx.rcx = (uint32_t)pExit->CpuidAccess.Rcx;
+            pVCpu->cpum.GstCtx.rdx = (uint32_t)pExit->CpuidAccess.Rdx;
+            pVCpu->cpum.GstCtx.rbx = (uint32_t)pExit->CpuidAccess.Rbx;
+            pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX);
+
+            /* Execute the decoded instruction. */
+            rcStrict = IEMExecDecodedCpuid(pVCpu, pExit->VpContext.InstructionLength);
+
+            Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 -> %08RX32 / %08RX32 / %08RX32 / %08RX32 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64)\n",
+                  pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                  pExit->CpuidAccess.Rax,                           pExit->CpuidAccess.Rcx,              pExit->CpuidAccess.Rdx,              pExit->CpuidAccess.Rbx,
+                  pVCpu->cpum.GstCtx.eax,                           pVCpu->cpum.GstCtx.ecx,              pVCpu->cpum.GstCtx.edx,              pVCpu->cpum.GstCtx.ebx,
+                  pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx));
+        }
+
+        RT_NOREF_PV(pVM);
+        return rcStrict;
+    }
+
+    /*
+     * Frequent exit or something needing probing.
+     * Get state and call EMHistoryExec.
+     */
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    pVCpu->cpum.GstCtx.rax = pExit->CpuidAccess.Rax;
+    pVCpu->cpum.GstCtx.rcx = pExit->CpuidAccess.Rcx;
+    pVCpu->cpum.GstCtx.rdx = pExit->CpuidAccess.Rdx;
+    pVCpu->cpum.GstCtx.rbx = pExit->CpuidAccess.Rbx;
+    pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RBX);
+    Log4(("CpuIdExit/%u: %04x:%08RX64/%s: rax=%08RX64 / rcx=%08RX64 / rdx=%08RX64 / rbx=%08RX64 (hv: %08RX64 / %08RX64 / %08RX64 / %08RX64) ==> EMHistoryExec\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+          pExit->CpuidAccess.Rax,                           pExit->CpuidAccess.Rcx,              pExit->CpuidAccess.Rdx,              pExit->CpuidAccess.Rbx,
+          pExit->CpuidAccess.DefaultResultRax, pExit->CpuidAccess.DefaultResultRcx, pExit->CpuidAccess.DefaultResultRdx, pExit->CpuidAccess.DefaultResultRbx));
+    int rc = nemHCWinCopyStateFromHyperV(pVM, pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM);
+    AssertRCReturn(rc, rc);
+    VBOXSTRICTRC rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+    Log4(("CpuIdExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+          VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    return rcStrict;
+}
+
+
+/**
+ * Deals with MSR access exits (WHvRunVpExitReasonX64MsrAccess).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageMsr
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitMsr(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    /*
+     * Check CPL as that's common to both RDMSR and WRMSR.
+     */
+    VBOXSTRICTRC rcStrict;
+    if (pExit->VpContext.ExecutionState.Cpl == 0)
+    {
+        /*
+         * Get all the MSR state.  Since we're getting EFER, we also need to
+         * get CR0, CR4 and CR3.
+         */
+        PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
+                                                  pExit->MsrAccess.AccessInfo.IsWrite
+                                                ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE)
+                                                : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
+                                                pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+        nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+        rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu,
+                                                     (!pExitRec ? 0 : IEM_CPUMCTX_EXTRN_MUST_MASK)
+                                                     | CPUMCTX_EXTRN_ALL_MSRS | CPUMCTX_EXTRN_CR0
+                                                     | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4,
+                                                     "MSRs");
+        if (rcStrict == VINF_SUCCESS)
+        {
+            if (!pExitRec)
+            {
+                /*
+                 * Handle writes.
+                 */
+                if (pExit->MsrAccess.AccessInfo.IsWrite)
+                {
+                    rcStrict = CPUMSetGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber,
+                                               RT_MAKE_U64((uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx));
+                    Log4(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                          pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber,
+                          (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx, VBOXSTRICTRC_VAL(rcStrict) ));
+                    if (rcStrict == VINF_SUCCESS)
+                    {
+                        nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
+                        return VINF_SUCCESS;
+                    }
+                    LogRel(("MsrExit/%u: %04x:%08RX64/%s: WRMSR %08x, %08x:%08x -> %Rrc!\n", pVCpu->idCpu,
+                            pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                            pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx,
+                            VBOXSTRICTRC_VAL(rcStrict) ));
+                }
+                /*
+                 * Handle reads.
+                 */
+                else
+                {
+                    uint64_t uValue = 0;
+                    rcStrict = CPUMQueryGuestMsr(pVCpu, pExit->MsrAccess.MsrNumber, &uValue);
+                    Log4(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu,
+                          pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                          pExit->MsrAccess.MsrNumber, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
+                    if (rcStrict == VINF_SUCCESS)
+                    {
+                        pVCpu->cpum.GstCtx.rax = (uint32_t)uValue;
+                        pVCpu->cpum.GstCtx.rdx = uValue >> 32;
+                        pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX);
+                        nemR3WinAdvanceGuestRipAndClearRF(pVCpu, &pExit->VpContext, 2);
+                        return VINF_SUCCESS;
+                    }
+                    LogRel(("MsrExit/%u: %04x:%08RX64/%s: RDMSR %08x -> %08RX64 / %Rrc\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                            pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->MsrAccess.MsrNumber,
+                            uValue, VBOXSTRICTRC_VAL(rcStrict) ));
+                }
+            }
+            else
+            {
+                /*
+                 * Handle frequent exit or something needing probing.
+                 */
+                Log4(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %#08x\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                      pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber));
+                rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+                Log4(("MsrExit/%u: %04x:%08RX64/%s: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                      VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+                return rcStrict;
+            }
+        }
+        else
+        {
+            LogRel(("MsrExit/%u: %04x:%08RX64/%s: %sMSR %08x -> %Rrc - msr state import\n",
+                    pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                    pExit->MsrAccess.AccessInfo.IsWrite ? "WR" : "RD", pExit->MsrAccess.MsrNumber, VBOXSTRICTRC_VAL(rcStrict) ));
+            return rcStrict;
+        }
+    }
+    else if (pExit->MsrAccess.AccessInfo.IsWrite)
+        Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); WRMSR %08x, %08x:%08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+              pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl,
+              pExit->MsrAccess.MsrNumber, (uint32_t)pExit->MsrAccess.Rax, (uint32_t)pExit->MsrAccess.Rdx ));
+    else
+        Log4(("MsrExit/%u: %04x:%08RX64/%s: CPL %u -> #GP(0); RDMSR %08x\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+              pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.ExecutionState.Cpl,
+              pExit->MsrAccess.MsrNumber));
+
+    /*
+     * If we get down here, we're supposed to #GP(0).
+     */
+    rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_ALL_MSRS, "MSR");
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_GP, TRPM_TRAP, 0, 0, 0);
+        if (rcStrict == VINF_IEM_RAISED_XCPT)
+            rcStrict = VINF_SUCCESS;
+        else if (rcStrict != VINF_SUCCESS)
+            Log4(("MsrExit/%u: Injecting #GP(0) failed: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
+    }
+
+    RT_NOREF_PV(pVM);
+    return rcStrict;
+}
+
+
+/**
+ * Worker for nemHCWinHandleMessageException & nemR3WinHandleExitException that
+ * checks if the given opcodes are of interest at all.
+ *
+ * @returns true if interesting, false if not.
+ * @param   cbOpcodes           Number of opcode bytes available.
+ * @param   pbOpcodes           The opcode bytes.
+ * @param   f64BitMode          Whether we're in 64-bit mode.
+ */
+DECLINLINE(bool) nemHcWinIsInterestingUndefinedOpcode(uint8_t cbOpcodes, uint8_t const *pbOpcodes, bool f64BitMode)
+{
+    /*
+     * Currently only interested in VMCALL and VMMCALL.
+     */
+    while (cbOpcodes >= 3)
+    {
+        switch (pbOpcodes[0])
+        {
+            case 0x0f:
+                switch (pbOpcodes[1])
+                {
+                    case 0x01:
+                        switch (pbOpcodes[2])
+                        {
+                            case 0xc1: /* 0f 01 c1  VMCALL */
+                                return true;
+                            case 0xd9: /* 0f 01 d9  VMMCALL */
+                                return true;
+                            default:
+                                break;
+                        }
+                        break;
+                }
+                break;
+
+            default:
+                return false;
+
+            /* prefixes */
+            case 0x40: case 0x41: case 0x42: case 0x43: case 0x44: case 0x45: case 0x46: case 0x47:
+            case 0x48: case 0x49: case 0x4a: case 0x4b: case 0x4c: case 0x4d: case 0x4e: case 0x4f:
+                if (!f64BitMode)
+                    return false;
+                RT_FALL_THRU();
+            case X86_OP_PRF_CS:
+            case X86_OP_PRF_SS:
+            case X86_OP_PRF_DS:
+            case X86_OP_PRF_ES:
+            case X86_OP_PRF_FS:
+            case X86_OP_PRF_GS:
+            case X86_OP_PRF_SIZE_OP:
+            case X86_OP_PRF_SIZE_ADDR:
+            case X86_OP_PRF_LOCK:
+            case X86_OP_PRF_REPZ:
+            case X86_OP_PRF_REPNZ:
+                cbOpcodes--;
+                pbOpcodes++;
+                continue;
+        }
+        break;
+    }
+    return false;
+}
+
+
+/**
+ * Copies state included in a exception intercept exit.
+ *
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information.
+ * @param   fClearXcpt      Clear pending exception.
+ */
+DECLINLINE(void) nemR3WinCopyStateFromExceptionMessage(PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit, bool fClearXcpt)
+{
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    if (fClearXcpt)
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
+}
+
+
+/**
+ * Advances the guest RIP by the number of bytes specified in @a cb.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cb          RIP increment value in bytes.
+ */
+DECLINLINE(void) nemHcWinAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
+{
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    pCtx->rip += cb;
+    /** @todo Why not clear RF too? */
+    CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
+}
+
+
+/**
+ * Hacks its way around the lovely mesa driver's backdoor accesses.
+ *
+ * @sa hmR0VmxHandleMesaDrvGp
+ * @sa hmR0SvmHandleMesaDrvGp
+ */
+static int nemHcWinHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx)
+{
+    Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_GPRS_MASK)));
+    RT_NOREF(pCtx);
+
+    /* For now we'll just skip the instruction. */
+    nemHcWinAdvanceRip(pVCpu, 1);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
+ * backdoor logging w/o checking what it is running inside.
+ *
+ * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
+ * backdoor port and magic numbers loaded in registers.
+ *
+ * @returns true if it is, false if it isn't.
+ * @sa      hmR0VmxIsMesaDrvGp
+ * @sa      hmR0SvmIsMesaDrvGp
+ */
+DECLINLINE(bool) nemHcWinIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, const uint8_t *pbInsn, uint32_t cbInsn)
+{
+    /* #GP(0) is already checked by caller. */
+
+    /* Check magic and port. */
+    Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RAX)));
+    if (pCtx->dx != UINT32_C(0x5658))
+        return false;
+    if (pCtx->rax != UINT32_C(0x564d5868))
+        return false;
+
+    /* Flat ring-3 CS. */
+    if (CPUMGetGuestCPL(pVCpu) != 3)
+        return false;
+    if (pCtx->cs.u64Base != 0)
+        return false;
+
+    /* 0xed:  IN eAX,dx */
+    if (cbInsn < 1) /* Play safe (shouldn't happen). */
+    {
+        uint8_t abInstr[1];
+        int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
+        if (RT_FAILURE(rc))
+            return false;
+        if (abInstr[0] != 0xed)
+            return false;
+    }
+    else
+    {
+        if (pbInsn[0] != 0xed)
+            return false;
+    }
+
+    return true;
+}
+
+
+/**
+ * Deals with MSR access exits (WHvRunVpExitReasonException).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemR3WinHandleExitException
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitException(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    /*
+     * Get most of the register state since we'll end up making IEM inject the
+     * event.  The exception isn't normally flaged as a pending event, so duh.
+     *
+     * Note! We can optimize this later with event injection.
+     */
+    Log4(("XcptExit/%u: %04x:%08RX64/%s: %x errcd=%#x parm=%RX64\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+          pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType,
+          pExit->VpException.ErrorCode, pExit->VpException.ExceptionParameter ));
+    nemR3WinCopyStateFromExceptionMessage(pVCpu, pExit, true /*fClearXcpt*/);
+    uint64_t fWhat = NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM;
+    if (pExit->VpException.ExceptionType == X86_XCPT_DB)
+        fWhat |= CPUMCTX_EXTRN_DR0_DR3 | CPUMCTX_EXTRN_DR7 | CPUMCTX_EXTRN_DR6;
+    VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, fWhat, "Xcpt");
+    if (rcStrict != VINF_SUCCESS)
+        return rcStrict;
+
+    /*
+     * Handle the intercept.
+     */
+    TRPMEVENT enmEvtType = TRPM_TRAP;
+    switch (pExit->VpException.ExceptionType)
+    {
+        /*
+         * We get undefined opcodes on VMMCALL(AMD) & VMCALL(Intel) instructions
+         * and need to turn them over to GIM.
+         *
+         * Note! We do not check fGIMTrapXcptUD here ASSUMING that GIM only wants
+         *       #UD for handling non-native hypercall instructions.  (IEM will
+         *       decode both and let the GIM provider decide whether to accept it.)
+         */
+        case X86_XCPT_UD:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUd);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_UD),
+                             pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+            if (nemHcWinIsInterestingUndefinedOpcode(pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes,
+                                                     pExit->VpContext.ExecutionState.EferLma && pExit->VpContext.Cs.Long ))
+            {
+                rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, pExit->VpContext.Rip,
+                                                        pExit->VpException.InstructionBytes,
+                                                        pExit->VpException.InstructionByteCount);
+                Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD -> emulated -> %Rrc\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
+                      nemR3WinExecStateToLogStr(&pExit->VpContext), VBOXSTRICTRC_VAL(rcStrict) ));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
+                return rcStrict;
+            }
+
+            Log4(("XcptExit/%u: %04x:%08RX64/%s: #UD [%.*Rhxs] -> re-injected\n", pVCpu->idCpu,
+                  pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext),
+                  pExit->VpException.InstructionByteCount, pExit->VpException.InstructionBytes ));
+            break;
+
+        /*
+         * Workaround the lovely mesa driver assuming that vmsvga means vmware
+         * hypervisor and tries to log stuff to the host.
+         */
+        case X86_XCPT_GP:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGp);
+            /** @todo r=bird: Need workaround in IEM for this, right?
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_GP),
+                             pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC()); */
+            if (   !pVCpu->nem.s.fTrapXcptGpForLovelyMesaDrv
+                || !nemHcWinIsMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx, pExit->VpException.InstructionBytes,
+                                        pExit->VpException.InstructionByteCount))
+            {
+#if 1 /** @todo Need to emulate instruction or we get a triple fault when trying to inject the \#GP... */
+                rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, pExit->VpContext.Rip,
+                                                        pExit->VpException.InstructionBytes,
+                                                        pExit->VpException.InstructionByteCount);
+                Log4(("XcptExit/%u: %04x:%08RX64/%s: #GP -> emulated -> %Rrc\n",
+                      pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
+                      nemR3WinExecStateToLogStr(&pExit->VpContext), VBOXSTRICTRC_VAL(rcStrict) ));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionUdHandled);
+                return rcStrict;
+#else
+                break;
+#endif
+            }
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionGpMesa);
+            return nemHcWinHandleMesaDrvGp(pVCpu, &pVCpu->cpum.GstCtx);
+
+        /*
+         * Filter debug exceptions.
+         */
+        case X86_XCPT_DB:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionDb);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_DB),
+                             pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+            Log4(("XcptExit/%u: %04x:%08RX64/%s: #DB - TODO\n",
+                  pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext) ));
+            break;
+
+        case X86_XCPT_BP:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitExceptionBp);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_XCPT_BP),
+                             pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+            Log4(("XcptExit/%u: %04x:%08RX64/%s: #BP - TODO - %u\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                  pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.InstructionLength));
+            enmEvtType = TRPM_SOFTWARE_INT; /* We're at the INT3 instruction, not after it. */
+            break;
+
+        /* This shouldn't happen. */
+        default:
+            AssertLogRelMsgFailedReturn(("ExceptionType=%#x\n", pExit->VpException.ExceptionType),  VERR_IEM_IPE_6);
+    }
+
+    /*
+     * Inject it.
+     */
+    rcStrict = IEMInjectTrap(pVCpu, pExit->VpException.ExceptionType, enmEvtType, pExit->VpException.ErrorCode,
+                             pExit->VpException.ExceptionParameter /*??*/, pExit->VpContext.InstructionLength);
+    Log4(("XcptExit/%u: %04x:%08RX64/%s: %#u -> injected -> %Rrc\n",
+          pVCpu->idCpu, pExit->VpContext.Cs.Selector, pExit->VpContext.Rip,
+          nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpException.ExceptionType, VBOXSTRICTRC_VAL(rcStrict) ));
+
+    RT_NOREF_PV(pVM);
+    return rcStrict;
+}
+
+
+/**
+ * Deals with MSR access exits (WHvRunVpExitReasonUnrecoverableException).
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessageUnrecoverableException
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExitUnrecoverableException(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+#if 0
+    /*
+     * Just copy the state we've got and handle it in the loop for now.
+     */
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    Log(("TripleExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+         pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
+    RT_NOREF_PV(pVM);
+    return VINF_EM_TRIPLE_FAULT;
+#else
+    /*
+     * Let IEM decide whether this is really it.
+     */
+    EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_UNRECOVERABLE_EXCEPTION),
+                     pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+    nemR3WinCopyStateFromX64Header(pVCpu, &pExit->VpContext);
+    VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM | CPUMCTX_EXTRN_ALL, "TripleExit");
+    if (rcStrict == VINF_SUCCESS)
+    {
+        rcStrict = IEMExecOne(pVCpu);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_SUCCESS\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                 pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags));
+            pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT; /* Make sure to reset pending #DB(0). */
+            return VINF_SUCCESS;
+        }
+        if (rcStrict == VINF_EM_TRIPLE_FAULT)
+            Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> VINF_EM_TRIPLE_FAULT!\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                 pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
+        else
+            Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (IEMExecOne)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+                 pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
+    }
+    else
+        Log(("UnrecovExit/%u: %04x:%08RX64/%s: RFL=%#RX64 -> %Rrc (state import)\n", pVCpu->idCpu, pExit->VpContext.Cs.Selector,
+             pExit->VpContext.Rip, nemR3WinExecStateToLogStr(&pExit->VpContext), pExit->VpContext.Rflags, VBOXSTRICTRC_VAL(rcStrict) ));
+    RT_NOREF_PV(pVM);
+    return rcStrict;
+#endif
+}
+
+
+/**
+ * Handles VM exits.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ * @param   pExit           The VM exit information to handle.
+ * @sa      nemHCWinHandleMessage
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemR3WinHandleExit(PVMCC pVM, PVMCPUCC pVCpu, WHV_RUN_VP_EXIT_CONTEXT const *pExit)
+{
+    switch (pExit->ExitReason)
+    {
+        case WHvRunVpExitReasonMemoryAccess:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMemUnmapped);
+            return nemR3WinHandleExitMemory(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonX64IoPortAccess:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitPortIo);
+            return nemR3WinHandleExitIoPort(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonX64Halt:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
+            EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
+                             pExit->VpContext.Rip + pExit->VpContext.Cs.Base, ASMReadTSC());
+            Log4(("HaltExit/%u\n", pVCpu->idCpu));
+            return VINF_EM_HALT;
+
+        case WHvRunVpExitReasonCanceled:
+            Log4(("CanceledExit/%u\n", pVCpu->idCpu));
+            return VINF_SUCCESS;
+
+        case WHvRunVpExitReasonX64InterruptWindow:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInterruptWindow);
+            return nemR3WinHandleExitInterruptWindow(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonX64Cpuid:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitCpuId);
+            return nemR3WinHandleExitCpuId(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonX64MsrAccess:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMsr);
+            return nemR3WinHandleExitMsr(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonException:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitException);
+            return nemR3WinHandleExitException(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonUnrecoverableException:
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitUnrecoverable);
+            return nemR3WinHandleExitUnrecoverableException(pVM, pVCpu, pExit);
+
+        case WHvRunVpExitReasonUnsupportedFeature:
+        case WHvRunVpExitReasonInvalidVpRegisterValue:
+            LogRel(("Unimplemented exit:\n%.*Rhxd\n", (int)sizeof(*pExit), pExit));
+            AssertLogRelMsgFailedReturn(("Unexpected exit on CPU #%u: %#x\n%.32Rhxd\n",
+                                         pVCpu->idCpu, pExit->ExitReason, pExit), VERR_NEM_IPE_3);
+
+        /* Undesired exits: */
+        case WHvRunVpExitReasonNone:
+        default:
+            LogRel(("Unknown exit:\n%.*Rhxd\n", (int)sizeof(*pExit), pExit));
+            AssertLogRelMsgFailedReturn(("Unknown exit on CPU #%u: %#x!\n", pVCpu->idCpu, pExit->ExitReason), VERR_NEM_IPE_3);
+    }
+}
+
+
+/**
+ * Deals with pending interrupt related force flags, may inject interrupt.
+ *
+ * @returns VBox strict status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context per CPU structure.
+ * @param   pfInterruptWindows  Where to return interrupt window flags.
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinHandleInterruptFF(PVMCC pVM, PVMCPUCC pVCpu, uint8_t *pfInterruptWindows)
+{
+    Assert(!TRPMHasTrap(pVCpu));
+    RT_NOREF_PV(pVM);
+
+    /*
+     * First update APIC.  We ASSUME this won't need TPR/CR8.
+     */
+    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
+    {
+        APICUpdatePendingInterrupts(pVCpu);
+        if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC
+                                      | VMCPU_FF_INTERRUPT_NMI  | VMCPU_FF_INTERRUPT_SMI))
+            return VINF_SUCCESS;
+    }
+
+    /*
+     * We don't currently implement SMIs.
+     */
+    AssertReturn(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), VERR_NEM_IPE_0);
+
+    /*
+     * Check if we've got the minimum of state required for deciding whether we
+     * can inject interrupts and NMIs.  If we don't have it, get all we might require
+     * for injection via IEM.
+     */
+    bool const fPendingNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+    uint64_t   fNeedExtrn  = CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS
+                           | (fPendingNmi ? CPUMCTX_EXTRN_INHIBIT_NMI : 0);
+    if (pVCpu->cpum.GstCtx.fExtrn & fNeedExtrn)
+    {
+        VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "IntFF");
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * NMI? Try deliver it first.
+     */
+    if (fPendingNmi)
+    {
+        if (   !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)
+            && !CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
+        {
+            VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI");
+            if (rcStrict == VINF_SUCCESS)
+            {
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+                rcStrict = IEMInjectTrap(pVCpu, X86_XCPT_NMI, TRPM_HARDWARE_INT, 0, 0, 0);
+                Log8(("Injected NMI on %u (%d)\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
+            }
+            return rcStrict;
+        }
+        *pfInterruptWindows |= NEM_WIN_INTW_F_NMI;
+        Log8(("NMI window pending on %u\n", pVCpu->idCpu));
+    }
+
+    /*
+     * APIC or PIC interrupt?
+     */
+    if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
+    {
+        /** @todo check NMI inhibiting here too! */
+        if (   !CPUMIsInInterruptShadow(&pVCpu->cpum.GstCtx)
+            && pVCpu->cpum.GstCtx.rflags.Bits.u1IF)
+        {
+            AssertCompile(NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT & CPUMCTX_EXTRN_APIC_TPR);
+            VBOXSTRICTRC rcStrict = nemHCWinImportStateIfNeededStrict(pVCpu, NEM_WIN_CPUMCTX_EXTRN_MASK_FOR_IEM_XCPT, "NMI");
+            if (rcStrict == VINF_SUCCESS)
+            {
+                uint8_t bInterrupt;
+                int rc = PDMGetInterrupt(pVCpu, &bInterrupt);
+                if (RT_SUCCESS(rc))
+                {
+                    Log8(("Injecting interrupt %#x on %u: %04x:%08RX64 efl=%#x\n", bInterrupt, pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eflags.u));
+                    rcStrict = IEMInjectTrap(pVCpu, bInterrupt, TRPM_HARDWARE_INT, 0, 0, 0);
+                    Log8(("Injected interrupt %#x on %u (%d)\n", bInterrupt, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
+                }
+                else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
+                {
+                    *pfInterruptWindows |= ((bInterrupt >> 4) << NEM_WIN_INTW_F_PRIO_SHIFT) | NEM_WIN_INTW_F_REGULAR;
+                    Log8(("VERR_APIC_INTR_MASKED_BY_TPR: *pfInterruptWindows=%#x\n", *pfInterruptWindows));
+                }
+                else
+                    Log8(("PDMGetInterrupt failed -> %Rrc\n", rc));
+            }
+            return rcStrict;
+        }
+
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC) && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC))
+        {
+            /* If only an APIC interrupt is pending, we need to know its priority. Otherwise we'll
+             * likely get pointless deliverability notifications with IF=1 but TPR still too high.
+             */
+            bool    fPendingIntr = false;
+            uint8_t bTpr = 0;
+            uint8_t bPendingIntr = 0;
+            int rc = APICGetTpr(pVCpu, &bTpr, &fPendingIntr, &bPendingIntr);
+            AssertRC(rc);
+            *pfInterruptWindows |= ((bPendingIntr >> 4) << NEM_WIN_INTW_F_PRIO_SHIFT) | NEM_WIN_INTW_F_REGULAR;
+            Log8(("Interrupt window pending on %u: %#x (bTpr=%#x fPendingIntr=%d bPendingIntr=%#x)\n",
+                  pVCpu->idCpu, *pfInterruptWindows, bTpr, fPendingIntr, bPendingIntr));
+        }
+        else
+        {
+            *pfInterruptWindows |= NEM_WIN_INTW_F_REGULAR;
+            Log8(("Interrupt window pending on %u: %#x\n", pVCpu->idCpu, *pfInterruptWindows));
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Inner NEM runloop for windows.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.
+ */
+NEM_TMPL_STATIC VBOXSTRICTRC nemHCWinRunGC(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 <=\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u));
+#ifdef LOG_ENABLED
+    if (LogIs3Enabled())
+        nemHCWinLogState(pVM, pVCpu);
+#endif
+
+    /*
+     * Try switch to NEM runloop state.
+     */
+    if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
+    { /* likely */ }
+    else
+    {
+        VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
+        LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * The run loop.
+     *
+     * Current approach to state updating to use the sledgehammer and sync
+     * everything every time.  This will be optimized later.
+     */
+    const bool      fSingleStepping     = DBGFIsStepping(pVCpu);
+//    const uint32_t  fCheckVmFFs         = !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK
+//                                                           : VM_FF_HP_R0_PRE_HM_STEP_MASK;
+//    const uint32_t  fCheckCpuFFs        = !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK;
+    VBOXSTRICTRC    rcStrict            = VINF_SUCCESS;
+    for (unsigned iLoop = 0;; iLoop++)
+    {
+        /*
+         * Pending interrupts or such?  Need to check and deal with this prior
+         * to the state syncing.
+         */
+        pVCpu->nem.s.fDesiredInterruptWindows = 0;
+        if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_PIC
+                                     | VMCPU_FF_INTERRUPT_NMI  | VMCPU_FF_INTERRUPT_SMI))
+        {
+            /* Try inject interrupt. */
+            rcStrict = nemHCWinHandleInterruptFF(pVM, pVCpu, &pVCpu->nem.s.fDesiredInterruptWindows);
+            if (rcStrict == VINF_SUCCESS)
+            { /* likely */ }
+            else
+            {
+                LogFlow(("NEM/%u: breaking: nemHCWinHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
+                break;
+            }
+        }
+
+#ifndef NEM_WIN_WITH_A20
+        /*
+         * Do not execute in hyper-V if the A20 isn't enabled.
+         */
+        if (PGMPhysIsA20Enabled(pVCpu))
+        { /* likely */ }
+        else
+        {
+            rcStrict = VINF_EM_RESCHEDULE_REM;
+            LogFlow(("NEM/%u: breaking: A20 disabled\n", pVCpu->idCpu));
+            break;
+        }
+#endif
+
+        /*
+         * Ensure that hyper-V has the whole state.
+         * (We always update the interrupt windows settings when active as hyper-V seems
+         * to forget about it after an exit.)
+         */
+        if (      (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK))
+               !=                              (CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_NEM_WIN_MASK)
+            || (  (   pVCpu->nem.s.fDesiredInterruptWindows
+                   || pVCpu->nem.s.fCurrentInterruptWindows != pVCpu->nem.s.fDesiredInterruptWindows) ) )
+        {
+            int rc2 = nemHCWinCopyStateToHyperV(pVM, pVCpu);
+            AssertRCReturn(rc2, rc2);
+        }
+
+        /*
+         * Poll timers and run for a bit.
+         *
+         * With the VID approach (ring-0 or ring-3) we can specify a timeout here,
+         * so we take the time of the next timer event and uses that as a deadline.
+         * The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
+         */
+        /** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
+         *        the whole polling job when timers have changed... */
+        uint64_t       offDeltaIgnored;
+        uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
+        if (   !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
+            && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
+        {
+            if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
+            {
+#ifdef LOG_ENABLED
+                if (LogIsFlowEnabled())
+                {
+                    static const WHV_REGISTER_NAME s_aNames[6] = { WHvX64RegisterCs, WHvX64RegisterRip, WHvX64RegisterRflags,
+                                                                   WHvX64RegisterSs, WHvX64RegisterRsp, WHvX64RegisterCr0 };
+                    WHV_REGISTER_VALUE aRegs[RT_ELEMENTS(s_aNames)] = { {{0, 0} } };
+                    WHvGetVirtualProcessorRegisters(pVM->nem.s.hPartition, pVCpu->idCpu, s_aNames, RT_ELEMENTS(s_aNames), aRegs);
+                    LogFlow(("NEM/%u: Entry @ %04x:%08RX64 IF=%d EFL=%#RX64 SS:RSP=%04x:%08RX64 cr0=%RX64\n",
+                             pVCpu->idCpu, aRegs[0].Segment.Selector, aRegs[1].Reg64, RT_BOOL(aRegs[2].Reg64 & X86_EFL_IF),
+                             aRegs[2].Reg64, aRegs[3].Segment.Selector, aRegs[4].Reg64, aRegs[5].Reg64));
+                }
+#endif
+                WHV_RUN_VP_EXIT_CONTEXT ExitReason = {0};
+                TMNotifyStartOfExecution(pVM, pVCpu);
+
+                HRESULT hrc = WHvRunVirtualProcessor(pVM->nem.s.hPartition, pVCpu->idCpu, &ExitReason, sizeof(ExitReason));
+
+                VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
+                TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
+#ifdef LOG_ENABLED
+                LogFlow(("NEM/%u: Exit  @ %04X:%08RX64 IF=%d CR8=%#x Reason=%#x\n", pVCpu->idCpu, ExitReason.VpContext.Cs.Selector,
+                         ExitReason.VpContext.Rip, RT_BOOL(ExitReason.VpContext.Rflags & X86_EFL_IF), ExitReason.VpContext.Cr8,
+                         ExitReason.ExitReason));
+#endif
+                if (SUCCEEDED(hrc))
+                {
+                    /*
+                     * Deal with the message.
+                     */
+                    rcStrict = nemR3WinHandleExit(pVM, pVCpu, &ExitReason);
+                    if (rcStrict == VINF_SUCCESS)
+                    { /* hopefully likely */ }
+                    else
+                    {
+                        LogFlow(("NEM/%u: breaking: nemHCWinHandleMessage -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
+                        STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
+                        break;
+                    }
+                }
+                else
+                    AssertLogRelMsgFailedReturn(("WHvRunVirtualProcessor failed for CPU #%u: %#x (%u)\n",
+                                                 pVCpu->idCpu, hrc, GetLastError()),
+                                                VERR_NEM_IPE_0);
+
+                /*
+                 * If no relevant FFs are pending, loop.
+                 */
+                if (   !VM_FF_IS_ANY_SET(   pVM,   !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK    : VM_FF_HP_R0_PRE_HM_STEP_MASK)
+                    && !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
+                    continue;
+
+                /** @todo Try handle pending flags, not just return to EM loops.  Take care
+                 *        not to set important RCs here unless we've handled a message. */
+                LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
+                         pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
+            }
+            else
+            {
+                LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
+                STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
+            }
+        }
+        else
+        {
+            LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
+        }
+        break;
+    } /* the run loop */
+
+
+    /*
+     * If the CPU is running, make sure to stop it before we try sync back the
+     * state and return to EM.  We don't sync back the whole state if we can help it.
+     */
+    if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
+        VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
+
+    if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT)))
+    {
+        /* Try anticipate what we might need. */
+        uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI;
+        if (   (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
+            || RT_FAILURE(rcStrict))
+            fImport = CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT);
+        else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC | VMCPU_FF_INTERRUPT_APIC
+                                          | VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI))
+            fImport |= IEM_CPUMCTX_EXTRN_XCPT_MASK;
+
+        if (pVCpu->cpum.GstCtx.fExtrn & fImport)
+        {
+            int rc2 = nemHCWinCopyStateFromHyperV(pVM, pVCpu, fImport | CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT);
+            if (RT_SUCCESS(rc2))
+                pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
+            else if (RT_SUCCESS(rcStrict))
+                rcStrict = rc2;
+            if (!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_ALL | (CPUMCTX_EXTRN_NEM_WIN_MASK & ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT))))
+                pVCpu->cpum.GstCtx.fExtrn = 0;
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
+        }
+        else
+        {
+            STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
+            pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_NEM_WIN_EVENT_INJECT;
+        }
+    }
+    else
+    {
+        STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
+        pVCpu->cpum.GstCtx.fExtrn = 0;
+    }
+
+    LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel,
+             pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u, VBOXSTRICTRC_VAL(rcStrict) ));
+    return rcStrict;
+}
+
+
+/**
+ * @callback_method_impl{FNPGMPHYSNEMCHECKPAGE}
+ */
+NEM_TMPL_STATIC DECLCALLBACK(int) nemHCWinUnsetForA20CheckerCallback(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys,
+                                                                     PPGMPHYSNEMPAGEINFO pInfo, void *pvUser)
+{
+    /* We'll just unmap the memory. */
+    if (pInfo->u2NemState > NEM_WIN_PAGE_STATE_UNMAPPED)
+    {
+        HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhys, X86_PAGE_SIZE);
+        if (SUCCEEDED(hrc))
+        {
+            STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
+            uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+            Log5(("NEM GPA unmapped/A20: %RGp (was %s, cMappedPages=%u)\n", GCPhys, g_apszPageStates[pInfo->u2NemState], cMappedPages));
+            pInfo->u2NemState = NEM_WIN_PAGE_STATE_UNMAPPED;
+        }
+        else
+        {
+            STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
+            LogRel(("nemHCWinUnsetForA20CheckerCallback/unmap: GCPhys=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                    GCPhys, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+            return VERR_NEM_IPE_2;
+        }
+    }
+    RT_NOREF(pVCpu, pvUser);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Unmaps a page from Hyper-V for the purpose of emulating A20 gate behavior.
+ *
+ * @returns The PGMPhysNemQueryPageInfo result.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhys          The page to unmap.
+ */
+NEM_TMPL_STATIC int nemHCWinUnmapPageForA20Gate(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys)
+{
+    PGMPHYSNEMPAGEINFO Info;
+    return PGMPhysNemPageInfoChecker(pVM, pVCpu, GCPhys, false /*fMakeWritable*/, &Info,
+                                     nemHCWinUnsetForA20CheckerCallback, NULL);
+}
+
+
+void nemHCNativeNotifyHandlerPhysicalRegister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb)
+{
+    Log5(("nemHCNativeNotifyHandlerPhysicalRegister: %RGp LB %RGp enmKind=%d\n", GCPhys, cb, enmKind));
+    NOREF(pVM); NOREF(enmKind); NOREF(GCPhys); NOREF(cb);
+}
+
+
+VMM_INT_DECL(void) NEMHCNotifyHandlerPhysicalDeregister(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhys, RTGCPHYS cb,
+                                                        RTR3PTR pvMemR3, uint8_t *pu2State)
+{
+    Log5(("NEMHCNotifyHandlerPhysicalDeregister: %RGp LB %RGp enmKind=%d pvMemR3=%p pu2State=%p (%d)\n",
+          GCPhys, cb, enmKind, pvMemR3, pu2State, *pu2State));
+
+    *pu2State = UINT8_MAX;
+    if (pvMemR3)
+    {
+        STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRange, a);
+        HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvMemR3, GCPhys, cb,
+                                     WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute | WHvMapGpaRangeFlagWrite);
+        STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRange, a);
+        if (SUCCEEDED(hrc))
+            *pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
+        else
+            AssertLogRelMsgFailed(("NEMHCNotifyHandlerPhysicalDeregister: WHvMapGpaRange(,%p,%RGp,%RGp,) -> %Rhrc\n",
+                                   pvMemR3, GCPhys, cb, hrc));
+    }
+    RT_NOREF(enmKind);
+}
+
+
+void nemHCNativeNotifyHandlerPhysicalModify(PVMCC pVM, PGMPHYSHANDLERKIND enmKind, RTGCPHYS GCPhysOld,
+                                            RTGCPHYS GCPhysNew, RTGCPHYS cb, bool fRestoreAsRAM)
+{
+    Log5(("nemHCNativeNotifyHandlerPhysicalModify: %RGp LB %RGp -> %RGp enmKind=%d fRestoreAsRAM=%d\n",
+          GCPhysOld, cb, GCPhysNew, enmKind, fRestoreAsRAM));
+    NOREF(pVM); NOREF(enmKind); NOREF(GCPhysOld); NOREF(GCPhysNew); NOREF(cb); NOREF(fRestoreAsRAM);
+}
+
+
+/**
+ * Worker that maps pages into Hyper-V.
+ *
+ * This is used by the PGM physical page notifications as well as the memory
+ * access VMEXIT handlers.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ * @param   GCPhysSrc       The source page address.
+ * @param   GCPhysDst       The hyper-V destination page.  This may differ from
+ *                          GCPhysSrc when A20 is disabled.
+ * @param   fPageProt       NEM_PAGE_PROT_XXX.
+ * @param   pu2State        Our page state (input/output).
+ * @param   fBackingChanged Set if the page backing is being changed.
+ * @thread  EMT(pVCpu)
+ */
+NEM_TMPL_STATIC int nemHCNativeSetPhysPage(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhysSrc, RTGCPHYS GCPhysDst,
+                                           uint32_t fPageProt, uint8_t *pu2State, bool fBackingChanged)
+{
+    /*
+     * Looks like we need to unmap a page before we can change the backing
+     * or even modify the protection.  This is going to be *REALLY* efficient.
+     * PGM lends us two bits to keep track of the state here.
+     */
+    RT_NOREF(pVCpu);
+    uint8_t const u2OldState = *pu2State;
+    uint8_t const u2NewState = fPageProt & NEM_PAGE_PROT_WRITE ? NEM_WIN_PAGE_STATE_WRITABLE
+                             : fPageProt & NEM_PAGE_PROT_READ  ? NEM_WIN_PAGE_STATE_READABLE : NEM_WIN_PAGE_STATE_UNMAPPED;
+    if (   fBackingChanged
+        || u2NewState != u2OldState)
+    {
+        if (u2OldState > NEM_WIN_PAGE_STATE_UNMAPPED)
+        {
+            STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
+            HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst, X86_PAGE_SIZE);
+            STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
+            if (SUCCEEDED(hrc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+                STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
+                uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+                if (u2NewState == NEM_WIN_PAGE_STATE_UNMAPPED)
+                {
+                    Log5(("NEM GPA unmapped/set: %RGp (was %s, cMappedPages=%u)\n",
+                          GCPhysDst, g_apszPageStates[u2OldState], cMappedPages));
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+            {
+                STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
+                LogRel(("nemHCNativeSetPhysPage/unmap: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                        GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+                return VERR_NEM_INIT_FAILED;
+            }
+        }
+    }
+
+    /*
+     * Writeable mapping?
+     */
+    if (fPageProt & NEM_PAGE_PROT_WRITE)
+    {
+        void *pvPage;
+        int rc = nemR3NativeGCPhys2R3PtrWriteable(pVM, GCPhysSrc, &pvPage);
+        if (RT_SUCCESS(rc))
+        {
+            HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, pvPage, GCPhysDst, X86_PAGE_SIZE,
+                                         WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute | WHvMapGpaRangeFlagWrite);
+            if (SUCCEEDED(hrc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_WRITABLE;
+                STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
+                uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+                Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
+                      GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
+                return VINF_SUCCESS;
+            }
+            STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
+            LogRel(("nemHCNativeSetPhysPage/writable: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                    GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+            return VERR_NEM_INIT_FAILED;
+        }
+        LogRel(("nemHCNativeSetPhysPage/writable: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
+        return rc;
+    }
+
+    if (fPageProt & NEM_PAGE_PROT_READ)
+    {
+        const void *pvPage;
+        int rc = nemR3NativeGCPhys2R3PtrReadOnly(pVM, GCPhysSrc, &pvPage);
+        if (RT_SUCCESS(rc))
+        {
+            STAM_REL_PROFILE_START(&pVM->nem.s.StatProfMapGpaRangePage, a);
+            HRESULT hrc = WHvMapGpaRange(pVM->nem.s.hPartition, (void *)pvPage, GCPhysDst, X86_PAGE_SIZE,
+                                         WHvMapGpaRangeFlagRead | WHvMapGpaRangeFlagExecute);
+            STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfMapGpaRangePage, a);
+            if (SUCCEEDED(hrc))
+            {
+                *pu2State = NEM_WIN_PAGE_STATE_READABLE;
+                STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPage);
+                uint32_t cMappedPages = ASMAtomicIncU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+                Log5(("NEM GPA mapped/set: %RGp %s (was %s, cMappedPages=%u)\n",
+                      GCPhysDst, g_apszPageStates[u2NewState], g_apszPageStates[u2OldState], cMappedPages));
+                return VINF_SUCCESS;
+            }
+            STAM_REL_COUNTER_INC(&pVM->nem.s.StatMapPageFailed);
+            LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysDst=%RGp hrc=%Rhrc (%#x) Last=%#x/%u\n",
+                    GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+            return VERR_NEM_INIT_FAILED;
+        }
+        LogRel(("nemHCNativeSetPhysPage/readonly: GCPhysSrc=%RGp rc=%Rrc\n", GCPhysSrc, rc));
+        return rc;
+    }
+
+    /* We already unmapped it above. */
+    *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+    return VINF_SUCCESS;
+}
+
+
+NEM_TMPL_STATIC int nemHCJustUnmapPageFromHyperV(PVMCC pVM, RTGCPHYS GCPhysDst, uint8_t *pu2State)
+{
+    if (*pu2State <= NEM_WIN_PAGE_STATE_UNMAPPED)
+    {
+        Log5(("nemHCJustUnmapPageFromHyperV: %RGp == unmapped\n", GCPhysDst));
+        *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+        return VINF_SUCCESS;
+    }
+
+    STAM_REL_PROFILE_START(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
+    HRESULT hrc = WHvUnmapGpaRange(pVM->nem.s.hPartition, GCPhysDst & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, X86_PAGE_SIZE);
+    STAM_REL_PROFILE_STOP(&pVM->nem.s.StatProfUnmapGpaRangePage, a);
+    if (SUCCEEDED(hrc))
+    {
+        STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPage);
+        uint32_t cMappedPages = ASMAtomicDecU32(&pVM->nem.s.cMappedPages); NOREF(cMappedPages);
+        *pu2State = NEM_WIN_PAGE_STATE_UNMAPPED;
+        Log5(("nemHCJustUnmapPageFromHyperV: %RGp => unmapped (total %u)\n", GCPhysDst, cMappedPages));
+        return VINF_SUCCESS;
+    }
+    STAM_REL_COUNTER_INC(&pVM->nem.s.StatUnmapPageFailed);
+    LogRel(("nemHCJustUnmapPageFromHyperV(%RGp): failed! hrc=%Rhrc (%#x) Last=%#x/%u\n",
+            GCPhysDst, hrc, hrc, RTNtLastStatusValue(), RTNtLastErrorValue()));
+    return VERR_NEM_IPE_6;
+}
+
+
+int nemHCNativeNotifyPhysPageAllocated(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, uint32_t fPageProt,
+                                       PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    Log5(("nemHCNativeNotifyPhysPageAllocated: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
+          GCPhys, HCPhys, fPageProt, enmType, *pu2State));
+    RT_NOREF_PV(HCPhys); RT_NOREF_PV(enmType);
+
+    int rc;
+    RT_NOREF_PV(fPageProt);
+#ifdef NEM_WIN_WITH_A20
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
+#endif
+        rc = nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+#ifdef NEM_WIN_WITH_A20
+    else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
+        rc = nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+    else
+        rc = VINF_SUCCESS; /* ignore since we've got the alias page at this address. */
+#endif
+    return rc;
+}
+
+
+VMM_INT_DECL(void) NEMHCNotifyPhysPageProtChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhys, RTR3PTR pvR3, uint32_t fPageProt,
+                                                  PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    Log5(("NEMHCNotifyPhysPageProtChanged: %RGp HCPhys=%RHp fPageProt=%#x enmType=%d *pu2State=%d\n",
+          GCPhys, HCPhys, fPageProt, enmType, *pu2State));
+    Assert(VM_IS_NEM_ENABLED(pVM));
+    RT_NOREF(HCPhys, enmType, pvR3);
+
+    RT_NOREF_PV(fPageProt);
+#ifdef NEM_WIN_WITH_A20
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
+#endif
+        nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+#ifdef NEM_WIN_WITH_A20
+    else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
+        nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+    /* else: ignore since we've got the alias page at this address. */
+#endif
+}
+
+
+VMM_INT_DECL(void) NEMHCNotifyPhysPageChanged(PVMCC pVM, RTGCPHYS GCPhys, RTHCPHYS HCPhysPrev, RTHCPHYS HCPhysNew,
+                                              RTR3PTR pvNewR3, uint32_t fPageProt, PGMPAGETYPE enmType, uint8_t *pu2State)
+{
+    Log5(("nemHCNativeNotifyPhysPageChanged: %RGp HCPhys=%RHp->%RHp pvNewR3=%p fPageProt=%#x enmType=%d *pu2State=%d\n",
+          GCPhys, HCPhysPrev, HCPhysNew, pvNewR3, fPageProt, enmType, *pu2State));
+    Assert(VM_IS_NEM_ENABLED(pVM));
+    RT_NOREF(HCPhysPrev, HCPhysNew, pvNewR3, enmType);
+
+    RT_NOREF_PV(fPageProt);
+#ifdef NEM_WIN_WITH_A20
+    if (   pVM->nem.s.fA20Enabled
+        || !NEM_WIN_IS_RELEVANT_TO_A20(GCPhys))
+#endif
+        nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+#ifdef NEM_WIN_WITH_A20
+    else if (!NEM_WIN_IS_SUBJECT_TO_A20(GCPhys))
+        nemHCJustUnmapPageFromHyperV(pVM, GCPhys, pu2State);
+    /* else: ignore since we've got the alias page at this address. */
+#endif
+}
+
+
+/**
+ * Returns features supported by the NEM backend.
+ *
+ * @returns Flags of features supported by the native NEM backend.
+ * @param   pVM             The cross context VM structure.
+ */
+VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
+{
+    RT_NOREF(pVM);
+    /** @todo Make use of the WHvGetVirtualProcessorXsaveState/WHvSetVirtualProcessorXsaveState
+     * interface added in 2019 to enable passthrough of xsave/xrstor (and depending) features to the guest. */
+    /** @todo Is NEM_FEAT_F_FULL_GST_EXEC always true? */
+    return NEM_FEAT_F_NESTED_PAGING | NEM_FEAT_F_FULL_GST_EXEC;
+}
+
diff --git a/src/VBox/VMM/VMMAll/PDMAll.cpp b/src/VBox/VMM/VMMAll/PDMAll.cpp
new file mode 100644
index 00000000..79301070
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAll.cpp
@@ -0,0 +1,428 @@
+/* $Id: PDMAll.cpp $ */
+/** @file
+ * PDM Critical Sections
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/vmm/apic.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+
+#include "PDMInline.h"
+#include "dtrace/VBoxVMM.h"
+
+
+
+/**
+ * Gets the pending interrupt.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_APIC_INTR_MASKED_BY_TPR when an APIC interrupt is pending but
+ *          can't be delivered due to TPR priority.
+ * @retval  VERR_NO_DATA if there is no interrupt to be delivered (either APIC
+ *          has been software-disabled since it flagged something was pending,
+ *          or other reasons).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pu8Interrupt    Where to store the interrupt.
+ */
+VMMDECL(int) PDMGetInterrupt(PVMCPUCC pVCpu, uint8_t *pu8Interrupt)
+{
+    /*
+     * The local APIC has a higher priority than the PIC.
+     */
+    int rc = VERR_NO_DATA;
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC))
+    {
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_APIC);
+        uint32_t uTagSrc;
+        rc = APICGetInterrupt(pVCpu, pu8Interrupt, &uTagSrc);
+        if (RT_SUCCESS(rc))
+        {
+            VBOXVMM_PDM_IRQ_GET(pVCpu, RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc), *pu8Interrupt);
+            Log8(("PDMGetInterrupt: irq=%#x tag=%#x (apic)\n", *pu8Interrupt, uTagSrc));
+            return VINF_SUCCESS;
+        }
+        /* else if it's masked by TPR/PPR/whatever, go ahead checking the PIC. Such masked
+           interrupts shouldn't prevent ExtINT from being delivered. */
+    }
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    pdmLock(pVM);
+
+    /*
+     * Check the PIC.
+     */
+    if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC))
+    {
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_PIC);
+        Assert(pVM->pdm.s.Pic.CTX_SUFF(pDevIns));
+        Assert(pVM->pdm.s.Pic.CTX_SUFF(pfnGetInterrupt));
+        uint32_t uTagSrc;
+        int i = pVM->pdm.s.Pic.CTX_SUFF(pfnGetInterrupt)(pVM->pdm.s.Pic.CTX_SUFF(pDevIns), &uTagSrc);
+        AssertMsg(i <= 255 && i >= 0, ("i=%d\n", i));
+        if (i >= 0)
+        {
+            pdmUnlock(pVM);
+            *pu8Interrupt = (uint8_t)i;
+            VBOXVMM_PDM_IRQ_GET(pVCpu, RT_LOWORD(uTagSrc), RT_HIWORD(uTagSrc), i);
+            Log8(("PDMGetInterrupt: irq=%#x tag=%#x (pic)\n", i, uTagSrc));
+            return VINF_SUCCESS;
+        }
+    }
+
+    /*
+     * One scenario where we may possibly get here is if the APIC signaled a pending interrupt,
+     * got an APIC MMIO/MSR VM-exit which disabled the APIC. We could, in theory, clear the APIC
+     * force-flag from all the places which disables the APIC but letting PDMGetInterrupt() fail
+     * without returning a valid interrupt still needs to be handled for the TPR masked case,
+     * so we shall just handle it here regardless if we choose to update the APIC code in the future.
+     */
+
+    pdmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Sets the pending interrupt coming from ISA source or HPET.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   u8Irq           The IRQ line.
+ * @param   u8Level         The new level.
+ * @param   uTagSrc         The IRQ tag and source tracer ID.
+ */
+VMMDECL(int) PDMIsaSetIrq(PVMCC pVM, uint8_t u8Irq, uint8_t u8Level, uint32_t uTagSrc)
+{
+    pdmLock(pVM);
+
+    /** @todo put the IRQ13 code elsewhere to avoid this unnecessary bloat. */
+    if (!uTagSrc && (u8Level & PDM_IRQ_LEVEL_HIGH)) /* FPU IRQ */
+    {
+        if (u8Level == PDM_IRQ_LEVEL_HIGH)
+            VBOXVMM_PDM_IRQ_HIGH(VMMGetCpu(pVM), 0, 0);
+        else
+            VBOXVMM_PDM_IRQ_HILO(VMMGetCpu(pVM), 0, 0);
+    }
+    Log9(("PDMIsaSetIrq: irq=%#x lvl=%u tag=%#x\n", u8Irq, u8Level, uTagSrc));
+
+    int rc = VERR_PDM_NO_PIC_INSTANCE;
+/** @todo r=bird: This code is incorrect, as it ASSUMES the PIC and I/O APIC
+ *        are always ring-0 enabled! */
+    if (pVM->pdm.s.Pic.CTX_SUFF(pDevIns))
+    {
+        Assert(pVM->pdm.s.Pic.CTX_SUFF(pfnSetIrq));
+        pVM->pdm.s.Pic.CTX_SUFF(pfnSetIrq)(pVM->pdm.s.Pic.CTX_SUFF(pDevIns), u8Irq, u8Level, uTagSrc);
+        rc = VINF_SUCCESS;
+    }
+
+    if (pVM->pdm.s.IoApic.CTX_SUFF(pDevIns))
+    {
+        Assert(pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq));
+
+        /*
+         * Apply Interrupt Source Override rules.
+         * See ACPI 4.0 specification 5.2.12.4 and 5.2.12.5 for details on
+         * interrupt source override.
+         * Shortly, ISA IRQ0 is electically connected to pin 2 on IO-APIC, and some OSes,
+         * notably recent OS X rely upon this configuration.
+         * If changing, also update override rules in MADT and MPS.
+         */
+        /* ISA IRQ0 routed to pin 2, all others ISA sources are identity mapped */
+        if (u8Irq == 0)
+            u8Irq = 2;
+
+        pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq)(pVM->pdm.s.IoApic.CTX_SUFF(pDevIns), NIL_PCIBDF, u8Irq, u8Level, uTagSrc);
+        rc = VINF_SUCCESS;
+    }
+
+    if (!uTagSrc && u8Level == PDM_IRQ_LEVEL_LOW)
+        VBOXVMM_PDM_IRQ_LOW(VMMGetCpu(pVM), 0, 0);
+    pdmUnlock(pVM);
+    return rc;
+}
+
+
+/**
+ * Sets the pending I/O APIC interrupt.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   u8Irq       The IRQ line.
+ * @param   uBusDevFn   The bus:device:function of the device initiating the IRQ.
+ *                      Pass NIL_PCIBDF when it's not a PCI device or interrupt.
+ * @param   u8Level     The new level.
+ * @param   uTagSrc     The IRQ tag and source tracer ID.
+ */
+VMM_INT_DECL(int) PDMIoApicSetIrq(PVM pVM, PCIBDF uBusDevFn, uint8_t u8Irq, uint8_t u8Level, uint32_t uTagSrc)
+{
+    Log9(("PDMIoApicSetIrq: irq=%#x lvl=%u tag=%#x src=%#x\n", u8Irq, u8Level, uTagSrc, uBusDevFn));
+    if (pVM->pdm.s.IoApic.CTX_SUFF(pDevIns))
+    {
+        Assert(pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq));
+        pVM->pdm.s.IoApic.CTX_SUFF(pfnSetIrq)(pVM->pdm.s.IoApic.CTX_SUFF(pDevIns), uBusDevFn, u8Irq, u8Level, uTagSrc);
+        return VINF_SUCCESS;
+    }
+    return VERR_PDM_NO_PIC_INSTANCE;
+}
+
+
+/**
+ * Broadcasts an EOI to the I/O APIC(s).
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   uVector     The interrupt vector corresponding to the EOI.
+ */
+VMM_INT_DECL(void) PDMIoApicBroadcastEoi(PVMCC pVM, uint8_t uVector)
+{
+    /*
+     * At present, we support only a maximum of one I/O APIC per-VM. If we ever implement having
+     * multiple I/O APICs per-VM, we'll have to broadcast this EOI to all of the I/O APICs.
+     */
+    PCPDMIOAPIC pIoApic = &pVM->pdm.s.IoApic;
+#ifdef IN_RING0
+    if (pIoApic->pDevInsR0)
+    {
+        Assert(pIoApic->pfnSetEoiR0);
+        pIoApic->pfnSetEoiR0(pIoApic->pDevInsR0, uVector);
+    }
+    else if (pIoApic->pDevInsR3)
+    {
+        /* Queue for ring-3 execution. */
+        PPDMDEVHLPTASK pTask = (PPDMDEVHLPTASK)PDMQueueAlloc(pVM, pVM->pdm.s.hDevHlpQueue, pVM);
+        if (pTask)
+        {
+            pTask->enmOp = PDMDEVHLPTASKOP_IOAPIC_SET_EOI;
+            pTask->pDevInsR3 = NIL_RTR3PTR; /* not required */
+            pTask->u.IoApicSetEoi.uVector = uVector;
+            PDMQueueInsert(pVM, pVM->pdm.s.hDevHlpQueue, pVM, &pTask->Core);
+        }
+        else
+            AssertMsgFailed(("We're out of devhlp queue items!!!\n"));
+    }
+#else
+    if (pIoApic->pDevInsR3)
+    {
+        Assert(pIoApic->pfnSetEoiR3);
+        pIoApic->pfnSetEoiR3(pIoApic->pDevInsR3, uVector);
+    }
+#endif
+}
+
+
+/**
+ * Send a MSI to an I/O APIC.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   uBusDevFn   The bus:device:function of the device initiating the MSI.
+ * @param   pMsi        The MSI to send.
+ * @param   uTagSrc     The IRQ tag and source tracer ID.
+ */
+VMM_INT_DECL(void) PDMIoApicSendMsi(PVMCC pVM, PCIBDF uBusDevFn, PCMSIMSG pMsi, uint32_t uTagSrc)
+{
+    Log9(("PDMIoApicSendMsi: addr=%#RX64 data=%#RX32 tag=%#x src=%#x\n", pMsi->Addr.u64, pMsi->Data.u32, uTagSrc, uBusDevFn));
+    PCPDMIOAPIC pIoApic = &pVM->pdm.s.IoApic;
+#ifdef IN_RING0
+    if (pIoApic->pDevInsR0)
+        pIoApic->pfnSendMsiR0(pIoApic->pDevInsR0, uBusDevFn, pMsi, uTagSrc);
+    else if (pIoApic->pDevInsR3)
+    {
+        /* Queue for ring-3 execution. */
+        PPDMDEVHLPTASK pTask = (PPDMDEVHLPTASK)PDMQueueAlloc(pVM, pVM->pdm.s.hDevHlpQueue, pVM);
+        if (pTask)
+        {
+            pTask->enmOp = PDMDEVHLPTASKOP_IOAPIC_SEND_MSI;
+            pTask->pDevInsR3 = NIL_RTR3PTR; /* not required */
+            pTask->u.IoApicSendMsi.uBusDevFn = uBusDevFn;
+            pTask->u.IoApicSendMsi.Msi       = *pMsi;
+            pTask->u.IoApicSendMsi.uTagSrc   = uTagSrc;
+            PDMQueueInsert(pVM, pVM->pdm.s.hDevHlpQueue, pVM, &pTask->Core);
+        }
+        else
+            AssertMsgFailed(("We're out of devhlp queue items!!!\n"));
+    }
+#else
+    if (pIoApic->pDevInsR3)
+    {
+        Assert(pIoApic->pfnSendMsiR3);
+        pIoApic->pfnSendMsiR3(pIoApic->pDevInsR3, uBusDevFn, pMsi, uTagSrc);
+    }
+#endif
+}
+
+
+
+/**
+ * Returns the presence of an IO-APIC.
+ *
+ * @returns true if an IO-APIC is present.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(bool) PDMHasIoApic(PVM pVM)
+{
+    return pVM->pdm.s.IoApic.pDevInsR3 != NULL;
+}
+
+
+/**
+ * Returns the presence of an APIC.
+ *
+ * @returns true if an APIC is present.
+ * @param   pVM         The cross context VM structure.
+ */
+VMM_INT_DECL(bool) PDMHasApic(PVM pVM)
+{
+    return pVM->pdm.s.Apic.pDevInsR3 != NIL_RTR3PTR;
+}
+
+
+/**
+ * Translates a ring-0 device instance index to a pointer.
+ *
+ * This is used by PGM for device access handlers.
+ *
+ * @returns Device instance pointer if valid index, otherwise NULL (asserted).
+ * @param   pVM         The cross context VM structure.
+ * @param   idxR0Device The ring-0 device instance index.
+ */
+VMM_INT_DECL(PPDMDEVINS) PDMDeviceRing0IdxToInstance(PVMCC pVM, uint64_t idxR0Device)
+{
+#ifdef IN_RING0
+    AssertMsgReturn(idxR0Device < RT_ELEMENTS(pVM->pdmr0.s.apDevInstances), ("%#RX64\n", idxR0Device), NULL);
+    PPDMDEVINS pDevIns = pVM->pdmr0.s.apDevInstances[idxR0Device];
+#elif defined(IN_RING3)
+    AssertMsgReturn(idxR0Device < RT_ELEMENTS(pVM->pdm.s.apDevRing0Instances), ("%#RX64\n", idxR0Device), NULL);
+    PPDMDEVINS pDevIns = pVM->pdm.s.apDevRing0Instances[idxR0Device];
+#else
+# error "Unsupported context"
+#endif
+    AssertMsg(pDevIns, ("%#RX64\n", idxR0Device));
+    return pDevIns;
+}
+
+
+/**
+ * Locks PDM.
+ *
+ * This might block.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+void pdmLock(PVMCC pVM)
+{
+    int rc = PDMCritSectEnter(pVM, &pVM->pdm.s.CritSect, VINF_SUCCESS);
+    PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pdm.s.CritSect, rc);
+}
+
+
+/**
+ * Locks PDM but don't go to ring-3 if it's owned by someone.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns rc if we're in GC or R0 and can't get the lock.
+ * @param   pVM     The cross context VM structure.
+ * @param   rcBusy  The RC to return in GC or R0 when we can't get the lock.
+ */
+int pdmLockEx(PVMCC pVM, int rcBusy)
+{
+    return PDMCritSectEnter(pVM, &pVM->pdm.s.CritSect, rcBusy);
+}
+
+
+/**
+ * Unlocks PDM.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+void pdmUnlock(PVMCC pVM)
+{
+    PDMCritSectLeave(pVM, &pVM->pdm.s.CritSect);
+}
+
+
+/**
+ * Checks if this thread is owning the PDM lock.
+ *
+ * @returns @c true if the lock is taken, @c false otherwise.
+ * @param   pVM     The cross context VM structure.
+ */
+bool pdmLockIsOwner(PVMCC pVM)
+{
+    return PDMCritSectIsOwner(pVM, &pVM->pdm.s.CritSect);
+}
+
+
+/**
+ * Converts ring 3 VMM heap pointer to a guest physical address
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pv              Ring-3 pointer.
+ * @param   pGCPhys         GC phys address (out).
+ */
+VMM_INT_DECL(int) PDMVmmDevHeapR3ToGCPhys(PVM pVM, RTR3PTR pv, RTGCPHYS *pGCPhys)
+{
+    if (RT_LIKELY(pVM->pdm.s.GCPhysVMMDevHeap != NIL_RTGCPHYS))
+    {
+        RTR3UINTPTR const offHeap = (RTR3UINTPTR)pv - (RTR3UINTPTR)pVM->pdm.s.pvVMMDevHeap;
+        if (RT_LIKELY(offHeap < pVM->pdm.s.cbVMMDevHeap))
+        {
+            *pGCPhys = pVM->pdm.s.GCPhysVMMDevHeap + offHeap;
+            return VINF_SUCCESS;
+        }
+
+        /* Don't assert here as this is called before we can catch ring-0 assertions. */
+        Log(("PDMVmmDevHeapR3ToGCPhys: pv=%p pvVMMDevHeap=%p cbVMMDevHeap=%#x\n",
+             pv, pVM->pdm.s.pvVMMDevHeap, pVM->pdm.s.cbVMMDevHeap));
+    }
+    else
+        Log(("PDMVmmDevHeapR3ToGCPhys: GCPhysVMMDevHeap=%RGp (pv=%p)\n", pVM->pdm.s.GCPhysVMMDevHeap, pv));
+    return VERR_PDM_DEV_HEAP_R3_TO_GCPHYS;
+}
+
+
+/**
+ * Checks if the vmm device heap is enabled (== vmm device's pci region mapped)
+ *
+ * @returns dev heap enabled status (true/false)
+ * @param   pVM             The cross context VM structure.
+ */
+VMM_INT_DECL(bool) PDMVmmDevHeapIsEnabled(PVM pVM)
+{
+    return pVM->pdm.s.GCPhysVMMDevHeap != NIL_RTGCPHYS;
+}
diff --git a/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp b/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp
new file mode 100644
index 00000000..2a190f90
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllCritSect.cpp
@@ -0,0 +1,1208 @@
+/* $Id: PDMAllCritSect.cpp $ */
+/** @file
+ * PDM - Write-Only Critical Section, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_CRITSECT
+#include "PDMInternal.h"
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/vmm/hm.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#ifdef IN_RING3
+# include <iprt/lockvalidator.h>
+#endif
+#if defined(IN_RING3) || defined(IN_RING0)
+# include <iprt/semaphore.h>
+#endif
+#ifdef IN_RING0
+# include <iprt/time.h>
+#endif
+#if defined(IN_RING3) || defined(IN_RING0)
+# include <iprt/thread.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** The number loops to spin for in ring-3. */
+#define PDMCRITSECT_SPIN_COUNT_R3       20
+/** The number loops to spin for in ring-0. */
+#define PDMCRITSECT_SPIN_COUNT_R0       256
+/** The number loops to spin for in the raw-mode context. */
+#define PDMCRITSECT_SPIN_COUNT_RC       256
+
+
+/** Skips some of the overly paranoid atomic updates.
+ * Makes some assumptions about cache coherence, though not brave enough not to
+ * always end with an atomic update. */
+#define PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+
+/* Undefine the automatic VBOX_STRICT API mappings. */
+#undef PDMCritSectEnter
+#undef PDMCritSectTryEnter
+
+
+/**
+ * Gets the ring-3 native thread handle of the calling thread.
+ *
+ * @returns native thread handle (ring-3).
+ * @param   pVM         The cross context VM structure.
+ * @param   pCritSect   The critical section. This is used in R0 and RC.
+ */
+DECL_FORCE_INLINE(RTNATIVETHREAD) pdmCritSectGetNativeSelf(PVMCC pVM, PCPDMCRITSECT pCritSect)
+{
+#ifdef IN_RING3
+    RT_NOREF(pVM, pCritSect);
+    RTNATIVETHREAD  hNativeSelf = RTThreadNativeSelf();
+
+#elif defined(IN_RING0)
+    AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, ("%RX32\n", pCritSect->s.Core.u32Magic),
+                    NIL_RTNATIVETHREAD);
+    RTNATIVETHREAD  hNativeSelf = GVMMR0GetRing3ThreadForSelf(pVM);
+    Assert(hNativeSelf != NIL_RTNATIVETHREAD);
+
+#else
+# error "Invalid context"
+#endif
+    return hNativeSelf;
+}
+
+
+#ifdef IN_RING0
+/**
+ * Marks the critical section as corrupted.
+ */
+DECL_NO_INLINE(static, int) pdmCritSectCorrupted(PPDMCRITSECT pCritSect, const char *pszMsg)
+{
+    ASMAtomicWriteU32(&pCritSect->s.Core.u32Magic, PDMCRITSECT_MAGIC_CORRUPTED);
+    LogRel(("PDMCritSect: %s pCritSect=%p\n", pszMsg, pCritSect));
+    return VERR_PDM_CRITSECT_IPE;
+}
+#endif
+
+
+/**
+ * Tail code called when we've won the battle for the lock.
+ *
+ * @returns VINF_SUCCESS.
+ *
+ * @param   pCritSect       The critical section.
+ * @param   hNativeSelf     The native handle of this thread.
+ * @param   pSrcPos         The source position of the lock operation.
+ */
+DECL_FORCE_INLINE(int) pdmCritSectEnterFirst(PPDMCRITSECT pCritSect, RTNATIVETHREAD hNativeSelf, PCRTLOCKVALSRCPOS pSrcPos)
+{
+    Assert(hNativeSelf != NIL_RTNATIVETHREAD);
+    AssertMsg(pCritSect->s.Core.NativeThreadOwner == NIL_RTNATIVETHREAD, ("NativeThreadOwner=%p\n", pCritSect->s.Core.NativeThreadOwner));
+    Assert(!(pCritSect->s.Core.fFlags & PDMCRITSECT_FLAGS_PENDING_UNLOCK));
+
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+    pCritSect->s.Core.cNestings = 1;
+# else
+    ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1);
+# endif
+    Assert(pCritSect->s.Core.cNestings == 1);
+    ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeSelf);
+
+# ifdef PDMCRITSECT_STRICT
+    RTLockValidatorRecExclSetOwner(pCritSect->s.Core.pValidatorRec, NIL_RTTHREAD, pSrcPos, true);
+# else
+    NOREF(pSrcPos);
+# endif
+    if (pSrcPos)
+        Log12Func(("%p: uId=%p ln=%u fn=%s\n", pCritSect, pSrcPos->uId, pSrcPos->uLine, pSrcPos->pszFunction));
+    else
+        Log12Func(("%p\n", pCritSect));
+
+    STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l);
+    return VINF_SUCCESS;
+}
+
+
+#if defined(IN_RING3) || defined(IN_RING0)
+/**
+ * Deals with the contended case in ring-3 and ring-0.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_DESTROYED if destroyed.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure if ring-0 and on
+ *                      an EMT, otherwise NULL.
+ * @param   pCritSect   The critsect.
+ * @param   hNativeSelf The native thread handle.
+ * @param   pSrcPos     The source position of the lock operation.
+ * @param   rcBusy      The status code to return when we're in RC or R0
+ */
+static int pdmR3R0CritSectEnterContended(PVMCC pVM, PVMCPU pVCpu, PPDMCRITSECT pCritSect, RTNATIVETHREAD hNativeSelf,
+                                         PCRTLOCKVALSRCPOS pSrcPos, int rcBusy)
+{
+# ifdef IN_RING0
+    /*
+     * If we've got queued critical section leave operations and rcBusy isn't
+     * VINF_SUCCESS, return to ring-3 immediately to avoid deadlocks.
+     */
+    if (   !pVCpu
+        || !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PDM_CRITSECT)
+        || rcBusy == VINF_SUCCESS )
+    { /* likely */ }
+    else
+    {
+        /** @todo statistics. */
+        STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock);
+        return rcBusy;
+    }
+# endif
+
+    /*
+     * Start waiting.
+     */
+    if (ASMAtomicIncS32(&pCritSect->s.Core.cLockers) == 0)
+        return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+# ifdef IN_RING3
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionR3);
+# else
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock);
+# endif
+
+    /*
+     * The wait loop.
+     *
+     * This handles VERR_TIMEOUT and VERR_INTERRUPTED.
+     */
+    STAM_REL_PROFILE_START(&pCritSect->s.CTX_MID_Z(StatContention,Wait), a);
+    PSUPDRVSESSION const    pSession    = pVM->pSession;
+    SUPSEMEVENT const       hEvent      = (SUPSEMEVENT)pCritSect->s.Core.EventSem;
+# ifdef IN_RING3
+#  ifdef PDMCRITSECT_STRICT
+    RTTHREAD const          hThreadSelf = RTThreadSelfAutoAdopt();
+    int rc2 = RTLockValidatorRecExclCheckOrder(pCritSect->s.Core.pValidatorRec, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
+    if (RT_FAILURE(rc2))
+        return rc2;
+#  else
+    RTTHREAD const          hThreadSelf = RTThreadSelf();
+#  endif
+# else /* IN_RING0 */
+    uint64_t const          tsStart     = RTTimeNanoTS();
+    uint64_t const          cNsMaxTotalDef = RT_NS_5MIN;
+    uint64_t                cNsMaxTotal = cNsMaxTotalDef;
+    uint64_t const          cNsMaxRetry = RT_NS_15SEC;
+    uint32_t                cMsMaxOne   = RT_MS_5SEC;
+    bool                    fNonInterruptible = false;
+# endif
+    for (;;)
+    {
+        /*
+         * Do the wait.
+         *
+         * In ring-3 this gets cluttered by lock validation and thread state
+         * maintainence.
+         *
+         * In ring-0 we have to deal with the possibility that the thread has
+         * been signalled and the interruptible wait function returning
+         * immediately.  In that case we do normal R0/RC rcBusy handling.
+         *
+         * We always do a timed wait here, so the event handle is revalidated
+         * regularly and we won't end up stuck waiting for a destroyed critsect.
+         */
+        /** @todo Make SUPSemEventClose wake up all waiters. */
+# ifdef IN_RING3
+#  ifdef PDMCRITSECT_STRICT
+        int rc9 = RTLockValidatorRecExclCheckBlocking(pCritSect->s.Core.pValidatorRec, hThreadSelf, pSrcPos,
+                                                      !(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NO_NESTING),
+                                                      RT_INDEFINITE_WAIT, RTTHREADSTATE_CRITSECT, true);
+        if (RT_FAILURE(rc9))
+            return rc9;
+#  else
+        RTThreadBlocking(hThreadSelf, RTTHREADSTATE_CRITSECT, true);
+#  endif
+        int const rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_MS_5SEC);
+        RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_CRITSECT);
+# else  /* IN_RING0 */
+        int const rc = !fNonInterruptible
+                     ? SUPSemEventWaitNoResume(pSession, hEvent, cMsMaxOne)
+                     : SUPSemEventWait(pSession, hEvent, cMsMaxOne);
+        Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hOwner=%p)\n",
+                   pCritSect, rc, RTTimeNanoTS() - tsStart, cMsMaxOne, pCritSect->s.Core.NativeThreadOwner));
+# endif /* IN_RING0 */
+
+        /*
+         * Make sure the critical section hasn't been delete before continuing.
+         */
+        if (RT_LIKELY(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC))
+        { /* likely */ }
+        else
+        {
+            LogRel(("PDMCritSectEnter: Destroyed while waiting; pCritSect=%p rc=%Rrc\n", pCritSect, rc));
+            return VERR_SEM_DESTROYED;
+        }
+
+        /*
+         * Most likely we're here because we got signalled.
+         */
+        if (rc == VINF_SUCCESS)
+        {
+            STAM_REL_PROFILE_STOP(&pCritSect->s.CTX_MID_Z(StatContention,Wait), a);
+            return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+        }
+
+        /*
+         * Timeout and interrupted waits needs careful handling in ring-0
+         * because we're cooperating with ring-3 on this critical section
+         * and thus need to make absolutely sure we won't get stuck here.
+         *
+         * The r0 interrupted case means something is pending (termination,
+         * signal, APC, debugger, whatever), so we must try our best to
+         * return to the caller and to ring-3 so it can be dealt with.
+         */
+        if (RT_LIKELY(rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED))
+        {
+# ifdef IN_RING0
+            uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart;
+            int const      rcTerm     = RTThreadQueryTerminationStatus(NIL_RTTHREAD);
+            AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING,
+                      ("rcTerm=%Rrc\n", rcTerm));
+            if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef)
+                cNsMaxTotal = RT_NS_1MIN;
+
+            if (rc == VERR_TIMEOUT)
+            {
+                /* Try return get out of here with a non-VINF_SUCCESS status if
+                   the thread is terminating or if the timeout has been exceeded. */
+                STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectVerrTimeout);
+                if (   rcTerm != VINF_THREAD_IS_TERMINATING
+                    && cNsElapsed <= cNsMaxTotal)
+                    continue;
+            }
+            else
+            {
+                /* For interrupt cases, we must return if we can.  If rcBusy is VINF_SUCCESS,
+                   we will try non-interruptible sleep for a while to help resolve the issue
+                   w/o guru'ing. */
+                STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectVerrInterrupted);
+                if (   rcTerm != VINF_THREAD_IS_TERMINATING
+                    && rcBusy == VINF_SUCCESS
+                    && pVCpu != NULL
+                    && cNsElapsed <= cNsMaxTotal)
+                {
+                    if (!fNonInterruptible)
+                    {
+                        STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectNonInterruptibleWaits);
+                        fNonInterruptible   = true;
+                        cMsMaxOne           = 32;
+                        uint64_t cNsLeft = cNsMaxTotal - cNsElapsed;
+                        if (cNsLeft > RT_NS_10SEC)
+                            cNsMaxTotal = cNsElapsed + RT_NS_10SEC;
+                    }
+                    continue;
+                }
+            }
+
+            /*
+             * Let try get out of here.  We must very carefully undo the
+             * cLockers increment we did using compare-and-exchange so that
+             * we don't race the semaphore signalling in PDMCritSectLeave
+             * and end up with spurious wakeups and two owners at once.
+             */
+            uint32_t cNoIntWaits = 0;
+            uint32_t cCmpXchgs   = 0;
+            int32_t  cLockers    = ASMAtomicReadS32(&pCritSect->s.Core.cLockers);
+            for (;;)
+            {
+                if (pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC)
+                {
+                    if (cLockers > 0 && cCmpXchgs < _64M)
+                    {
+                        bool fRc = ASMAtomicCmpXchgExS32(&pCritSect->s.Core.cLockers, cLockers - 1, cLockers, &cLockers);
+                        if (fRc)
+                        {
+                            LogFunc(("Aborting wait on %p (rc=%Rrc rcTerm=%Rrc cNsElapsed=%'RU64) -> %Rrc\n", pCritSect,
+                                     rc, rcTerm, cNsElapsed, rcBusy != VINF_SUCCESS ? rcBusy : rc));
+                            STAM_REL_COUNTER_INC(&pVM->pdm.s.StatAbortedCritSectEnters);
+                            return rcBusy != VINF_SUCCESS ? rcBusy : rc;
+                        }
+                        cCmpXchgs++;
+                        if ((cCmpXchgs & 0xffff) == 0)
+                            Log11Func(("%p: cLockers=%d cCmpXchgs=%u (hOwner=%p)\n",
+                                       pCritSect, cLockers, cCmpXchgs, pCritSect->s.Core.NativeThreadOwner));
+                        ASMNopPause();
+                        continue;
+                    }
+
+                    if (cLockers == 0)
+                    {
+                        /*
+                         * We are racing someone in PDMCritSectLeave.
+                         *
+                         * For the VERR_TIMEOUT case we'll just retry taking it the normal
+                         * way for a while.  For VERR_INTERRUPTED we're in for more fun as
+                         * the previous owner might not have signalled the semaphore yet,
+                         * so we'll do a short non-interruptible wait instead and then guru.
+                         */
+                        if (   rc == VERR_TIMEOUT
+                            && RTTimeNanoTS() - tsStart <= cNsMaxTotal + cNsMaxRetry)
+                            break;
+
+                        if (   rc == VERR_INTERRUPTED
+                            && (   cNoIntWaits == 0
+                                || RTTimeNanoTS() - (tsStart + cNsElapsed) < RT_NS_100MS))
+                        {
+                            int const rc2 = SUPSemEventWait(pSession, hEvent, 1 /*ms*/);
+                            if (rc2 == VINF_SUCCESS)
+                            {
+                                STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectEntersWhileAborting);
+                                STAM_REL_PROFILE_STOP(&pCritSect->s.CTX_MID_Z(StatContention,Wait), a);
+                                return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+                            }
+                            cNoIntWaits++;
+                            cLockers = ASMAtomicReadS32(&pCritSect->s.Core.cLockers);
+                            continue;
+                        }
+                    }
+                    else
+                        LogFunc(("Critical section %p has a broken cLockers count. Aborting.\n", pCritSect));
+
+                    /* Sabotage the critical section and return error to caller. */
+                    ASMAtomicWriteU32(&pCritSect->s.Core.u32Magic, PDMCRITSECT_MAGIC_FAILED_ABORT);
+                    LogRel(("PDMCritSectEnter: Failed to abort wait on pCritSect=%p (rc=%Rrc rcTerm=%Rrc)\n",
+                            pCritSect, rc, rcTerm));
+                    return VERR_PDM_CRITSECT_ABORT_FAILED;
+                }
+                LogRel(("PDMCritSectEnter: Destroyed while aborting wait; pCritSect=%p/%#x rc=%Rrc rcTerm=%Rrc\n",
+                        pCritSect, pCritSect->s.Core.u32Magic, rc, rcTerm));
+                return VERR_SEM_DESTROYED;
+            }
+
+            /* We get here if we timed out.  Just retry now that it
+               appears someone left already. */
+            Assert(rc == VERR_TIMEOUT);
+            cMsMaxOne = 10 /*ms*/;
+
+# else  /* IN_RING3 */
+            RT_NOREF(pVM, pVCpu, rcBusy);
+# endif /* IN_RING3 */
+        }
+        /*
+         * Any other return code is fatal.
+         */
+        else
+        {
+            AssertMsgFailed(("rc=%Rrc\n", rc));
+            return RT_FAILURE_NP(rc) ? rc : -rc;
+        }
+    }
+    /* won't get here */
+}
+#endif /* IN_RING3 || IN_RING0 */
+
+
+/**
+ * Common worker for the debug and normal APIs.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @returns rcBusy when encountering a busy critical section in RC/R0.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pCritSect           The PDM critical section to enter.
+ * @param   rcBusy              The status code to return when we're in RC or R0
+ * @param   pSrcPos             The source position of the lock operation.
+ */
+DECL_FORCE_INLINE(int) pdmCritSectEnter(PVMCC pVM, PPDMCRITSECT pCritSect, int rcBusy, PCRTLOCKVALSRCPOS pSrcPos)
+{
+    Assert(pCritSect->s.Core.cNestings < 8);  /* useful to catch incorrect locking */
+    Assert(pCritSect->s.Core.cNestings >= 0);
+#if defined(VBOX_STRICT) && defined(IN_RING0)
+    /* Hope we're not messing with critical sections while in the no-block
+       zone, that would complicate things a lot. */
+    PVMCPUCC pVCpuAssert = VMMGetCpu(pVM);
+    Assert(pVCpuAssert && VMMRZCallRing3IsEnabled(pVCpuAssert));
+#endif
+
+    /*
+     * If the critical section has already been destroyed, then inform the caller.
+     */
+    AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC,
+                    ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic),
+                    VERR_SEM_DESTROYED);
+
+    /*
+     * See if we're lucky.
+     */
+    /* NOP ... */
+    if (!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP))
+    { /* We're more likely to end up here with real critsects than a NOP one. */ }
+    else
+        return VINF_SUCCESS;
+
+    RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pVM, pCritSect);
+    AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
+    /* ... not owned ... */
+    if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
+        return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+
+    /* ... or nested. */
+    if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf)
+    {
+        Assert(pCritSect->s.Core.cNestings >= 1);
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        pCritSect->s.Core.cNestings += 1;
+# else
+        ASMAtomicIncS32(&pCritSect->s.Core.cNestings);
+# endif
+        ASMAtomicIncS32(&pCritSect->s.Core.cLockers);
+        Log12Func(("%p: cNestings=%d cLockers=%d\n", pCritSect, pCritSect->s.Core.cNestings, pCritSect->s.Core.cLockers));
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Spin for a bit without incrementing the counter.
+     */
+    /** @todo Move this to cfgm variables since it doesn't make sense to spin on UNI
+     *        cpu systems. */
+    int32_t cSpinsLeft = CTX_SUFF(PDMCRITSECT_SPIN_COUNT_);
+    while (cSpinsLeft-- > 0)
+    {
+        if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
+            return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+        ASMNopPause();
+        /** @todo Should use monitor/mwait on e.g. &cLockers here, possibly with a
+           cli'ed pendingpreemption check up front using sti w/ instruction fusing
+           for avoiding races. Hmm ... This is assuming the other party is actually
+           executing code on another CPU ... which we could keep track of if we
+           wanted. */
+    }
+
+#ifdef IN_RING3
+    /*
+     * Take the slow path.
+     */
+    NOREF(rcBusy);
+    return pdmR3R0CritSectEnterContended(pVM, NULL, pCritSect, hNativeSelf, pSrcPos, rcBusy);
+
+#elif defined(IN_RING0)
+# if 1 /* new code */
+    /*
+     * In ring-0 context we have to take the special VT-x/AMD-V HM context into
+     * account when waiting on contended locks.
+     *
+     * While we usually (it can be VINF_SUCCESS) have the option of returning
+     * rcBusy and force the caller to go back to ring-3 and to re-start the work
+     * there, it's almost always more efficient to try wait for the lock here.
+     * The rcBusy will be used if we encounter an VERR_INTERRUPTED situation
+     * though.
+     */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    if (pVCpu)
+    {
+        VMMR0EMTBLOCKCTX Ctx;
+        int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pCritSect, &Ctx);
+        if (rc == VINF_SUCCESS)
+        {
+            Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+            rc = pdmR3R0CritSectEnterContended(pVM, pVCpu, pCritSect, hNativeSelf, pSrcPos, rcBusy);
+
+            VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
+        }
+        else
+            STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLockBusy);
+        return rc;
+    }
+
+    /* Non-EMT. */
+    Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    return pdmR3R0CritSectEnterContended(pVM, NULL, pCritSect, hNativeSelf, pSrcPos, rcBusy);
+
+# else /* old code: */
+    /*
+     * We preemption hasn't been disabled, we can block here in ring-0.
+     */
+    if (   RTThreadPreemptIsEnabled(NIL_RTTHREAD)
+        && ASMIntAreEnabled())
+        return pdmR3R0CritSectEnterContended(pVM, VMMGetCpu(pVM), pCritSect, hNativeSelf, pSrcPos, rcBusy);
+
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLock);
+
+    /*
+     * Call ring-3 to acquire the critical section?
+     */
+    if (rcBusy == VINF_SUCCESS)
+    {
+        PVMCPUCC pVCpu = VMMGetCpu(pVM);
+        AssertReturn(pVCpu, VERR_PDM_CRITSECT_IPE);
+        return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_ENTER, MMHyperCCToR3(pVM, pCritSect));
+    }
+
+    /*
+     * Return busy.
+     */
+    LogFlow(("PDMCritSectEnter: locked => R3 (%Rrc)\n", rcBusy));
+    return rcBusy;
+# endif  /* old code */
+#else
+# error "Unsupported context"
+#endif
+}
+
+
+/**
+ * Enters a PDM critical section.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @returns rcBusy when encountering a busy critical section in RC/R0.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pCritSect           The PDM critical section to enter.
+ * @param   rcBusy              The status code to return when we're in RC or R0
+ *                              and the section is busy.  Pass VINF_SUCCESS to
+ *                              acquired the critical section thru a ring-3
+ *                              call if necessary.
+ *
+ * @note    Even callers setting @a rcBusy to VINF_SUCCESS must either handle
+ *          possible failures in ring-0 or apply
+ *          PDM_CRITSECT_RELEASE_ASSERT_RC(),
+ *          PDM_CRITSECT_RELEASE_ASSERT_RC_DEV(),
+ *          PDM_CRITSECT_RELEASE_ASSERT_RC_DRV() or
+ *          PDM_CRITSECT_RELEASE_ASSERT_RC_USB() to the return value of this
+ *          function.
+ */
+VMMDECL(DECL_CHECK_RETURN_NOT_R3(int)) PDMCritSectEnter(PVMCC pVM, PPDMCRITSECT pCritSect, int rcBusy)
+{
+#ifndef PDMCRITSECT_STRICT
+    return pdmCritSectEnter(pVM, pCritSect, rcBusy, NULL);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectEnter(pVM, pCritSect, rcBusy, &SrcPos);
+#endif
+}
+
+
+/**
+ * Enters a PDM critical section, with location information for debugging.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @returns rcBusy when encountering a busy critical section in RC/R0.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pCritSect           The PDM critical section to enter.
+ * @param   rcBusy              The status code to return when we're in RC or R0
+ *                              and the section is busy.   Pass VINF_SUCCESS to
+ *                              acquired the critical section thru a ring-3
+ *                              call if necessary.
+ * @param   uId                 Some kind of locking location ID.  Typically a
+ *                              return address up the stack.  Optional (0).
+ * @param   SRC_POS             The source position where to lock is being
+ *                              acquired from.  Optional.
+ */
+VMMDECL(DECL_CHECK_RETURN_NOT_R3(int))
+PDMCritSectEnterDebug(PVMCC pVM, PPDMCRITSECT pCritSect, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+#ifdef PDMCRITSECT_STRICT
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectEnter(pVM, pCritSect, rcBusy, &SrcPos);
+#else
+    NOREF(uId); RT_SRC_POS_NOREF();
+    return pdmCritSectEnter(pVM, pCritSect, rcBusy, NULL);
+#endif
+}
+
+
+/**
+ * Common worker for the debug and normal APIs.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pCritSect   The critical section.
+ * @param   pSrcPos     The source position of the lock operation.
+ */
+static int pdmCritSectTryEnter(PVMCC pVM, PPDMCRITSECT pCritSect, PCRTLOCKVALSRCPOS pSrcPos)
+{
+    /*
+     * If the critical section has already been destroyed, then inform the caller.
+     */
+    AssertMsgReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC,
+                    ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic),
+                    VERR_SEM_DESTROYED);
+
+    /*
+     * See if we're lucky.
+     */
+    /* NOP ... */
+    if (!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP))
+    { /* We're more likely to end up here with real critsects than a NOP one. */ }
+    else
+        return VINF_SUCCESS;
+
+    RTNATIVETHREAD hNativeSelf = pdmCritSectGetNativeSelf(pVM, pCritSect);
+    AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
+    /* ... not owned ... */
+    if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, 0, -1))
+        return pdmCritSectEnterFirst(pCritSect, hNativeSelf, pSrcPos);
+
+    /* ... or nested. */
+    if (pCritSect->s.Core.NativeThreadOwner == hNativeSelf)
+    {
+        Assert(pCritSect->s.Core.cNestings >= 1);
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        pCritSect->s.Core.cNestings += 1;
+# else
+        ASMAtomicIncS32(&pCritSect->s.Core.cNestings);
+# endif
+        ASMAtomicIncS32(&pCritSect->s.Core.cLockers);
+        Log12Func(("%p: cNestings=%d cLockers=%d\n", pCritSect, pCritSect->s.Core.cNestings, pCritSect->s.Core.cLockers));
+        return VINF_SUCCESS;
+    }
+
+    /* no spinning */
+
+    /*
+     * Return busy.
+     */
+#ifdef IN_RING3
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionR3);
+#else
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZLockBusy);
+#endif
+    LogFlow(("PDMCritSectTryEnter: locked\n"));
+    return VERR_SEM_BUSY;
+}
+
+
+/**
+ * Try enter a critical section.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(DECL_CHECK_RETURN(int)) PDMCritSectTryEnter(PVMCC pVM, PPDMCRITSECT pCritSect)
+{
+#ifndef PDMCRITSECT_STRICT
+    return pdmCritSectTryEnter(pVM, pCritSect, NULL);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectTryEnter(pVM, pCritSect, &SrcPos);
+#endif
+}
+
+
+/**
+ * Try enter a critical section, with location information for debugging.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pCritSect           The critical section.
+ * @param   uId                 Some kind of locking location ID.  Typically a
+ *                              return address up the stack.  Optional (0).
+ * @param   SRC_POS             The source position where to lock is being
+ *                              acquired from.  Optional.
+ */
+VMMDECL(DECL_CHECK_RETURN(int))
+PDMCritSectTryEnterDebug(PVMCC pVM, PPDMCRITSECT pCritSect, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+#ifdef PDMCRITSECT_STRICT
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectTryEnter(pVM, pCritSect, &SrcPos);
+#else
+    NOREF(uId); RT_SRC_POS_NOREF();
+    return pdmCritSectTryEnter(pVM, pCritSect, NULL);
+#endif
+}
+
+
+#ifdef IN_RING3
+/**
+ * Enters a PDM critical section.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @returns rcBusy when encountering a busy critical section in GC/R0.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pCritSect           The PDM critical section to enter.
+ * @param   fCallRing3          Whether this is a VMMRZCallRing3()request.
+ */
+VMMR3DECL(int) PDMR3CritSectEnterEx(PVM pVM, PPDMCRITSECT pCritSect, bool fCallRing3)
+{
+    int rc = PDMCritSectEnter(pVM, pCritSect, VERR_IGNORED);
+    if (    rc == VINF_SUCCESS
+        &&  fCallRing3
+        &&  pCritSect->s.Core.pValidatorRec
+        &&  pCritSect->s.Core.pValidatorRec->hThread != NIL_RTTHREAD)
+        RTLockValidatorRecExclReleaseOwnerUnchecked(pCritSect->s.Core.pValidatorRec);
+    return rc;
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Leaves a critical section entered with PDMCritSectEnter().
+ *
+ * @returns Indication whether we really exited the critical section.
+ * @retval  VINF_SUCCESS if we really exited.
+ * @retval  VINF_SEM_NESTED if we only reduced the nesting count.
+ * @retval  VERR_NOT_OWNER if you somehow ignore release assertions.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pCritSect   The PDM critical section to leave.
+ *
+ * @remarks Can be called from no-ring-3-call context in ring-0 (TM/VirtualSync)
+ *          where we'll queue leaving operation for ring-3 processing.
+ */
+VMMDECL(int) PDMCritSectLeave(PVMCC pVM, PPDMCRITSECT pCritSect)
+{
+    AssertMsg(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, ("%p %RX32\n", pCritSect, pCritSect->s.Core.u32Magic));
+    Assert(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC);
+
+    /*
+     * Check for NOP sections before asserting ownership.
+     */
+    if (!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP))
+    { /* We're more likely to end up here with real critsects than a NOP one. */ }
+    else
+        return VINF_SUCCESS;
+
+    /*
+     * Always check that the caller is the owner (screw performance).
+     */
+    RTNATIVETHREAD const hNativeSelf = pdmCritSectGetNativeSelf(pVM, pCritSect);
+    VMM_ASSERT_RELEASE_MSG_RETURN(pVM, pCritSect->s.Core.NativeThreadOwner == hNativeSelf && hNativeSelf != NIL_RTNATIVETHREAD,
+                                  ("%p %s: %p != %p; cLockers=%d cNestings=%d\n", pCritSect, R3STRING(pCritSect->s.pszName),
+                                   pCritSect->s.Core.NativeThreadOwner, hNativeSelf,
+                                   pCritSect->s.Core.cLockers, pCritSect->s.Core.cNestings),
+                                  VERR_NOT_OWNER);
+
+    /*
+     * Nested leave.
+     */
+    int32_t const cNestings = pCritSect->s.Core.cNestings;
+    Assert(cNestings >= 1);
+    if (cNestings > 1)
+    {
+#ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        pCritSect->s.Core.cNestings = cNestings - 1;
+#else
+        ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, cNestings - 1);
+#endif
+        int32_t const cLockers = ASMAtomicDecS32(&pCritSect->s.Core.cLockers);
+        Assert(cLockers >= 0); RT_NOREF(cLockers);
+        Log12Func(("%p: cNestings=%d cLockers=%d\n", pCritSect, cNestings - 1, cLockers));
+        return VINF_SEM_NESTED;
+    }
+
+    Log12Func(("%p: cNestings=%d cLockers=%d hOwner=%p - leave for real\n",
+               pCritSect, cNestings, pCritSect->s.Core.cLockers, pCritSect->s.Core.NativeThreadOwner));
+
+#ifdef IN_RING3
+    /*
+     * Ring-3: Leave for real.
+     */
+    SUPSEMEVENT const hEventToSignal = pCritSect->s.hEventToSignal;
+    pCritSect->s.hEventToSignal = NIL_SUPSEMEVENT;
+
+# if defined(PDMCRITSECT_STRICT)
+    if (pCritSect->s.Core.pValidatorRec->hThread != NIL_RTTHREAD)
+        RTLockValidatorRecExclReleaseOwnerUnchecked(pCritSect->s.Core.pValidatorRec);
+# endif
+    Assert(!pCritSect->s.Core.pValidatorRec || pCritSect->s.Core.pValidatorRec->hThread == NIL_RTTHREAD);
+
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+    //pCritSect->s.Core.cNestings = 0; /* not really needed */
+    pCritSect->s.Core.NativeThreadOwner = NIL_RTNATIVETHREAD;
+# else
+    ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 0);
+    ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD);
+# endif
+    ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK);
+
+    /* Stop profiling and decrement lockers. */
+    STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l);
+    ASMCompilerBarrier();
+    int32_t const cLockers = ASMAtomicDecS32(&pCritSect->s.Core.cLockers);
+    if (cLockers < 0)
+        AssertMsg(cLockers == -1, ("cLockers=%d\n", cLockers));
+    else
+    {
+        /* Someone is waiting, wake up one of them. */
+        Assert(cLockers < _8K);
+        Log8(("PDMCritSectLeave: Waking up %p (cLockers=%u)\n", pCritSect, cLockers));
+        SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->s.Core.EventSem;
+        int rc = SUPSemEventSignal(pVM->pSession, hEvent);
+        AssertRC(rc);
+    }
+
+    /* Signal exit event. */
+    if (RT_LIKELY(hEventToSignal == NIL_SUPSEMEVENT))
+    { /* likely */ }
+    else
+    {
+        Log8(("PDMCritSectLeave: Signalling %#p (%p)\n", hEventToSignal, pCritSect));
+        int rc = SUPSemEventSignal(pVM->pSession, hEventToSignal);
+        AssertRC(rc);
+    }
+
+    return VINF_SUCCESS;
+
+
+#elif defined(IN_RING0)
+    /*
+     * Ring-0: Try leave for real, depends on host and context.
+     */
+    SUPSEMEVENT const hEventToSignal = pCritSect->s.hEventToSignal;
+    pCritSect->s.hEventToSignal = NIL_SUPSEMEVENT;
+    PVMCPUCC pVCpu           = VMMGetCpu(pVM);
+    bool     fQueueOnTrouble = false; /* Set this to true to test queueing. */
+    if (   pVCpu == NULL /* non-EMT access, if we implement it must be able to block */
+        || VMMRZCallRing3IsEnabled(pVCpu)
+        || RTSemEventIsSignalSafe()
+        || (   VMMR0ThreadCtxHookIsEnabled(pVCpu)       /* Doesn't matter if Signal() blocks if we have hooks, ... */
+            && RTThreadPreemptIsEnabled(NIL_RTTHREAD)   /* ... and preemption is still enabled, */
+            && ASMIntAreEnabled())                      /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */
+        || (fQueueOnTrouble = (   hEventToSignal == NIL_SUPSEMEVENT
+                               && ASMAtomicUoReadS32(&pCritSect->s.Core.cLockers) == 0)) )
+    {
+        pCritSect->s.hEventToSignal = NIL_SUPSEMEVENT;
+
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        //pCritSect->s.Core.cNestings = 0; /* not really needed */
+        pCritSect->s.Core.NativeThreadOwner = NIL_RTNATIVETHREAD;
+# else
+        ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 0);
+        ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD);
+# endif
+        ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK);
+
+        /*
+         * Stop profiling and decrement lockers.
+         */
+        STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l);
+        ASMCompilerBarrier();
+
+        bool    fQueueIt = false;
+        int32_t cLockers;
+        if (!fQueueOnTrouble)
+            cLockers = ASMAtomicDecS32(&pCritSect->s.Core.cLockers);
+        else
+        {
+            cLockers = -1;
+            if (!ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, -1, 0))
+                fQueueIt = true;
+        }
+        if (!fQueueIt)
+        {
+            VMMR0EMTBLOCKCTX    Ctx;
+            bool                fLeaveCtx = false;
+            if (cLockers < 0)
+                AssertMsg(cLockers == -1, ("cLockers=%d\n", cLockers));
+            else
+            {
+                /* Someone is waiting, wake up one of them. */
+                Assert(cLockers < _8K);
+                SUPSEMEVENT hEvent = (SUPSEMEVENT)pCritSect->s.Core.EventSem;
+                if (!RTSemEventIsSignalSafe() && (pVCpu = VMMGetCpu(pVM)) != NULL)
+                {
+                    int rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pCritSect, &Ctx);
+                    VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
+                    fLeaveCtx = true;
+                }
+                int rc = SUPSemEventSignal(pVM->pSession, hEvent);
+                AssertRC(rc);
+            }
+
+            /*
+             * Signal exit event.
+             */
+            if (RT_LIKELY(hEventToSignal == NIL_SUPSEMEVENT))
+            { /* likely */ }
+            else
+            {
+                if (!fLeaveCtx && pVCpu != NULL && !RTSemEventIsSignalSafe() && (pVCpu = VMMGetCpu(pVM)) != NULL)
+                {
+                    int rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pCritSect, &Ctx);
+                    VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
+                    fLeaveCtx = true;
+                }
+                Log8(("Signalling %#p\n", hEventToSignal));
+                int rc = SUPSemEventSignal(pVM->pSession, hEventToSignal);
+                AssertRC(rc);
+            }
+
+            /*
+             * Restore HM context if needed.
+             */
+            if (!fLeaveCtx)
+            { /* contention should be unlikely */ }
+            else
+                VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
+
+# ifdef DEBUG_bird
+            VMMTrashVolatileXMMRegs();
+# endif
+            return VINF_SUCCESS;
+        }
+
+        /*
+         * Darn, someone raced in on us.  Restore the state (this works only
+         * because the semaphore is effectively controlling ownership).
+         */
+        bool fRc;
+        RTNATIVETHREAD hMessOwner = NIL_RTNATIVETHREAD;
+        ASMAtomicCmpXchgExHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeSelf, NIL_RTNATIVETHREAD, fRc, &hMessOwner);
+        AssertLogRelMsgReturn(fRc, ("pCritSect=%p hMessOwner=%p\n", pCritSect, hMessOwner),
+                              pdmCritSectCorrupted(pCritSect, "owner race"));
+        STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l);
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        //pCritSect->s.Core.cNestings = 1;
+        Assert(pCritSect->s.Core.cNestings == 1);
+# else
+        //Assert(pCritSect->s.Core.cNestings == 0);
+        ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1);
+# endif
+        Assert(hEventToSignal == NIL_SUPSEMEVENT);
+    }
+
+
+#else /* IN_RC */
+    /*
+     * Raw-mode: Try leave it.
+     */
+# error "This context is not use..."
+    if (pCritSect->s.Core.cLockers == 0)
+    {
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        //pCritSect->s.Core.cNestings = 0; /* not really needed */
+# else
+        ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 0);
+# endif
+        ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK);
+        STAM_PROFILE_ADV_STOP(&pCritSect->s.StatLocked, l);
+
+        ASMAtomicWriteHandle(&pCritSect->s.Core.NativeThreadOwner, NIL_RTNATIVETHREAD);
+        if (ASMAtomicCmpXchgS32(&pCritSect->s.Core.cLockers, -1, 0))
+            return VINF_SUCCESS;
+
+        /*
+         * Darn, someone raced in on us.  Restore the state (this works only
+         * because the semaphore is effectively controlling ownership).
+         */
+        bool fRc;
+        RTNATIVETHREAD hMessOwner = NIL_RTNATIVETHREAD;
+        ASMAtomicCmpXchgExHandle(&pCritSect->s.Core.NativeThreadOwner, hNativeSelf, NIL_RTNATIVETHREAD, fRc, &hMessOwner);
+        AssertLogRelMsgReturn(fRc, ("pCritSect=%p hMessOwner=%p\n", pCritSect, hMessOwner),
+                              pdmCritSectCorrupted(pCritSect, "owner race"));
+        STAM_PROFILE_ADV_START(&pCritSect->s.StatLocked, l);
+# ifdef PDMCRITSECT_WITH_LESS_ATOMIC_STUFF
+        //pCritSect->s.Core.cNestings = 1;
+        Assert(pCritSect->s.Core.cNestings == 1);
+# else
+        //Assert(pCritSect->s.Core.cNestings == 0);
+        ASMAtomicWriteS32(&pCritSect->s.Core.cNestings, 1);
+# endif
+    }
+#endif /* IN_RC */
+
+
+#ifndef IN_RING3
+    /*
+     * Ring-0/raw-mode: Unable to leave. Queue the leave for ring-3.
+     */
+    ASMAtomicOrU32(&pCritSect->s.Core.fFlags, PDMCRITSECT_FLAGS_PENDING_UNLOCK);
+# ifndef IN_RING0
+    PVMCPUCC    pVCpu = VMMGetCpu(pVM);
+# endif
+    uint32_t    i     = pVCpu->pdm.s.cQueuedCritSectLeaves++;
+    LogFlow(("PDMCritSectLeave: [%d]=%p => R3\n", i, pCritSect));
+    VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectLeaves), ("%d\n", i), VERR_PDM_CRITSECT_IPE);
+    pVCpu->pdm.s.apQueuedCritSectLeaves[i] = pCritSect->s.pSelfR3;
+    VMM_ASSERT_RELEASE_MSG_RETURN(pVM,
+                                     RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectLeaves[i])
+                                  &&    ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectLeaves[i] & HOST_PAGE_OFFSET_MASK)
+                                     == ((uintptr_t)pCritSect & HOST_PAGE_OFFSET_MASK),
+                                  ("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectLeaves[i], pCritSect),
+                                  pdmCritSectCorrupted(pCritSect, "Invalid pSelfR3 value"));
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT); /** @todo handle VMCPU_FF_PDM_CRITSECT in ring-0 outside the no-call-ring-3 part. */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3); /* unnecessary paranoia */
+    STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
+    STAM_REL_COUNTER_INC(&pCritSect->s.StatContentionRZUnlock);
+
+    return VINF_SUCCESS;
+#endif /* IN_RING3 */
+}
+
+
+#if defined(IN_RING0) || defined(IN_RING3)
+/**
+ * Schedule a event semaphore for signalling upon critsect exit.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_TOO_MANY_SEMAPHORES if an event was already scheduled.
+ * @returns VERR_NOT_OWNER if we're not the critsect owner (ring-3 only).
+ * @returns VERR_SEM_DESTROYED if RTCritSectDelete was called while waiting.
+ *
+ * @param   pCritSect       The critical section.
+ * @param   hEventToSignal  The support driver event semaphore that should be
+ *                          signalled.
+ */
+VMMDECL(int) PDMHCCritSectScheduleExitEvent(PPDMCRITSECT pCritSect, SUPSEMEVENT hEventToSignal)
+{
+    AssertPtr(pCritSect);
+    Assert(!(pCritSect->s.Core.fFlags & RTCRITSECT_FLAGS_NOP));
+    Assert(hEventToSignal != NIL_SUPSEMEVENT);
+# ifdef IN_RING3
+    if (RT_UNLIKELY(!RTCritSectIsOwner(&pCritSect->s.Core)))
+        return VERR_NOT_OWNER;
+# endif
+    if (RT_LIKELY(   pCritSect->s.hEventToSignal == NIL_RTSEMEVENT
+                  || pCritSect->s.hEventToSignal == hEventToSignal))
+    {
+        pCritSect->s.hEventToSignal = hEventToSignal;
+        return VINF_SUCCESS;
+    }
+    return VERR_TOO_MANY_SEMAPHORES;
+}
+#endif /* IN_RING0 || IN_RING3 */
+
+
+/**
+ * Checks the caller is the owner of the critical section.
+ *
+ * @returns true if owner.
+ * @returns false if not owner.
+ * @param   pVM         The cross context VM structure.
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(bool) PDMCritSectIsOwner(PVMCC pVM, PCPDMCRITSECT pCritSect)
+{
+#ifdef IN_RING3
+    RT_NOREF(pVM);
+    return RTCritSectIsOwner(&pCritSect->s.Core);
+#else
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    if (   !pVCpu
+        || pCritSect->s.Core.NativeThreadOwner != pVCpu->hNativeThread)
+        return false;
+    return (pCritSect->s.Core.fFlags & PDMCRITSECT_FLAGS_PENDING_UNLOCK) == 0
+        || pCritSect->s.Core.cNestings > 1;
+#endif
+}
+
+
+/**
+ * Checks the specified VCPU is the owner of the critical section.
+ *
+ * @returns true if owner.
+ * @returns false if not owner.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(bool) PDMCritSectIsOwnerEx(PVMCPUCC pVCpu, PCPDMCRITSECT pCritSect)
+{
+#ifdef IN_RING3
+    NOREF(pVCpu);
+    return RTCritSectIsOwner(&pCritSect->s.Core);
+#else
+    Assert(VMCC_GET_CPU(pVCpu->CTX_SUFF(pVM), pVCpu->idCpu) == pVCpu);
+    if (pCritSect->s.Core.NativeThreadOwner != pVCpu->hNativeThread)
+        return false;
+    return (pCritSect->s.Core.fFlags & PDMCRITSECT_FLAGS_PENDING_UNLOCK) == 0
+        || pCritSect->s.Core.cNestings > 1;
+#endif
+}
+
+
+/**
+ * Checks if anyone is waiting on the critical section we own.
+ *
+ * @returns true if someone is waiting.
+ * @returns false if no one is waiting.
+ * @param   pVM         The cross context VM structure.
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(bool) PDMCritSectHasWaiters(PVMCC pVM, PCPDMCRITSECT pCritSect)
+{
+    AssertReturn(pCritSect->s.Core.u32Magic == RTCRITSECT_MAGIC, false);
+    Assert(pCritSect->s.Core.NativeThreadOwner == pdmCritSectGetNativeSelf(pVM, pCritSect)); RT_NOREF(pVM);
+    return pCritSect->s.Core.cLockers >= pCritSect->s.Core.cNestings;
+}
+
+
+/**
+ * Checks if a critical section is initialized or not.
+ *
+ * @returns true if initialized.
+ * @returns false if not initialized.
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(bool) PDMCritSectIsInitialized(PCPDMCRITSECT pCritSect)
+{
+    return RTCritSectIsInitialized(&pCritSect->s.Core);
+}
+
+
+/**
+ * Gets the recursion depth.
+ *
+ * @returns The recursion depth.
+ * @param   pCritSect   The critical section.
+ */
+VMMDECL(uint32_t) PDMCritSectGetRecursion(PCPDMCRITSECT pCritSect)
+{
+    return RTCritSectGetRecursion(&pCritSect->s.Core);
+}
+
diff --git a/src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp b/src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp
new file mode 100644
index 00000000..3c9fb021
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllCritSectBoth.cpp
@@ -0,0 +1,118 @@
+/* $Id: PDMAllCritSectBoth.cpp $ */
+/** @file
+ * PDM - Code Common to Both Critical Section Types, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_CRITSECT
+#include "PDMInternal.h"
+#include <VBox/vmm/pdmcritsect.h>
+#include <VBox/vmm/pdmcritsectrw.h>
+#include <VBox/vmm/vmcc.h>
+#include <iprt/errcore.h>
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+
+
+#if defined(IN_RING3) /*|| defined(IN_RING0) - not called from ring-0 */
+/**
+ * Process the critical sections (both types) queued for ring-3 'leave'.
+ *
+ * @param   pVM           The cross context VM structure.
+ * @param   pVCpu         The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(void) PDMCritSectBothFF(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    uint32_t i;
+    Assert(   pVCpu->pdm.s.cQueuedCritSectLeaves       > 0
+           || pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves > 0
+           || pVCpu->pdm.s.cQueuedCritSectRwExclLeaves > 0);
+
+    /* Shared leaves. */
+    i = pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves;
+    pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves = 0;
+    while (i-- > 0)
+    {
+# ifdef IN_RING3
+        PPDMCRITSECTRW pCritSectRw = pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i];
+# else
+        PPDMCRITSECTRW pCritSectRw = (PPDMCRITSECTRW)MMHyperR3ToCC(pVCpu->CTX_SUFF(pVM),
+                                                                   pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i]);
+# endif
+
+        pdmCritSectRwLeaveSharedQueued(pVM, pCritSectRw);
+        LogIt(RTLOGGRPFLAGS_FLOW, LOG_GROUP_PDM_CRITSECTRW, ("PDMR3CritSectFF: %p (shared)\n", pCritSectRw));
+    }
+
+    /* Last, exclusive leaves. */
+    i = pVCpu->pdm.s.cQueuedCritSectRwExclLeaves;
+    pVCpu->pdm.s.cQueuedCritSectRwExclLeaves = 0;
+    while (i-- > 0)
+    {
+# ifdef IN_RING3
+        PPDMCRITSECTRW pCritSectRw = pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i];
+# else
+        PPDMCRITSECTRW pCritSectRw = (PPDMCRITSECTRW)MMHyperR3ToCC(pVCpu->CTX_SUFF(pVM),
+                                                                   pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i]);
+# endif
+
+        pdmCritSectRwLeaveExclQueued(pVM, pCritSectRw);
+        LogIt(RTLOGGRPFLAGS_FLOW, LOG_GROUP_PDM_CRITSECTRW, ("PDMR3CritSectFF: %p (exclusive)\n", pCritSectRw));
+    }
+
+    /* Normal leaves. */
+    i = pVCpu->pdm.s.cQueuedCritSectLeaves;
+    pVCpu->pdm.s.cQueuedCritSectLeaves = 0;
+    while (i-- > 0)
+    {
+# ifdef IN_RING3
+        PPDMCRITSECT pCritSect = pVCpu->pdm.s.apQueuedCritSectLeaves[i];
+# else
+        PPDMCRITSECT pCritSect = (PPDMCRITSECT)MMHyperR3ToCC(pVCpu->CTX_SUFF(pVM), pVCpu->pdm.s.apQueuedCritSectLeaves[i]);
+# endif
+        Assert(pCritSect->s.Core.NativeThreadOwner == pVCpu->hNativeThread);
+
+        /* Note! We *must* clear the pending-unlock flag here and not depend on
+                 PDMCritSectLeave to do it, as the EMT might be sitting on
+                 further nestings since it queued the section to be left, and
+                 leaving it set would throw subsequent PDMCritSectIsOwner calls.
+
+                 This will happen with the PGM lock if we nip back to ring-3 for
+                 more handy pages or similar where the lock is supposed to be
+                 held while in ring-3. */
+        ASMAtomicAndU32(&pCritSect->s.Core.fFlags, ~PDMCRITSECT_FLAGS_PENDING_UNLOCK);
+        PDMCritSectLeave(pVM, pCritSect);
+        LogFlow(("PDMR3CritSectFF: %p\n", pCritSect));
+    }
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PDM_CRITSECT);
+}
+#endif /* IN_RING3 || IN_RING0 */
+
diff --git a/src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp b/src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp
new file mode 100644
index 00000000..7e66d6ae
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllCritSectRw.cpp
@@ -0,0 +1,2134 @@
+/* $Id: PDMAllCritSectRw.cpp $ */
+/** @file
+ * IPRT - Read/Write Critical Section, Generic.
+ */
+
+/*
+ * Copyright (C) 2009-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_CRITSECTRW
+#include "PDMInternal.h"
+#include <VBox/vmm/pdmcritsectrw.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/vmm/hm.h>
+
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#ifdef IN_RING3
+# include <iprt/lockvalidator.h>
+#endif
+#if defined(IN_RING3) || defined(IN_RING0)
+# include <iprt/semaphore.h>
+# include <iprt/thread.h>
+#endif
+#ifdef IN_RING0
+# include <iprt/time.h>
+#endif
+#ifdef RT_ARCH_AMD64
+# include <iprt/x86.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#if 0 /* unused */
+/** The number loops to spin for shared access in ring-3. */
+#define PDMCRITSECTRW_SHRD_SPIN_COUNT_R3        20
+/** The number loops to spin for shared access in ring-0. */
+#define PDMCRITSECTRW_SHRD_SPIN_COUNT_R0        128
+/** The number loops to spin for shared access in the raw-mode context. */
+#define PDMCRITSECTRW_SHRD_SPIN_COUNT_RC        128
+
+/** The number loops to spin for exclusive access in ring-3. */
+#define PDMCRITSECTRW_EXCL_SPIN_COUNT_R3        20
+/** The number loops to spin for exclusive access in ring-0. */
+#define PDMCRITSECTRW_EXCL_SPIN_COUNT_R0        256
+/** The number loops to spin for exclusive access in the raw-mode context. */
+#define PDMCRITSECTRW_EXCL_SPIN_COUNT_RC        256
+#endif
+
+/** Max number of write or write/read recursions. */
+#define PDM_CRITSECTRW_MAX_RECURSIONS           _1M
+
+/** Skips some of the overly paranoid atomic reads and updates.
+ * Makes some assumptions about cache coherence, though not brave enough not to
+ * always end with an atomic update. */
+#define PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
+
+/** For reading RTCRITSECTRWSTATE::s::u64State. */
+#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
+# define PDMCRITSECTRW_READ_STATE(a_pu64State)  ASMAtomicUoReadU64(a_pu64State)
+#else
+# define PDMCRITSECTRW_READ_STATE(a_pu64State)  ASMAtomicReadU64(a_pu64State)
+#endif
+
+
+/* Undefine the automatic VBOX_STRICT API mappings. */
+#undef PDMCritSectRwEnterExcl
+#undef PDMCritSectRwTryEnterExcl
+#undef PDMCritSectRwEnterShared
+#undef PDMCritSectRwTryEnterShared
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#if defined(RTASM_HAVE_CMP_WRITE_U128) && defined(RT_ARCH_AMD64)
+static int32_t g_fCmpWriteSupported = -1;
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int pdmCritSectRwLeaveSharedWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal);
+
+
+#ifdef RTASM_HAVE_CMP_WRITE_U128
+
+# ifdef RT_ARCH_AMD64
+/**
+ * Called once to initialize g_fCmpWriteSupported.
+ */
+DECL_NO_INLINE(static, bool) pdmCritSectRwIsCmpWriteU128SupportedSlow(void)
+{
+    bool const fCmpWriteSupported = RT_BOOL(ASMCpuId_ECX(1) & X86_CPUID_FEATURE_ECX_CX16);
+    ASMAtomicWriteS32(&g_fCmpWriteSupported, fCmpWriteSupported);
+    return fCmpWriteSupported;
+}
+# endif
+
+
+/**
+ * Indicates whether hardware actually supports 128-bit compare & write.
+ */
+DECL_FORCE_INLINE(bool) pdmCritSectRwIsCmpWriteU128Supported(void)
+{
+# ifdef RT_ARCH_AMD64
+    int32_t const fCmpWriteSupported = g_fCmpWriteSupported;
+    if (RT_LIKELY(fCmpWriteSupported >= 0))
+        return fCmpWriteSupported != 0;
+    return pdmCritSectRwIsCmpWriteU128SupportedSlow();
+# else
+    return true;
+# endif
+}
+
+#endif /* RTASM_HAVE_CMP_WRITE_U128 */
+
+/**
+ * Gets the ring-3 native thread handle of the calling thread.
+ *
+ * @returns native thread handle (ring-3).
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       The read/write critical section.  This is only used in
+ *                      R0 and RC.
+ */
+DECL_FORCE_INLINE(RTNATIVETHREAD) pdmCritSectRwGetNativeSelf(PVMCC pVM, PCPDMCRITSECTRW pThis)
+{
+#ifdef IN_RING3
+    RT_NOREF(pVM, pThis);
+    RTNATIVETHREAD  hNativeSelf = RTThreadNativeSelf();
+
+#elif defined(IN_RING0)
+    AssertMsgReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, ("%RX32\n", pThis->s.Core.u32Magic),
+                    NIL_RTNATIVETHREAD);
+    RTNATIVETHREAD  hNativeSelf = GVMMR0GetRing3ThreadForSelf(pVM);
+    Assert(hNativeSelf != NIL_RTNATIVETHREAD);
+
+#else
+# error "invalid context"
+#endif
+    return hNativeSelf;
+}
+
+
+DECL_NO_INLINE(static, int) pdmCritSectRwCorrupted(PPDMCRITSECTRW pThis, const char *pszMsg)
+{
+    ASMAtomicWriteU32(&pThis->s.Core.u32Magic, PDMCRITSECTRW_MAGIC_CORRUPT);
+    LogRel(("PDMCritSect: %s pCritSect=%p\n", pszMsg, pThis));
+    return VERR_PDM_CRITSECTRW_IPE;
+}
+
+
+
+#ifdef IN_RING3
+/**
+ * Changes the lock validator sub-class of the read/write critical section.
+ *
+ * It is recommended to try make sure that nobody is using this critical section
+ * while changing the value.
+ *
+ * @returns The old sub-class.  RTLOCKVAL_SUB_CLASS_INVALID is returns if the
+ *          lock validator isn't compiled in or either of the parameters are
+ *          invalid.
+ * @param   pThis           Pointer to the read/write critical section.
+ * @param   uSubClass       The new sub-class value.
+ */
+VMMDECL(uint32_t) PDMR3CritSectRwSetSubClass(PPDMCRITSECTRW pThis, uint32_t uSubClass)
+{
+    AssertPtrReturn(pThis, RTLOCKVAL_SUB_CLASS_INVALID);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, RTLOCKVAL_SUB_CLASS_INVALID);
+# if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    AssertReturn(!(pThis->s.Core.fFlags & RTCRITSECT_FLAGS_NOP), RTLOCKVAL_SUB_CLASS_INVALID);
+
+    RTLockValidatorRecSharedSetSubClass(pThis->s.Core.pValidatorRead, uSubClass);
+    return RTLockValidatorRecExclSetSubClass(pThis->s.Core.pValidatorWrite, uSubClass);
+# else
+    NOREF(uSubClass);
+    return RTLOCKVAL_SUB_CLASS_INVALID;
+# endif
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Worker for pdmCritSectRwEnterShared returning with read-ownership of the CS.
+ */
+DECL_FORCE_INLINE(int) pdmCritSectRwEnterSharedGotIt(PPDMCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos,
+                                                     bool fNoVal, RTTHREAD hThreadSelf)
+{
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    if (!fNoVal)
+        RTLockValidatorRecSharedAddOwner(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos);
+#else
+    RT_NOREF(pSrcPos, fNoVal, hThreadSelf);
+#endif
+
+    /* got it! */
+    STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared));
+    Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT));
+    return VINF_SUCCESS;
+}
+
+/**
+ * Worker for pdmCritSectRwEnterShared and pdmCritSectRwEnterSharedBailOut
+ * that decrement the wait count and maybe resets the semaphore.
+ */
+DECLINLINE(int) pdmCritSectRwEnterSharedGotItAfterWaiting(PVMCC pVM, PPDMCRITSECTRW pThis, uint64_t u64State,
+                                                          PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf)
+{
+    for (;;)
+    {
+        uint64_t const u64OldState = u64State;
+        uint64_t       cWait       = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
+        AssertReturn(cWait > 0, pdmCritSectRwCorrupted(pThis, "Invalid waiting read count"));
+        AssertReturn((u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT > 0,
+                     pdmCritSectRwCorrupted(pThis, "Invalid read count"));
+        cWait--;
+        u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
+        u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
+
+        if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+        {
+            if (cWait == 0)
+            {
+                if (ASMAtomicXchgBool(&pThis->s.Core.fNeedReset, false))
+                {
+                    int rc = SUPSemEventMultiReset(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
+                    AssertRCReturn(rc, rc);
+                }
+            }
+            return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
+        }
+
+        ASMNopPause();
+        AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+        ASMNopPause();
+
+        u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+    }
+    /* not reached */
+}
+
+
+#if defined(IN_RING0) || (defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT))
+/**
+ * Worker for pdmCritSectRwEnterSharedContended that decrements both read counts
+ * and returns @a rc.
+ *
+ * @note May return VINF_SUCCESS if we race the exclusive leave function and
+ *       come out on the bottom.
+ *
+ *       Ring-3 only calls in a case where it is _not_ acceptable to take the
+ *       lock, so even if we get the lock we'll have to leave.  In the ring-0
+ *       contexts, we can safely return VINF_SUCCESS in case of a race.
+ */
+DECL_NO_INLINE(static, int) pdmCritSectRwEnterSharedBailOut(PVMCC pVM, PPDMCRITSECTRW pThis, int rc,
+                                                            PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf)
+{
+#ifdef IN_RING0
+    uint64_t const tsStart    = RTTimeNanoTS();
+    uint64_t       cNsElapsed = 0;
+#endif
+    for (;;)
+    {
+        uint64_t u64State    = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+        uint64_t u64OldState = u64State;
+
+        uint64_t cWait       = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
+        AssertReturn(cWait > 0, pdmCritSectRwCorrupted(pThis, "Invalid waiting read count on bailout"));
+        cWait--;
+
+        uint64_t c           = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
+        AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid read count on bailout"));
+
+        if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
+        {
+            c--;
+            u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK);
+            u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT);
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                return rc;
+        }
+        else
+        {
+            /*
+             * The direction changed, so we can actually get the lock now.
+             *
+             * This means that we _have_ to wait on the semaphore to be signalled
+             * so we can properly reset it.  Otherwise the stuff gets out of wack,
+             * because signalling and resetting will race one another.  An
+             * exception would be if we're not the last reader waiting and don't
+             * need to worry about the resetting.
+             *
+             * An option would be to do the resetting in PDMCritSectRwEnterExcl,
+             * but that would still leave a racing PDMCritSectRwEnterShared
+             * spinning hard for a little bit, which isn't great...
+             */
+            if (cWait == 0)
+            {
+# ifdef IN_RING0
+                /* Do timeout processing first to avoid redoing the above. */
+                uint32_t cMsWait;
+                if (cNsElapsed <= RT_NS_10SEC)
+                    cMsWait = 32;
+                else
+                {
+                    u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
+                    u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
+                    if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                    {
+                        LogFunc(("%p: giving up\n", pThis));
+                        return rc;
+                    }
+                    cMsWait = 2;
+                }
+
+                int rcWait = SUPSemEventMultiWait(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead, cMsWait);
+                Log11Func(("%p: rc=%Rrc %'RU64 ns (hNativeWriter=%p u64State=%#RX64)\n", pThis, rcWait,
+                           RTTimeNanoTS() - tsStart, pThis->s.Core.u.s.hNativeWriter, pThis->s.Core.u.s.u64State));
+# else
+                RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
+                int rcWait = SUPSemEventMultiWaitNoResume(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead, RT_MS_5SEC);
+                RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
+# endif
+                if (rcWait == VINF_SUCCESS)
+                {
+# ifdef IN_RING0
+                    return pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf);
+# else
+                    /* ring-3: Cannot return VINF_SUCCESS. */
+                    Assert(RT_FAILURE_NP(rc));
+                    int rc2 = pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf);
+                    if (RT_SUCCESS(rc2))
+                        rc2 = pdmCritSectRwLeaveSharedWorker(pVM, pThis, fNoVal);
+                    return rc;
+# endif
+                }
+                AssertMsgReturn(rcWait == VERR_TIMEOUT || rcWait == VERR_INTERRUPTED,
+                                ("%p: rcWait=%Rrc rc=%Rrc", pThis, rcWait, rc),
+                                RT_FAILURE_NP(rcWait) ? rcWait : -rcWait);
+            }
+            else
+            {
+                u64State &= ~RTCSRW_WAIT_CNT_RD_MASK;
+                u64State |= cWait << RTCSRW_WAIT_CNT_RD_SHIFT;
+                if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                    return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
+            }
+
+# ifdef IN_RING0
+            /* Calculate the elapsed time here to avoid redoing state work. */
+            cNsElapsed = RTTimeNanoTS() - tsStart;
+# endif
+        }
+
+        ASMNopPause();
+        AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+        ASMNopPause();
+    }
+}
+#endif /* IN_RING0  || (IN_RING3 && PDMCRITSECTRW_STRICT) */
+
+
+/**
+ * Worker for pdmCritSectRwEnterShared that handles waiting for a contended CS.
+ * Caller has already added us to the read and read-wait counters.
+ */
+static int pdmCritSectRwEnterSharedContended(PVMCC pVM, PVMCPUCC pVCpu, PPDMCRITSECTRW pThis,
+                                             int rcBusy, PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, RTTHREAD hThreadSelf)
+{
+    PSUPDRVSESSION const    pSession          = pVM->pSession;
+    SUPSEMEVENTMULTI const  hEventMulti       = (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead;
+# ifdef IN_RING0
+    uint64_t const          tsStart           = RTTimeNanoTS();
+    uint64_t const          cNsMaxTotalDef    = RT_NS_5MIN;
+    uint64_t                cNsMaxTotal       = cNsMaxTotalDef;
+    uint32_t                cMsMaxOne         = RT_MS_5SEC;
+    bool                    fNonInterruptible = false;
+# endif
+
+    for (uint32_t iLoop = 0; ; iLoop++)
+    {
+        /*
+         * Wait for the direction to switch.
+         */
+        int rc;
+# ifdef IN_RING3
+#  if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+        rc = RTLockValidatorRecSharedCheckBlocking(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, true,
+                                                   RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_READ, false);
+        if (RT_FAILURE(rc))
+            return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rc, pSrcPos, fNoVal, hThreadSelf);
+#  else
+        RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_READ, false);
+#  endif
+# endif
+
+        for (;;)
+        {
+            /*
+             * We always wait with a timeout so we can re-check the structure sanity
+             * and not get stuck waiting on a corrupt or deleted section.
+             */
+# ifdef IN_RING3
+            rc = SUPSemEventMultiWaitNoResume(pSession, hEventMulti, RT_MS_5SEC);
+# else
+            rc = !fNonInterruptible
+               ? SUPSemEventMultiWaitNoResume(pSession, hEventMulti, cMsMaxOne)
+               : SUPSemEventMultiWait(pSession, hEventMulti, cMsMaxOne);
+            Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hNativeWriter=%p u64State=%#RX64)\n", pThis, rc,
+                       RTTimeNanoTS() - tsStart, cMsMaxOne, pThis->s.Core.u.s.hNativeWriter, pThis->s.Core.u.s.u64State));
+# endif
+            if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
+            { /* likely */ }
+            else
+            {
+# ifdef IN_RING3
+                RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
+# endif
+                return VERR_SEM_DESTROYED;
+            }
+            if (RT_LIKELY(rc == VINF_SUCCESS))
+                break;
+
+            /*
+             * Timeout and interrupted waits needs careful handling in ring-0
+             * because we're cooperating with ring-3 on this critical section
+             * and thus need to make absolutely sure we won't get stuck here.
+             *
+             * The r0 interrupted case means something is pending (termination,
+             * signal, APC, debugger, whatever), so we must try our best to
+             * return to the caller and to ring-3 so it can be dealt with.
+             */
+            if (rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED)
+            {
+# ifdef IN_RING0
+                uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart;
+                int const      rcTerm     = RTThreadQueryTerminationStatus(NIL_RTTHREAD);
+                AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING,
+                          ("rcTerm=%Rrc\n", rcTerm));
+                if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef)
+                    cNsMaxTotal = RT_NS_1MIN;
+
+                if (rc == VERR_TIMEOUT)
+                {
+                    /* Try return get out of here with a non-VINF_SUCCESS status if
+                       the thread is terminating or if the timeout has been exceeded. */
+                    STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedVerrTimeout);
+                    if (   rcTerm == VINF_THREAD_IS_TERMINATING
+                        || cNsElapsed > cNsMaxTotal)
+                        return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc,
+                                                               pSrcPos, fNoVal, hThreadSelf);
+                }
+                else
+                {
+                    /* For interrupt cases, we must return if we can.  If rcBusy is VINF_SUCCESS,
+                       we will try non-interruptible sleep for a while to help resolve the issue
+                       w/o guru'ing. */
+                    STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedVerrInterrupted);
+                    if (   rcTerm != VINF_THREAD_IS_TERMINATING
+                        && rcBusy == VINF_SUCCESS
+                        && pVCpu != NULL
+                        && cNsElapsed <= cNsMaxTotal)
+                    {
+                        if (!fNonInterruptible)
+                        {
+                            STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwSharedNonInterruptibleWaits);
+                            fNonInterruptible   = true;
+                            cMsMaxOne           = 32;
+                            uint64_t cNsLeft = cNsMaxTotal - cNsElapsed;
+                            if (cNsLeft > RT_NS_10SEC)
+                                cNsMaxTotal = cNsElapsed + RT_NS_10SEC;
+                        }
+                    }
+                    else
+                        return pdmCritSectRwEnterSharedBailOut(pVM, pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc,
+                                                               pSrcPos, fNoVal, hThreadSelf);
+                }
+# else  /* IN_RING3 */
+                RT_NOREF(pVM, pVCpu, rcBusy);
+# endif /* IN_RING3 */
+            }
+            /*
+             * Any other return code is fatal.
+             */
+            else
+            {
+# ifdef IN_RING3
+                RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
+# endif
+                AssertMsgFailed(("rc=%Rrc\n", rc));
+                return RT_FAILURE_NP(rc) ? rc : -rc;
+            }
+        }
+
+# ifdef IN_RING3
+        RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_READ);
+# endif
+
+        /*
+         * Check the direction.
+         */
+        Assert(pThis->s.Core.fNeedReset);
+        uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+        if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
+        {
+            /*
+             * Decrement the wait count and maybe reset the semaphore (if we're last).
+             */
+            return pdmCritSectRwEnterSharedGotItAfterWaiting(pVM, pThis, u64State, pSrcPos, fNoVal, hThreadSelf);
+        }
+
+        AssertMsg(iLoop < 1,
+                  ("%p: %u u64State=%#RX64 hNativeWriter=%p\n", pThis, iLoop, u64State, pThis->s.Core.u.s.hNativeWriter));
+        RTThreadYield();
+    }
+
+    /* not reached */
+}
+
+
+/**
+ * Worker that enters a read/write critical section with shard access.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The busy return code for ring-0 and ring-3.
+ * @param   fTryOnly    Only try enter it, don't wait.
+ * @param   pSrcPos     The source position. (Can be NULL.)
+ * @param   fNoVal      No validation records.
+ */
+static int pdmCritSectRwEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly,
+                                    PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    RTTHREAD hThreadSelf = RTThreadSelfAutoAdopt();
+    if (!fTryOnly)
+    {
+        int            rc9;
+        RTNATIVETHREAD hNativeWriter;
+        ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
+        if (hNativeWriter != NIL_RTTHREAD && hNativeWriter == pdmCritSectRwGetNativeSelf(pVM, pThis))
+            rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
+        else
+            rc9 = RTLockValidatorRecSharedCheckOrder(pThis->s.Core.pValidatorRead, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
+        if (RT_FAILURE(rc9))
+            return rc9;
+    }
+#else
+    RTTHREAD hThreadSelf = NIL_RTTHREAD;
+#endif
+
+    /*
+     * Work the state.
+     */
+    uint64_t u64State    = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+    uint64_t u64OldState = u64State;
+    for (;;)
+    {
+        if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
+        {
+            /* It flows in the right direction, try follow it before it changes. */
+            uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
+            c++;
+            Assert(c < RTCSRW_CNT_MASK / 4);
+            AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS);
+            u64State &= ~RTCSRW_CNT_RD_MASK;
+            u64State |= c << RTCSRW_CNT_RD_SHIFT;
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
+        }
+        else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0)
+        {
+            /* Wrong direction, but we're alone here and can simply try switch the direction. */
+            u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
+            u64State |= (UINT64_C(1) << RTCSRW_CNT_RD_SHIFT) | (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT);
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+            {
+                Assert(!pThis->s.Core.fNeedReset);
+                return pdmCritSectRwEnterSharedGotIt(pThis, pSrcPos, fNoVal, hThreadSelf);
+            }
+        }
+        else
+        {
+            /* Is the writer perhaps doing a read recursion? */
+            RTNATIVETHREAD hNativeWriter;
+            ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
+            if (hNativeWriter != NIL_RTNATIVETHREAD)
+            {
+                RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
+                if (hNativeSelf == hNativeWriter)
+                {
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+                    if (!fNoVal)
+                    {
+                        int rc9 = RTLockValidatorRecExclRecursionMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core, pSrcPos);
+                        if (RT_FAILURE(rc9))
+                            return rc9;
+                    }
+#endif
+                    uint32_t const cReads = ASMAtomicIncU32(&pThis->s.Core.cWriterReads);
+                    Assert(cReads < _16K);
+                    AssertReturnStmt(cReads < PDM_CRITSECTRW_MAX_RECURSIONS, ASMAtomicDecU32(&pThis->s.Core.cWriterReads),
+                                     VERR_PDM_CRITSECTRW_TOO_MANY_RECURSIONS);
+                    STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterShared));
+                    return VINF_SUCCESS; /* don't break! */
+                }
+            }
+
+            /*
+             * If we're only trying, return already.
+             */
+            if (fTryOnly)
+            {
+                STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared));
+                return VERR_SEM_BUSY;
+            }
+
+#if defined(IN_RING3) || defined(IN_RING0)
+            /*
+             * Add ourselves to the queue and wait for the direction to change.
+             */
+            uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
+            c++;
+            Assert(c < RTCSRW_CNT_MASK / 2);
+            AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS);
+
+            uint64_t cWait = (u64State & RTCSRW_WAIT_CNT_RD_MASK) >> RTCSRW_WAIT_CNT_RD_SHIFT;
+            cWait++;
+            Assert(cWait <= c);
+            Assert(cWait < RTCSRW_CNT_MASK / 2);
+            AssertReturn(cWait < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_READERS);
+
+            u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_WAIT_CNT_RD_MASK);
+            u64State |= (c << RTCSRW_CNT_RD_SHIFT) | (cWait << RTCSRW_WAIT_CNT_RD_SHIFT);
+
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+            {
+                /*
+                 * In ring-3 it's straight forward, just optimize the RTThreadSelf() call.
+                 */
+# if defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT)
+                return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf);
+# elif defined(IN_RING3)
+                return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, RTThreadSelf());
+# else /* IN_RING0 */
+                /*
+                 * In ring-0 context we have to take the special VT-x/AMD-V HM context into
+                 * account when waiting on contended locks.
+                 */
+                PVMCPUCC pVCpu = VMMGetCpu(pVM);
+                if (pVCpu)
+                {
+                    VMMR0EMTBLOCKCTX Ctx;
+                    int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pThis, &Ctx);
+                    if (rc == VINF_SUCCESS)
+                    {
+                        Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+                        rc = pdmCritSectRwEnterSharedContended(pVM, pVCpu, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf);
+
+                        VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
+                    }
+                    else
+                    {
+                        //STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLockBusy);
+                        rc = pdmCritSectRwEnterSharedBailOut(pVM, pThis, rc, pSrcPos, fNoVal, hThreadSelf);
+                    }
+                    return rc;
+                }
+
+                /* Non-EMT. */
+                Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+                return pdmCritSectRwEnterSharedContended(pVM, NULL, pThis, rcBusy, pSrcPos, fNoVal, hThreadSelf);
+# endif /* IN_RING0 */
+            }
+
+#else  /* !IN_RING3 && !IN_RING0 */
+            /*
+             * We cannot call SUPSemEventMultiWaitNoResume in this context. Go
+             * back to ring-3 and do it there or return rcBusy.
+             */
+# error "Unused code."
+            STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterShared));
+            if (rcBusy == VINF_SUCCESS)
+            {
+                PVMCPUCC  pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
+                /** @todo Should actually do this in via VMMR0.cpp instead of going all the way
+                 *        back to ring-3. Goes for both kind of crit sects. */
+                return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_SHARED, MMHyperCCToR3(pVM, pThis));
+            }
+            return rcBusy;
+#endif /* !IN_RING3 && !IN_RING0 */
+        }
+
+        ASMNopPause();
+        if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
+        { /* likely */ }
+        else
+            return VERR_SEM_DESTROYED;
+        ASMNopPause();
+
+        u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+        u64OldState = u64State;
+    }
+    /* not reached */
+}
+
+
+/**
+ * Enter a critical section with shared (read) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  rcBusy if in ring-0 or raw-mode context and it is busy.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The status code to return when we're in RC or R0 and the
+ *                      section is busy.   Pass VINF_SUCCESS to acquired the
+ *                      critical section thru a ring-3 call if necessary.
+ * @sa      PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterShared,
+ *          PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared,
+ *          RTCritSectRwEnterShared.
+ */
+VMMDECL(int) PDMCritSectRwEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy)
+{
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Enter a critical section with shared (read) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  rcBusy if in ring-0 or raw-mode context and it is busy.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The status code to return when we're in RC or R0 and the
+ *                      section is busy.   Pass VINF_SUCCESS to acquired the
+ *                      critical section thru a ring-3 call if necessary.
+ * @param   uId         Where we're entering the section.
+ * @param   SRC_POS     The source position.
+ * @sa      PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
+ *          PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwLeaveShared,
+ *          RTCritSectRwEnterSharedDebug.
+ */
+VMMDECL(int) PDMCritSectRwEnterSharedDebug(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectRwEnterShared(pVM, pThis, rcBusy, false /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Try enter a critical section with shared (read) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwTryEnterSharedDebug, PDMCritSectRwEnterShared,
+ *          PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared,
+ *          RTCritSectRwTryEnterShared.
+ */
+VMMDECL(int) PDMCritSectRwTryEnterShared(PVMCC pVM, PPDMCRITSECTRW pThis)
+{
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Try enter a critical section with shared (read) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   uId         Where we're entering the section.
+ * @param   SRC_POS     The source position.
+ * @sa      PDMCritSectRwTryEnterShared, PDMCritSectRwEnterShared,
+ *          PDMCritSectRwEnterSharedDebug, PDMCritSectRwLeaveShared,
+ *          RTCritSectRwTryEnterSharedDebug.
+ */
+VMMDECL(int) PDMCritSectRwTryEnterSharedDebug(PVMCC pVM, PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, true /*fTryOnly*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+#ifdef IN_RING3
+/**
+ * Enters a PDM read/write critical section with shared (read) access.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   fCallRing3  Whether this is a VMMRZCallRing3()request.
+ */
+VMMR3DECL(int) PDMR3CritSectRwEnterSharedEx(PVM pVM, PPDMCRITSECTRW pThis, bool fCallRing3)
+{
+    return pdmCritSectRwEnterShared(pVM, pThis, VERR_SEM_BUSY, false /*fTryAgain*/, NULL, fCallRing3);
+}
+#endif
+
+
+/**
+ * Leave a critical section held with shared access.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   fNoVal      No validation records (i.e. queued release).
+ * @sa      PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
+ *          PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug,
+ *          PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared.
+ */
+static int pdmCritSectRwLeaveSharedWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    NOREF(fNoVal);
+#endif
+
+    /*
+     * Check the direction and take action accordingly.
+     */
+#ifdef IN_RING0
+    PVMCPUCC pVCpu       = NULL;
+#endif
+    uint64_t u64State    = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+    uint64_t u64OldState = u64State;
+    if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
+    {
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+        if (fNoVal)
+            Assert(!RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD));
+        else
+        {
+            int rc9 = RTLockValidatorRecSharedCheckAndRelease(pThis->s.Core.pValidatorRead, NIL_RTTHREAD);
+            if (RT_FAILURE(rc9))
+                return rc9;
+        }
+#endif
+        for (;;)
+        {
+            uint64_t c = (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
+            AssertReturn(c > 0, VERR_NOT_OWNER);
+            c--;
+
+            if (   c > 0
+                || (u64State & RTCSRW_CNT_WR_MASK) == 0)
+            {
+                /* Don't change the direction. */
+                u64State &= ~RTCSRW_CNT_RD_MASK;
+                u64State |= c << RTCSRW_CNT_RD_SHIFT;
+                if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                    break;
+            }
+            else
+            {
+#if defined(IN_RING3) || defined(IN_RING0)
+# ifdef IN_RING0
+                Assert(RTSemEventIsSignalSafe() == RTSemEventMultiIsSignalSafe());
+                if (!pVCpu)
+                    pVCpu = VMMGetCpu(pVM);
+                if (   pVCpu == NULL /* non-EMT access, if we implement it must be able to block */
+                    || VMMRZCallRing3IsEnabled(pVCpu)
+                    || RTSemEventIsSignalSafe()
+                    || (   VMMR0ThreadCtxHookIsEnabled(pVCpu)       /* Doesn't matter if Signal() blocks if we have hooks, ... */
+                        && RTThreadPreemptIsEnabled(NIL_RTTHREAD)   /* ... and preemption is still enabled, */
+                        && ASMIntAreEnabled())                      /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */
+                   )
+# endif
+                {
+                    /* Reverse the direction and signal the writer threads. */
+                    u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_DIR_MASK);
+                    u64State |= RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT;
+                    if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                    {
+                        int rc;
+# ifdef IN_RING0
+                        STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared);
+                        if (!RTSemEventIsSignalSafe() && pVCpu != NULL)
+                        {
+                            VMMR0EMTBLOCKCTX Ctx;
+                            rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx);
+                            VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
+
+                            rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
+
+                            VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
+                        }
+                        else
+# endif
+                            rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
+                        AssertRC(rc);
+                        return rc;
+                    }
+                }
+#endif /* IN_RING3 || IN_RING0 */
+#ifndef IN_RING3
+# ifdef IN_RING0
+                else
+# endif
+                {
+                    /* Queue the exit request (ring-3). */
+# ifndef IN_RING0
+                    PVMCPUCC    pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
+# endif
+                    uint32_t    i     = pVCpu->pdm.s.cQueuedCritSectRwShrdLeaves++;
+                    LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3 c=%d (%#llx)\n", i, pThis, c, u64State));
+                    VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves),
+                                                  ("i=%u\n", i), VERR_PDM_CRITSECTRW_IPE);
+                    pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] = pThis->s.pSelfR3;
+                    VMM_ASSERT_RELEASE_MSG_RETURN(pVM,
+                                                     RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i])
+                                                  &&    ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i] & HOST_PAGE_OFFSET_MASK)
+                                                     == ((uintptr_t)pThis & HOST_PAGE_OFFSET_MASK),
+                                                  ("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectRwShrdLeaves[i], pThis),
+                                                  pdmCritSectRwCorrupted(pThis, "Invalid self pointer"));
+                    VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
+                    VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
+                    STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
+                    STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveShared);
+                    break;
+                }
+#endif
+            }
+
+            ASMNopPause();
+            if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
+            { }
+            else
+                return VERR_SEM_DESTROYED;
+            ASMNopPause();
+
+            u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+            u64OldState = u64State;
+        }
+    }
+    else
+    {
+        /*
+         * Write direction. Check that it's the owner calling and that it has reads to undo.
+         */
+        RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
+        AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
+
+        RTNATIVETHREAD hNativeWriter;
+        ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
+        AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
+        AssertReturn(pThis->s.Core.cWriterReads > 0, VERR_NOT_OWNER);
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+        if (!fNoVal)
+        {
+            int rc = RTLockValidatorRecExclUnwindMixed(pThis->s.Core.pValidatorWrite, &pThis->s.Core.pValidatorRead->Core);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+#endif
+        uint32_t cDepth = ASMAtomicDecU32(&pThis->s.Core.cWriterReads);
+        AssertReturn(cDepth < PDM_CRITSECTRW_MAX_RECURSIONS, pdmCritSectRwCorrupted(pThis, "too many writer-read recursions"));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Leave a critical section held with shared access.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwEnterShared, PDMCritSectRwTryEnterShared,
+ *          PDMCritSectRwEnterSharedDebug, PDMCritSectRwTryEnterSharedDebug,
+ *          PDMCritSectRwLeaveExcl, RTCritSectRwLeaveShared.
+ */
+VMMDECL(int) PDMCritSectRwLeaveShared(PVMCC pVM, PPDMCRITSECTRW pThis)
+{
+    return pdmCritSectRwLeaveSharedWorker(pVM, pThis, false /*fNoVal*/);
+}
+
+
+#if defined(IN_RING3) || defined(IN_RING0)
+/**
+ * PDMCritSectBothFF interface.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ */
+void pdmCritSectRwLeaveSharedQueued(PVMCC pVM, PPDMCRITSECTRW pThis)
+{
+    pdmCritSectRwLeaveSharedWorker(pVM, pThis, true /*fNoVal*/);
+}
+#endif
+
+
+/**
+ * Worker for pdmCritSectRwEnterExcl that bails out on wait failure.
+ *
+ * @returns @a rc unless corrupted.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rc          The status to return.
+ */
+DECL_NO_INLINE(static, int) pdmCritSectRwEnterExclBailOut(PPDMCRITSECTRW pThis, int rc)
+{
+    /*
+     * Decrement the counts and return the error.
+     */
+    for (;;)
+    {
+        uint64_t       u64State    = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+        uint64_t const u64OldState = u64State;
+        uint64_t c                 = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
+        AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid write count on bailout"));
+        c--;
+        u64State &= ~RTCSRW_CNT_WR_MASK;
+        u64State |= c << RTCSRW_CNT_WR_SHIFT;
+        if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+            return rc;
+
+        ASMNopPause();
+        AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+        ASMNopPause();
+    }
+}
+
+
+/**
+ * Worker for pdmCritSectRwEnterExcl that handles the red tape after we've
+ * gotten exclusive ownership of the critical section.
+ */
+DECL_FORCE_INLINE(int) pdmCritSectRwEnterExclFirst(PPDMCRITSECTRW pThis, PCRTLOCKVALSRCPOS pSrcPos,
+                                                   bool fNoVal, RTTHREAD hThreadSelf)
+{
+    RT_NOREF(hThreadSelf, fNoVal, pSrcPos);
+    Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT));
+
+#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
+    pThis->s.Core.cWriteRecursions = 1;
+#else
+    ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 1);
+#endif
+    Assert(pThis->s.Core.cWriterReads == 0);
+
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    if (!fNoVal)
+    {
+        if (hThreadSelf == NIL_RTTHREAD)
+            hThreadSelf = RTThreadSelfAutoAdopt();
+        RTLockValidatorRecExclSetOwner(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true);
+    }
+#endif
+    STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl));
+    STAM_PROFILE_ADV_START(&pThis->s.StatWriteLocked, swl);
+    return VINF_SUCCESS;
+}
+
+
+#if defined(IN_RING3) || defined(IN_RING0)
+/**
+ * Worker for pdmCritSectRwEnterExcl that handles waiting when the section is
+ * contended.
+ */
+static int pdmR3R0CritSectRwEnterExclContended(PVMCC pVM, PVMCPUCC pVCpu, PPDMCRITSECTRW pThis, RTNATIVETHREAD hNativeSelf,
+                                               PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal, int rcBusy, RTTHREAD hThreadSelf)
+{
+    RT_NOREF(hThreadSelf, rcBusy, pSrcPos, fNoVal, pVCpu);
+
+    PSUPDRVSESSION const    pSession          = pVM->pSession;
+    SUPSEMEVENT const       hEvent            = (SUPSEMEVENT)pThis->s.Core.hEvtWrite;
+# ifdef IN_RING0
+    uint64_t const          tsStart           = RTTimeNanoTS();
+    uint64_t const          cNsMaxTotalDef    = RT_NS_5MIN;
+    uint64_t                cNsMaxTotal       = cNsMaxTotalDef;
+    uint32_t                cMsMaxOne         = RT_MS_5SEC;
+    bool                    fNonInterruptible = false;
+# endif
+
+    for (uint32_t iLoop = 0; ; iLoop++)
+    {
+        /*
+         * Wait for our turn.
+         */
+        int rc;
+# ifdef IN_RING3
+#  ifdef PDMCRITSECTRW_STRICT
+        rc = RTLockValidatorRecExclCheckBlocking(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, true,
+                                                 RT_INDEFINITE_WAIT, RTTHREADSTATE_RW_WRITE, false);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            return pdmCritSectRwEnterExclBailOut(pThis, rc);
+#  else
+        RTThreadBlocking(hThreadSelf, RTTHREADSTATE_RW_WRITE, false);
+#  endif
+# endif
+
+        for (;;)
+        {
+            /*
+             * We always wait with a timeout so we can re-check the structure sanity
+             * and not get stuck waiting on a corrupt or deleted section.
+             */
+# ifdef IN_RING3
+            rc = SUPSemEventWaitNoResume(pSession, hEvent, RT_MS_5SEC);
+# else
+            rc = !fNonInterruptible
+               ? SUPSemEventWaitNoResume(pSession, hEvent, cMsMaxOne)
+               : SUPSemEventWait(pSession, hEvent, cMsMaxOne);
+            Log11Func(("%p: rc=%Rrc %'RU64 ns (cMsMaxOne=%RU64 hNativeWriter=%p)\n",
+                       pThis, rc, RTTimeNanoTS() - tsStart, cMsMaxOne, pThis->s.Core.u.s.hNativeWriter));
+# endif
+            if (RT_LIKELY(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC))
+            { /* likely */ }
+            else
+            {
+# ifdef IN_RING3
+                RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
+# endif
+                return VERR_SEM_DESTROYED;
+            }
+            if (RT_LIKELY(rc == VINF_SUCCESS))
+                break;
+
+            /*
+             * Timeout and interrupted waits needs careful handling in ring-0
+             * because we're cooperating with ring-3 on this critical section
+             * and thus need to make absolutely sure we won't get stuck here.
+             *
+             * The r0 interrupted case means something is pending (termination,
+             * signal, APC, debugger, whatever), so we must try our best to
+             * return to the caller and to ring-3 so it can be dealt with.
+             */
+            if (rc == VERR_TIMEOUT || rc == VERR_INTERRUPTED)
+            {
+# ifdef IN_RING0
+                uint64_t const cNsElapsed = RTTimeNanoTS() - tsStart;
+                int const      rcTerm     = RTThreadQueryTerminationStatus(NIL_RTTHREAD);
+                AssertMsg(rcTerm == VINF_SUCCESS || rcTerm == VERR_NOT_SUPPORTED || rcTerm == VINF_THREAD_IS_TERMINATING,
+                          ("rcTerm=%Rrc\n", rcTerm));
+                if (rcTerm == VERR_NOT_SUPPORTED && cNsMaxTotal == cNsMaxTotalDef)
+                    cNsMaxTotal = RT_NS_1MIN;
+
+                if (rc == VERR_TIMEOUT)
+                {
+                    /* Try return get out of here with a non-VINF_SUCCESS status if
+                       the thread is terminating or if the timeout has been exceeded. */
+                    STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclVerrTimeout);
+                    if (   rcTerm == VINF_THREAD_IS_TERMINATING
+                        || cNsElapsed > cNsMaxTotal)
+                        return pdmCritSectRwEnterExclBailOut(pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc);
+                }
+                else
+                {
+                    /* For interrupt cases, we must return if we can.  If rcBusy is VINF_SUCCESS,
+                       we will try non-interruptible sleep for a while to help resolve the issue
+                       w/o guru'ing. */
+                    STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclVerrInterrupted);
+                    if (   rcTerm != VINF_THREAD_IS_TERMINATING
+                        && rcBusy == VINF_SUCCESS
+                        && pVCpu != NULL
+                        && cNsElapsed <= cNsMaxTotal)
+                    {
+                        if (!fNonInterruptible)
+                        {
+                            STAM_REL_COUNTER_INC(&pVM->pdm.s.StatCritSectRwExclNonInterruptibleWaits);
+                            fNonInterruptible   = true;
+                            cMsMaxOne           = 32;
+                            uint64_t cNsLeft = cNsMaxTotal - cNsElapsed;
+                            if (cNsLeft > RT_NS_10SEC)
+                                cNsMaxTotal = cNsElapsed + RT_NS_10SEC;
+                        }
+                    }
+                    else
+                        return pdmCritSectRwEnterExclBailOut(pThis, rcBusy != VINF_SUCCESS ? rcBusy : rc);
+                }
+# else  /* IN_RING3 */
+                RT_NOREF(pVM, pVCpu, rcBusy);
+# endif /* IN_RING3 */
+            }
+            /*
+             * Any other return code is fatal.
+             */
+            else
+            {
+# ifdef IN_RING3
+                RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
+# endif
+                AssertMsgFailed(("rc=%Rrc\n", rc));
+                return RT_FAILURE_NP(rc) ? rc : -rc;
+            }
+        }
+
+# ifdef IN_RING3
+        RTThreadUnblocked(hThreadSelf, RTTHREADSTATE_RW_WRITE);
+# endif
+
+        /*
+         * Try take exclusive write ownership.
+         */
+        uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+        if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
+        {
+            bool fDone;
+            ASMAtomicCmpXchgHandle(&pThis->s.Core.u.s.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
+            if (fDone)
+                return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf);
+        }
+        AssertMsg(iLoop < 1000, ("%u\n", iLoop)); /* may loop a few times here... */
+    }
+}
+#endif /* IN_RING3 || IN_RING0 */
+
+
+/**
+ * Worker that enters a read/write critical section with exclusive access.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The busy return code for ring-0 and ring-3.
+ * @param   fTryOnly    Only try enter it, don't wait.
+ * @param   pSrcPos     The source position. (Can be NULL.)
+ * @param   fNoVal      No validation records.
+ */
+static int pdmCritSectRwEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, bool fTryOnly,
+                                  PCRTLOCKVALSRCPOS pSrcPos, bool fNoVal)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+
+    RTTHREAD hThreadSelf = NIL_RTTHREAD;
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    if (!fTryOnly)
+    {
+        hThreadSelf = RTThreadSelfAutoAdopt();
+        int rc9 = RTLockValidatorRecExclCheckOrder(pThis->s.Core.pValidatorWrite, hThreadSelf, pSrcPos, RT_INDEFINITE_WAIT);
+        if (RT_FAILURE(rc9))
+            return rc9;
+    }
+#endif
+
+    /*
+     * Check if we're already the owner and just recursing.
+     */
+    RTNATIVETHREAD const hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
+    AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
+    RTNATIVETHREAD hNativeWriter;
+    ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
+    if (hNativeSelf == hNativeWriter)
+    {
+        Assert((PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State) & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT));
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+        if (!fNoVal)
+        {
+            int rc9 = RTLockValidatorRecExclRecursion(pThis->s.Core.pValidatorWrite, pSrcPos);
+            if (RT_FAILURE(rc9))
+                return rc9;
+        }
+#endif
+        STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(Stat,EnterExcl));
+#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
+        uint32_t const cDepth = ++pThis->s.Core.cWriteRecursions;
+#else
+        uint32_t const cDepth = ASMAtomicIncU32(&pThis->s.Core.cWriteRecursions);
+#endif
+        AssertReturnStmt(cDepth > 1 && cDepth <= PDM_CRITSECTRW_MAX_RECURSIONS,
+                         ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions),
+                         VERR_PDM_CRITSECTRW_TOO_MANY_RECURSIONS);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * First we try grab an idle critical section using 128-bit atomics.
+     */
+    /** @todo This could be moved up before the recursion check. */
+    uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+#ifdef RTASM_HAVE_CMP_WRITE_U128
+    if (   (u64State & ~RTCSRW_DIR_MASK) == 0
+        && pdmCritSectRwIsCmpWriteU128Supported())
+    {
+        RTCRITSECTRWSTATE OldState;
+        OldState.s.u64State      = u64State;
+        OldState.s.hNativeWriter = NIL_RTNATIVETHREAD;
+        AssertCompile(sizeof(OldState.s.hNativeWriter) == sizeof(OldState.u128.s.Lo));
+
+        RTCRITSECTRWSTATE NewState;
+        NewState.s.u64State      = (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT);
+        NewState.s.hNativeWriter = hNativeSelf;
+
+        if (ASMAtomicCmpWriteU128U(&pThis->s.Core.u.u128, NewState.u128, OldState.u128))
+            return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf);
+
+        u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+    }
+#endif
+
+    /*
+     * Do it step by step.  Update the state to reflect our desire.
+     */
+    uint64_t u64OldState = u64State;
+
+    for (;;)
+    {
+        if (   (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)
+            || (u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) != 0)
+        {
+            /* It flows in the right direction, try follow it before it changes. */
+            uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
+            AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_WRITERS);
+            c++;
+            Assert(c < RTCSRW_CNT_WR_MASK / 4);
+            u64State &= ~RTCSRW_CNT_WR_MASK;
+            u64State |= c << RTCSRW_CNT_WR_SHIFT;
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                break;
+        }
+        else if ((u64State & (RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK)) == 0)
+        {
+            /* Wrong direction, but we're alone here and can simply try switch the direction. */
+            u64State &= ~(RTCSRW_CNT_RD_MASK | RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
+            u64State |= (UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT);
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                break;
+        }
+        else if (fTryOnly)
+        {
+            /* Wrong direction and we're not supposed to wait, just return. */
+            STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl));
+            return VERR_SEM_BUSY;
+        }
+        else
+        {
+            /* Add ourselves to the write count and break out to do the wait. */
+            uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
+            AssertReturn(c < RTCSRW_CNT_MASK, VERR_PDM_CRITSECTRW_TOO_MANY_WRITERS);
+            c++;
+            Assert(c < RTCSRW_CNT_WR_MASK / 4);
+            u64State &= ~RTCSRW_CNT_WR_MASK;
+            u64State |= c << RTCSRW_CNT_WR_SHIFT;
+            if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                break;
+        }
+
+        ASMNopPause();
+
+        if (pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)
+        { /* likely */ }
+        else
+            return VERR_SEM_DESTROYED;
+
+        ASMNopPause();
+        u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+        u64OldState = u64State;
+    }
+
+    /*
+     * If we're in write mode now try grab the ownership. Play fair if there
+     * are threads already waiting.
+     */
+    bool fDone = (u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)
+              && (  ((u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT) == 1
+                  || fTryOnly);
+    if (fDone)
+    {
+        ASMAtomicCmpXchgHandle(&pThis->s.Core.u.s.hNativeWriter, hNativeSelf, NIL_RTNATIVETHREAD, fDone);
+        if (fDone)
+            return pdmCritSectRwEnterExclFirst(pThis, pSrcPos, fNoVal, hThreadSelf);
+    }
+
+    /*
+     * Okay, we have contention and will have to wait unless we're just trying.
+     */
+    if (fTryOnly)
+    {
+        STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl)); /** @todo different statistics for this */
+        return pdmCritSectRwEnterExclBailOut(pThis, VERR_SEM_BUSY);
+    }
+
+    STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,EnterExcl));
+
+    /*
+     * Ring-3 is pretty straight forward.
+     */
+#if defined(IN_RING3) && defined(PDMCRITSECTRW_STRICT)
+    return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, hThreadSelf);
+#elif defined(IN_RING3)
+    return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, RTThreadSelf());
+
+#elif defined(IN_RING0)
+    /*
+     * In ring-0 context we have to take the special VT-x/AMD-V HM context into
+     * account when waiting on contended locks.
+     */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    if (pVCpu)
+    {
+        VMMR0EMTBLOCKCTX Ctx;
+        int rc = VMMR0EmtPrepareToBlock(pVCpu, rcBusy, __FUNCTION__, pThis, &Ctx);
+        if (rc == VINF_SUCCESS)
+        {
+            Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+
+            rc = pdmR3R0CritSectRwEnterExclContended(pVM, pVCpu, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, NIL_RTTHREAD);
+
+            VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
+        }
+        else
+        {
+            //STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLockBusy);
+            rc = pdmCritSectRwEnterExclBailOut(pThis, rc);
+        }
+        return rc;
+    }
+
+    /* Non-EMT. */
+    Assert(RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    return pdmR3R0CritSectRwEnterExclContended(pVM, NULL, pThis, hNativeSelf, pSrcPos, fNoVal, rcBusy, NIL_RTTHREAD);
+
+#else
+# error "Unused."
+    /*
+     * Raw-mode: Call host and take it there if rcBusy is VINF_SUCCESS.
+     */
+    rcBusy = pdmCritSectRwEnterExclBailOut(pThis, rcBusy);
+    if (rcBusy == VINF_SUCCESS)
+    {
+        Assert(!fTryOnly);
+        PVMCPUCC  pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
+        /** @todo Should actually do this in via VMMR0.cpp instead of going all the way
+         *        back to ring-3. Goes for both kind of crit sects. */
+        return VMMRZCallRing3(pVM, pVCpu, VMMCALLRING3_PDM_CRIT_SECT_RW_ENTER_EXCL, MMHyperCCToR3(pVM, pThis));
+    }
+    return rcBusy;
+#endif
+}
+
+
+/**
+ * Try enter a critical section with exclusive (write) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  rcBusy if in ring-0 or raw-mode context and it is busy.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The status code to return when we're in RC or R0 and the
+ *                      section is busy.   Pass VINF_SUCCESS to acquired the
+ *                      critical section thru a ring-3 call if necessary.
+ * @sa      PDMCritSectRwEnterExclDebug, PDMCritSectRwTryEnterExcl,
+ *          PDMCritSectRwTryEnterExclDebug,
+ *          PDMCritSectEnterDebug, PDMCritSectEnter,
+ * RTCritSectRwEnterExcl.
+ */
+VMMDECL(int) PDMCritSectRwEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy)
+{
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Try enter a critical section with exclusive (write) access.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  rcBusy if in ring-0 or raw-mode context and it is busy.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   rcBusy      The status code to return when we're in RC or R0 and the
+ *                      section is busy.   Pass VINF_SUCCESS to acquired the
+ *                      critical section thru a ring-3 call if necessary.
+ * @param   uId         Where we're entering the section.
+ * @param   SRC_POS     The source position.
+ * @sa      PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExcl,
+ *          PDMCritSectRwTryEnterExclDebug,
+ *          PDMCritSectEnterDebug, PDMCritSectEnter,
+ *          RTCritSectRwEnterExclDebug.
+ */
+VMMDECL(int) PDMCritSectRwEnterExclDebug(PVMCC pVM, PPDMCRITSECTRW pThis, int rcBusy, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectRwEnterExcl(pVM, pThis, rcBusy, false /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Try enter a critical section with exclusive (write) access.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwEnterExcl, PDMCritSectRwTryEnterExclDebug,
+ *          PDMCritSectRwEnterExclDebug,
+ *          PDMCritSectTryEnter, PDMCritSectTryEnterDebug,
+ *          RTCritSectRwTryEnterExcl.
+ */
+VMMDECL(int) PDMCritSectRwTryEnterExcl(PVMCC pVM, PPDMCRITSECTRW pThis)
+{
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_NORMAL_API();
+    return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+/**
+ * Try enter a critical section with exclusive (write) access.
+ *
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_SEM_BUSY if the critsect was owned.
+ * @retval  VERR_SEM_NESTED if nested enter on a no nesting section. (Asserted.)
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   uId         Where we're entering the section.
+ * @param   SRC_POS     The source position.
+ * @sa      PDMCritSectRwTryEnterExcl, PDMCritSectRwEnterExcl,
+ *          PDMCritSectRwEnterExclDebug,
+ *          PDMCritSectTryEnterDebug, PDMCritSectTryEnter,
+ *          RTCritSectRwTryEnterExclDebug.
+ */
+VMMDECL(int) PDMCritSectRwTryEnterExclDebug(PVMCC pVM, PPDMCRITSECTRW pThis, RTHCUINTPTR uId, RT_SRC_POS_DECL)
+{
+    NOREF(uId); NOREF(pszFile); NOREF(iLine); NOREF(pszFunction);
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, NULL,    false /*fNoVal*/);
+#else
+    RTLOCKVALSRCPOS SrcPos = RTLOCKVALSRCPOS_INIT_DEBUG_API();
+    return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, true /*fTryAgain*/, &SrcPos, false /*fNoVal*/);
+#endif
+}
+
+
+#ifdef IN_RING3
+/**
+ * Enters a PDM read/write critical section with exclusive (write) access.
+ *
+ * @returns VINF_SUCCESS if entered successfully.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   fCallRing3  Whether this is a VMMRZCallRing3()request.
+ */
+VMMR3DECL(int) PDMR3CritSectRwEnterExclEx(PVM pVM, PPDMCRITSECTRW pThis, bool fCallRing3)
+{
+    return pdmCritSectRwEnterExcl(pVM, pThis, VERR_SEM_BUSY, false /*fTryAgain*/, NULL, fCallRing3 /*fNoVal*/);
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Leave a critical section held exclusively.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   fNoVal      No validation records (i.e. queued release).
+ * @sa      PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl.
+ */
+static int pdmCritSectRwLeaveExclWorker(PVMCC pVM, PPDMCRITSECTRW pThis, bool fNoVal)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, VERR_SEM_DESTROYED);
+
+#if !defined(PDMCRITSECTRW_STRICT) || !defined(IN_RING3)
+    NOREF(fNoVal);
+#endif
+
+    /*
+     * Check ownership.
+     */
+    RTNATIVETHREAD hNativeSelf = pdmCritSectRwGetNativeSelf(pVM, pThis);
+    AssertReturn(hNativeSelf != NIL_RTNATIVETHREAD, VERR_VM_THREAD_NOT_EMT);
+
+    RTNATIVETHREAD hNativeWriter;
+    ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
+    AssertReturn(hNativeSelf == hNativeWriter, VERR_NOT_OWNER);
+
+
+    /*
+     * Unwind one recursion. Not the last?
+     */
+    if (pThis->s.Core.cWriteRecursions != 1)
+    {
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+        if (fNoVal)
+            Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD);
+        else
+        {
+            int rc9 = RTLockValidatorRecExclUnwind(pThis->s.Core.pValidatorWrite);
+            if (RT_FAILURE(rc9))
+                return rc9;
+        }
+#endif
+#ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
+        uint32_t const cDepth = --pThis->s.Core.cWriteRecursions;
+#else
+        uint32_t const cDepth = ASMAtomicDecU32(&pThis->s.Core.cWriteRecursions);
+#endif
+        AssertReturn(cDepth != 0 && cDepth < UINT32_MAX, pdmCritSectRwCorrupted(pThis, "Invalid write recursion value on leave"));
+        return VINF_SUCCESS;
+    }
+
+
+    /*
+     * Final recursion.
+     */
+    AssertReturn(pThis->s.Core.cWriterReads == 0, VERR_WRONG_ORDER); /* (must release all read recursions before the final write.) */
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    if (fNoVal)
+        Assert(pThis->s.Core.pValidatorWrite->hThread == NIL_RTTHREAD);
+    else
+    {
+        int rc9 = RTLockValidatorRecExclReleaseOwner(pThis->s.Core.pValidatorWrite, true);
+        if (RT_FAILURE(rc9))
+            return rc9;
+    }
+#endif
+
+
+#ifdef RTASM_HAVE_CMP_WRITE_U128
+    /*
+     * See if we can get out w/o any signalling as this is a common case.
+     */
+    if (pdmCritSectRwIsCmpWriteU128Supported())
+    {
+        RTCRITSECTRWSTATE OldState;
+        OldState.s.u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+        if (OldState.s.u64State == ((UINT64_C(1) << RTCSRW_CNT_WR_SHIFT) | (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT)))
+        {
+            OldState.s.hNativeWriter = hNativeSelf;
+            AssertCompile(sizeof(OldState.s.hNativeWriter) == sizeof(OldState.u128.s.Lo));
+
+            RTCRITSECTRWSTATE NewState;
+            NewState.s.u64State      = RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT;
+            NewState.s.hNativeWriter = NIL_RTNATIVETHREAD;
+
+# ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
+            pThis->s.Core.cWriteRecursions = 0;
+# else
+            ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0);
+# endif
+            STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl);
+
+            if (ASMAtomicCmpWriteU128U(&pThis->s.Core.u.u128, NewState.u128, OldState.u128))
+                return VINF_SUCCESS;
+
+            /* bail out. */
+            pThis->s.Core.cWriteRecursions = 1;
+        }
+    }
+#endif /* RTASM_HAVE_CMP_WRITE_U128 */
+
+
+#if defined(IN_RING3) || defined(IN_RING0)
+    /*
+     * Ring-3: Straight forward, just update the state and if necessary signal waiters.
+     * Ring-0: Try leave for real, depends on host and context.
+     */
+# ifdef IN_RING0
+    Assert(RTSemEventIsSignalSafe() == RTSemEventMultiIsSignalSafe());
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    if (   pVCpu == NULL /* non-EMT access, if we implement it must be able to block */
+        || VMMRZCallRing3IsEnabled(pVCpu)
+        || RTSemEventIsSignalSafe()
+        || (   VMMR0ThreadCtxHookIsEnabled(pVCpu)       /* Doesn't matter if Signal() blocks if we have hooks, ... */
+            && RTThreadPreemptIsEnabled(NIL_RTTHREAD)   /* ... and preemption is still enabled, */
+            && ASMIntAreEnabled())                      /* ... and interrupts hasn't yet been disabled. Special pre-GC HM env. */
+       )
+# endif
+    {
+# ifdef PDMCRITSECTRW_WITH_LESS_ATOMIC_STUFF
+        pThis->s.Core.cWriteRecursions = 0;
+# else
+        ASMAtomicWriteU32(&pThis->s.Core.cWriteRecursions, 0);
+# endif
+        STAM_PROFILE_ADV_STOP(&pThis->s.StatWriteLocked, swl);
+        ASMAtomicWriteHandle(&pThis->s.Core.u.s.hNativeWriter, NIL_RTNATIVETHREAD);
+
+        for (;;)
+        {
+            uint64_t u64State    = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+            uint64_t u64OldState = u64State;
+
+            uint64_t c = (u64State & RTCSRW_CNT_WR_MASK) >> RTCSRW_CNT_WR_SHIFT;
+            AssertReturn(c > 0, pdmCritSectRwCorrupted(pThis, "Invalid write count on leave"));
+            c--;
+
+            if (   c > 0
+                || (u64State & RTCSRW_CNT_RD_MASK) == 0)
+            {
+                /*
+                 * Don't change the direction, wake up the next writer if any.
+                 */
+                u64State &= ~RTCSRW_CNT_WR_MASK;
+                u64State |= c << RTCSRW_CNT_WR_SHIFT;
+                if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                {
+                    STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,LeaveExcl));
+                    int rc;
+                    if (c == 0)
+                        rc = VINF_SUCCESS;
+# ifdef IN_RING0
+                    else if (!RTSemEventIsSignalSafe() && pVCpu != NULL)
+                    {
+                        VMMR0EMTBLOCKCTX Ctx;
+                        rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx);
+                        VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
+
+                        rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
+
+                        VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
+                    }
+# endif
+                    else
+                        rc = SUPSemEventSignal(pVM->pSession, (SUPSEMEVENT)pThis->s.Core.hEvtWrite);
+                    AssertRC(rc);
+                    return rc;
+                }
+            }
+            else
+            {
+                /*
+                 * Reverse the direction and signal the reader threads.
+                 */
+                u64State &= ~(RTCSRW_CNT_WR_MASK | RTCSRW_DIR_MASK);
+                u64State |= RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT;
+                if (ASMAtomicCmpXchgU64(&pThis->s.Core.u.s.u64State, u64State, u64OldState))
+                {
+                    Assert(!pThis->s.Core.fNeedReset);
+                    ASMAtomicWriteBool(&pThis->s.Core.fNeedReset, true);
+                    STAM_REL_COUNTER_INC(&pThis->s.CTX_MID_Z(StatContention,LeaveExcl));
+
+                    int rc;
+# ifdef IN_RING0
+                    if (!RTSemEventMultiIsSignalSafe() && pVCpu != NULL)
+                    {
+                        VMMR0EMTBLOCKCTX Ctx;
+                        rc = VMMR0EmtPrepareToBlock(pVCpu, VINF_SUCCESS, __FUNCTION__, pThis, &Ctx);
+                        VMM_ASSERT_RELEASE_MSG_RETURN(pVM, RT_SUCCESS(rc), ("rc=%Rrc\n", rc), rc);
+
+                        rc = SUPSemEventMultiSignal(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
+
+                        VMMR0EmtResumeAfterBlocking(pVCpu, &Ctx);
+                    }
+                    else
+# endif
+                        rc = SUPSemEventMultiSignal(pVM->pSession, (SUPSEMEVENTMULTI)pThis->s.Core.hEvtRead);
+                    AssertRC(rc);
+                    return rc;
+                }
+            }
+
+            ASMNopPause();
+            if (pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC)
+            { /*likely*/ }
+            else
+                return VERR_SEM_DESTROYED;
+            ASMNopPause();
+        }
+        /* not reached! */
+    }
+#endif /* IN_RING3 || IN_RING0 */
+
+
+#ifndef IN_RING3
+    /*
+     * Queue the requested exit for ring-3 execution.
+     */
+# ifndef IN_RING0
+    PVMCPUCC    pVCpu = VMMGetCpu(pVM); AssertPtr(pVCpu);
+# endif
+    uint32_t    i     = pVCpu->pdm.s.cQueuedCritSectRwExclLeaves++;
+    LogFlow(("PDMCritSectRwLeaveShared: [%d]=%p => R3\n", i, pThis));
+    VMM_ASSERT_RELEASE_MSG_RETURN(pVM, i < RT_ELEMENTS(pVCpu->pdm.s.apQueuedCritSectRwExclLeaves),
+                                  ("i=%u\n", i), VERR_PDM_CRITSECTRW_IPE);
+    pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] = pThis->s.pSelfR3;
+    VMM_ASSERT_RELEASE_MSG_RETURN(pVM,
+                                     RT_VALID_PTR(pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i])
+                                  &&    ((uintptr_t)pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i] & HOST_PAGE_OFFSET_MASK)
+                                     == ((uintptr_t)pThis & HOST_PAGE_OFFSET_MASK),
+                                  ("%p vs %p\n", pVCpu->pdm.s.apQueuedCritSectRwExclLeaves[i], pThis),
+                                  pdmCritSectRwCorrupted(pThis, "Invalid self pointer on queue (excl)"));
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PDM_CRITSECT);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
+    STAM_REL_COUNTER_INC(&pVM->pdm.s.StatQueuedCritSectLeaves);
+    STAM_REL_COUNTER_INC(&pThis->s.StatContentionRZLeaveExcl);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Leave a critical section held exclusively.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_SEM_DESTROYED if the critical section is delete before or
+ *          during the operation.
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwLeaveShared, RTCritSectRwLeaveExcl.
+ */
+VMMDECL(int) PDMCritSectRwLeaveExcl(PVMCC pVM, PPDMCRITSECTRW pThis)
+{
+    return pdmCritSectRwLeaveExclWorker(pVM, pThis, false /*fNoVal*/);
+}
+
+
+#if defined(IN_RING3) || defined(IN_RING0)
+/**
+ * PDMCritSectBothFF interface.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ */
+void pdmCritSectRwLeaveExclQueued(PVMCC pVM, PPDMCRITSECTRW pThis)
+{
+    pdmCritSectRwLeaveExclWorker(pVM, pThis, true /*fNoVal*/);
+}
+#endif
+
+
+/**
+ * Checks the caller is the exclusive (write) owner of the critical section.
+ *
+ * @retval  true if owner.
+ * @retval  false if not owner.
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwIsReadOwner, PDMCritSectIsOwner,
+ *          RTCritSectRwIsWriteOwner.
+ */
+VMMDECL(bool) PDMCritSectRwIsWriteOwner(PVMCC pVM, PPDMCRITSECTRW pThis)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false);
+
+    /*
+     * Check ownership.
+     */
+    RTNATIVETHREAD hNativeWriter;
+    ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hNativeWriter);
+    if (hNativeWriter == NIL_RTNATIVETHREAD)
+        return false;
+    return hNativeWriter == pdmCritSectRwGetNativeSelf(pVM, pThis);
+}
+
+
+/**
+ * Checks if the caller is one of the read owners of the critical section.
+ *
+ * @note    !CAUTION!  This API doesn't work reliably if lock validation isn't
+ *          enabled. Meaning, the answer is not trustworhty unless
+ *          RT_LOCK_STRICT or PDMCRITSECTRW_STRICT was defined at build time.
+ *          Also, make sure you do not use RTCRITSECTRW_FLAGS_NO_LOCK_VAL when
+ *          creating the semaphore.  And finally, if you used a locking class,
+ *          don't disable deadlock detection by setting cMsMinDeadlock to
+ *          RT_INDEFINITE_WAIT.
+ *
+ *          In short, only use this for assertions.
+ *
+ * @returns @c true if reader, @c false if not.
+ * @param   pVM         The cross context VM structure.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @param   fWannaHear  What you'd like to hear when lock validation is not
+ *                      available.  (For avoiding asserting all over the place.)
+ * @sa      PDMCritSectRwIsWriteOwner, RTCritSectRwIsReadOwner.
+ */
+VMMDECL(bool) PDMCritSectRwIsReadOwner(PVMCC pVM, PPDMCRITSECTRW pThis, bool fWannaHear)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, false);
+
+    /*
+     * Inspect the state.
+     */
+    uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+    if ((u64State & RTCSRW_DIR_MASK) == (RTCSRW_DIR_WRITE << RTCSRW_DIR_SHIFT))
+    {
+        /*
+         * It's in write mode, so we can only be a reader if we're also the
+         * current writer.
+         */
+        RTNATIVETHREAD hWriter;
+        ASMAtomicUoReadHandle(&pThis->s.Core.u.s.hNativeWriter, &hWriter);
+        if (hWriter == NIL_RTNATIVETHREAD)
+            return false;
+        return hWriter == pdmCritSectRwGetNativeSelf(pVM, pThis);
+    }
+
+    /*
+     * Read mode.  If there are no current readers, then we cannot be a reader.
+     */
+    if (!(u64State & RTCSRW_CNT_RD_MASK))
+        return false;
+
+#if defined(PDMCRITSECTRW_STRICT) && defined(IN_RING3)
+    /*
+     * Ask the lock validator.
+     * Note! It doesn't know everything, let's deal with that if it becomes an issue...
+     */
+    NOREF(fWannaHear);
+    return RTLockValidatorRecSharedIsOwner(pThis->s.Core.pValidatorRead, NIL_RTTHREAD);
+#else
+    /*
+     * Ok, we don't know, just tell the caller what he want to hear.
+     */
+    return fWannaHear;
+#endif
+}
+
+
+/**
+ * Gets the write recursion count.
+ *
+ * @returns The write recursion count (0 if bad critsect).
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwGetWriterReadRecursion, PDMCritSectRwGetReadCount,
+ *          RTCritSectRwGetWriteRecursion.
+ */
+VMMDECL(uint32_t) PDMCritSectRwGetWriteRecursion(PPDMCRITSECTRW pThis)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
+
+    /*
+     * Return the requested data.
+     */
+    return pThis->s.Core.cWriteRecursions;
+}
+
+
+/**
+ * Gets the read recursion count of the current writer.
+ *
+ * @returns The read recursion count (0 if bad critsect).
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetReadCount,
+ *          RTCritSectRwGetWriterReadRecursion.
+ */
+VMMDECL(uint32_t) PDMCritSectRwGetWriterReadRecursion(PPDMCRITSECTRW pThis)
+{
+    /*
+     * Validate handle.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
+
+    /*
+     * Return the requested data.
+     */
+    return pThis->s.Core.cWriterReads;
+}
+
+
+/**
+ * Gets the current number of reads.
+ *
+ * This includes all read recursions, so it might be higher than the number of
+ * read owners.  It does not include reads done by the current writer.
+ *
+ * @returns The read count (0 if bad critsect).
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectRwGetWriteRecursion, PDMCritSectRwGetWriterReadRecursion,
+ *          RTCritSectRwGetReadCount.
+ */
+VMMDECL(uint32_t) PDMCritSectRwGetReadCount(PPDMCRITSECTRW pThis)
+{
+    /*
+     * Validate input.
+     */
+    AssertPtr(pThis);
+    AssertReturn(pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC, 0);
+
+    /*
+     * Return the requested data.
+     */
+    uint64_t u64State = PDMCRITSECTRW_READ_STATE(&pThis->s.Core.u.s.u64State);
+    if ((u64State & RTCSRW_DIR_MASK) != (RTCSRW_DIR_READ << RTCSRW_DIR_SHIFT))
+        return 0;
+    return (u64State & RTCSRW_CNT_RD_MASK) >> RTCSRW_CNT_RD_SHIFT;
+}
+
+
+/**
+ * Checks if the read/write critical section is initialized or not.
+ *
+ * @retval  true if initialized.
+ * @retval  false if not initialized.
+ * @param   pThis       Pointer to the read/write critical section.
+ * @sa      PDMCritSectIsInitialized, RTCritSectRwIsInitialized.
+ */
+VMMDECL(bool) PDMCritSectRwIsInitialized(PCPDMCRITSECTRW pThis)
+{
+    AssertPtr(pThis);
+    return pThis->s.Core.u32Magic == RTCRITSECTRW_MAGIC;
+}
+
diff --git a/src/VBox/VMM/VMMAll/PDMAllIommu.cpp b/src/VBox/VMM/VMMAll/PDMAllIommu.cpp
new file mode 100644
index 00000000..185faa65
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllIommu.cpp
@@ -0,0 +1,468 @@
+/* $Id: PDMAllIommu.cpp $ */
+/** @file
+ * PDM IOMMU - All Contexts.
+ */
+
+/*
+ * Copyright (C) 2021-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM
+#define PDMPCIDEV_INCLUDE_PRIVATE  /* Hack to get pdmpcidevint.h included at the right point. */
+#include "PDMInternal.h"
+
+#include <VBox/vmm/vmcc.h>
+#include <iprt/string.h>
+#ifdef IN_RING3
+# include <iprt/mem.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/**
+ * Gets the PDM IOMMU for the current context from the PDM device instance.
+ */
+#ifdef IN_RING0
+#define PDMDEVINS_TO_IOMMU(a_pDevIns)   &(a_pDevIns)->Internal.s.pGVM->pdmr0.s.aIommus[0];
+#else
+#define PDMDEVINS_TO_IOMMU(a_pDevIns)   &(a_pDevIns)->Internal.s.pVMR3->pdm.s.aIommus[0];
+#endif
+
+
+/**
+ * Gets the PCI device ID (Bus:Dev:Fn) for the given PCI device.
+ *
+ * @returns PCI device ID.
+ * @param   pDevIns     The device instance.
+ * @param   pPciDev     The PCI device structure. Cannot be NULL.
+ */
+DECL_FORCE_INLINE(uint16_t) pdmIommuGetPciDeviceId(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev)
+{
+    uint8_t const idxBus = pPciDev->Int.s.idxPdmBus;
+#if defined(IN_RING0)
+    PGVM pGVM = pDevIns->Internal.s.pGVM;
+    Assert(idxBus < RT_ELEMENTS(pGVM->pdmr0.s.aPciBuses));
+    PCPDMPCIBUSR0 pBus = &pGVM->pdmr0.s.aPciBuses[idxBus];
+#elif defined(IN_RING3)
+    PVM pVM = pDevIns->Internal.s.pVMR3;
+    Assert(idxBus < RT_ELEMENTS(pVM->pdm.s.aPciBuses));
+    PCPDMPCIBUS pBus = &pVM->pdm.s.aPciBuses[idxBus];
+#endif
+    return PCIBDF_MAKE(pBus->iBus, pPciDev->uDevFn);
+}
+
+
+/**
+ * Returns whether an IOMMU instance is present.
+ *
+ * @returns @c true if an IOMMU is present, @c false otherwise.
+ * @param   pDevIns     The device instance.
+ */
+bool pdmIommuIsPresent(PPDMDEVINS pDevIns)
+{
+#ifdef IN_RING0
+    PCPDMIOMMUR3 pIommuR3 = &pDevIns->Internal.s.pGVM->pdm.s.aIommus[0];
+#else
+    PCPDMIOMMUR3 pIommuR3 = &pDevIns->Internal.s.pVMR3->pdm.s.aIommus[0];
+#endif
+    return pIommuR3->pDevInsR3 != NULL;
+}
+
+
+/** @copydoc PDMIOMMUREGR3::pfnMsiRemap */
+int pdmIommuMsiRemap(PPDMDEVINS pDevIns, uint16_t idDevice, PCMSIMSG pMsiIn, PMSIMSG pMsiOut)
+{
+    PPDMIOMMU  pIommu       = PDMDEVINS_TO_IOMMU(pDevIns);
+    PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns);
+    Assert(pDevInsIommu);
+    if (pDevInsIommu != pDevIns)
+        return pIommu->pfnMsiRemap(pDevInsIommu, idDevice, pMsiIn, pMsiOut);
+    return VERR_IOMMU_CANNOT_CALL_SELF;
+}
+
+
+/**
+ * Bus master physical memory read after translating the physical address using the
+ * IOMMU.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_IOMMU_NOT_PRESENT if an IOMMU is not present.
+ *
+ * @param   pDevIns     The device instance.
+ * @param   pPciDev     The PCI device. Cannot be NULL.
+ * @param   GCPhys      The guest-physical address to read.
+ * @param   pvBuf       Where to put the data read.
+ * @param   cbRead      How many bytes to read.
+ * @param   fFlags      Combination of PDM_DEVHLP_PHYS_RW_F_XXX.
+ *
+ * @thread  Any.
+ */
+int pdmIommuMemAccessRead(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, uint32_t fFlags)
+{
+    PPDMIOMMU  pIommu       = PDMDEVINS_TO_IOMMU(pDevIns);
+    PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns);
+    if (pDevInsIommu)
+    {
+        if (pDevInsIommu != pDevIns)
+        { /* likely */ }
+        else
+            return VERR_IOMMU_CANNOT_CALL_SELF;
+
+        uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev);
+        int rc = VINF_SUCCESS;
+        while (cbRead > 0)
+        {
+            RTGCPHYS GCPhysOut;
+            size_t   cbContig;
+            rc = pIommu->pfnMemAccess(pDevInsIommu, idDevice, GCPhys, cbRead, PDMIOMMU_MEM_F_READ, &GCPhysOut, &cbContig);
+            if (RT_SUCCESS(rc))
+            {
+                Assert(cbContig > 0 && cbContig <= cbRead);
+                /** @todo Handle strict return codes from PGMPhysRead. */
+                rc = pDevIns->CTX_SUFF(pHlp)->pfnPhysRead(pDevIns, GCPhysOut, pvBuf, cbContig, fFlags);
+                if (RT_SUCCESS(rc))
+                {
+                    cbRead -= cbContig;
+                    pvBuf   = (void *)((uintptr_t)pvBuf + cbContig);
+                    GCPhys += cbContig;
+                }
+                else
+                    break;
+            }
+            else
+            {
+                LogFunc(("IOMMU memory read failed. idDevice=%#x GCPhys=%#RGp cb=%zu rc=%Rrc\n", idDevice, GCPhys, cbRead, rc));
+
+                /*
+                 * We should initialize the read buffer on failure for devices that don't check
+                 * return codes (but would verify the data). But we still want to propagate the
+                 * error code from the IOMMU to the device, see @bugref{9936#c3}.
+                 */
+                memset(pvBuf, 0xff, cbRead);
+                break;
+            }
+        }
+        return rc;
+    }
+    return VERR_IOMMU_NOT_PRESENT;
+}
+
+
+/**
+ * Bus master physical memory write after translating the physical address using the
+ * IOMMU.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_IOMMU_NOT_PRESENT if an IOMMU is not present.
+ *
+ * @param   pDevIns     The device instance.
+ * @param   pPciDev     The PCI device structure. Cannot be NULL.
+ * @param   GCPhys      The guest-physical address to write.
+ * @param   pvBuf       The data to write.
+ * @param   cbWrite     How many bytes to write.
+ * @param   fFlags      Combination of PDM_DEVHLP_PHYS_RW_F_XXX.
+ *
+ * @thread  Any.
+ */
+int pdmIommuMemAccessWrite(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite,
+                           uint32_t fFlags)
+{
+    PPDMIOMMU  pIommu       = PDMDEVINS_TO_IOMMU(pDevIns);
+    PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns);
+    if (pDevInsIommu)
+    {
+        if (pDevInsIommu != pDevIns)
+        { /* likely */ }
+        else
+            return VERR_IOMMU_CANNOT_CALL_SELF;
+
+        uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev);
+        int rc = VINF_SUCCESS;
+        while (cbWrite > 0)
+        {
+            RTGCPHYS GCPhysOut;
+            size_t   cbContig;
+            rc = pIommu->pfnMemAccess(pDevInsIommu, idDevice, GCPhys, cbWrite, PDMIOMMU_MEM_F_WRITE, &GCPhysOut, &cbContig);
+            if (RT_SUCCESS(rc))
+            {
+                Assert(cbContig > 0 && cbContig <= cbWrite);
+                /** @todo Handle strict return codes from PGMPhysWrite. */
+                rc = pDevIns->CTX_SUFF(pHlp)->pfnPhysWrite(pDevIns, GCPhysOut, pvBuf, cbContig, fFlags);
+                if (RT_SUCCESS(rc))
+                {
+                    cbWrite -= cbContig;
+                    pvBuf    = (const void *)((uintptr_t)pvBuf + cbContig);
+                    GCPhys  += cbContig;
+                }
+                else
+                    break;
+            }
+            else
+            {
+                LogFunc(("IOMMU memory write failed. idDevice=%#x GCPhys=%#RGp cb=%zu rc=%Rrc\n", idDevice, GCPhys, cbWrite, rc));
+                break;
+            }
+        }
+        return rc;
+    }
+    return VERR_IOMMU_NOT_PRESENT;
+}
+
+
+#ifdef IN_RING3
+/**
+ * Requests the mapping of a guest page into ring-3 in preparation for a bus master
+ * physical memory read operation.
+ *
+ * Refer pfnPhysGCPhys2CCPtrReadOnly() for further details.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_IOMMU_NOT_PRESENT if an IOMMU is not present.
+ *
+ * @param   pDevIns     The device instance.
+ * @param   pPciDev     The PCI device structure. Cannot be NULL.
+ * @param   GCPhys      The guest physical address of the page that should be
+ *                      mapped.
+ * @param   fFlags      Flags reserved for future use, MBZ.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that
+ *                      pfnPhysReleasePageMappingLock needs.
+ */
+int pdmR3IommuMemAccessReadCCPtr(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys, uint32_t fFlags, void const **ppv,
+                                 PPGMPAGEMAPLOCK pLock)
+{
+    PPDMIOMMU  pIommu       = PDMDEVINS_TO_IOMMU(pDevIns);
+    PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns);
+    if (pDevInsIommu)
+    {
+        if (pDevInsIommu != pDevIns)
+        { /* likely */ }
+        else
+            return VERR_IOMMU_CANNOT_CALL_SELF;
+
+        uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev);
+        size_t   cbContig  = 0;
+        RTGCPHYS GCPhysOut = NIL_RTGCPHYS;
+        int rc = pIommu->pfnMemAccess(pDevInsIommu, idDevice, GCPhys & X86_PAGE_BASE_MASK, X86_PAGE_SIZE, PDMIOMMU_MEM_F_READ,
+                                      &GCPhysOut, &cbContig);
+        if (RT_SUCCESS(rc))
+        {
+            Assert(GCPhysOut != NIL_RTGCPHYS);
+            Assert(cbContig == X86_PAGE_SIZE);
+            return pDevIns->pHlpR3->pfnPhysGCPhys2CCPtrReadOnly(pDevIns, GCPhysOut, fFlags, ppv, pLock);
+        }
+
+        LogFunc(("IOMMU memory read for pointer access failed. idDevice=%#x GCPhys=%#RGp rc=%Rrc\n", idDevice, GCPhys, rc));
+        return rc;
+    }
+    return VERR_IOMMU_NOT_PRESENT;
+}
+
+
+/**
+ * Requests the mapping of a guest page into ring-3 in preparation for a bus master
+ * physical memory write operation.
+ *
+ * Refer pfnPhysGCPhys2CCPtr() for further details.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_IOMMU_NOT_PRESENT if an IOMMU is not present.
+ *
+ * @param   pDevIns     The device instance.
+ * @param   pPciDev     The PCI device structure. Cannot be NULL.
+ * @param   GCPhys      The guest physical address of the page that should be
+ *                      mapped.
+ * @param   fFlags      Flags reserved for future use, MBZ.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that
+ *                      pfnPhysReleasePageMappingLock needs.
+ */
+int pdmR3IommuMemAccessWriteCCPtr(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, RTGCPHYS GCPhys, uint32_t fFlags, void **ppv,
+                                  PPGMPAGEMAPLOCK pLock)
+{
+    PPDMIOMMU  pIommu       = PDMDEVINS_TO_IOMMU(pDevIns);
+    PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns);
+    if (pDevInsIommu)
+    {
+        if (pDevInsIommu != pDevIns)
+        { /* likely */ }
+        else
+            return VERR_IOMMU_CANNOT_CALL_SELF;
+
+        uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev);
+        size_t   cbContig  = 0;
+        RTGCPHYS GCPhysOut = NIL_RTGCPHYS;
+        int rc = pIommu->pfnMemAccess(pDevInsIommu, idDevice, GCPhys & X86_PAGE_BASE_MASK, X86_PAGE_SIZE, PDMIOMMU_MEM_F_WRITE,
+                                      &GCPhysOut, &cbContig);
+        if (RT_SUCCESS(rc))
+        {
+            Assert(GCPhysOut != NIL_RTGCPHYS);
+            Assert(cbContig == X86_PAGE_SIZE);
+            return pDevIns->pHlpR3->pfnPhysGCPhys2CCPtr(pDevIns, GCPhysOut, fFlags, ppv, pLock);
+        }
+
+        LogFunc(("IOMMU memory write for pointer access failed. idDevice=%#x GCPhys=%#RGp rc=%Rrc\n", idDevice, GCPhys, rc));
+        return rc;
+    }
+    return VERR_IOMMU_NOT_PRESENT;
+}
+
+
+/**
+ * Requests the mapping of multiple guest pages into ring-3 in prepartion for a bus
+ * master physical memory read operation.
+ *
+ * Refer pfnPhysBulkGCPhys2CCPtrReadOnly() for further details.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_IOMMU_NOT_PRESENT if an IOMMU is not present.
+ *
+ * @param   pDevIns             The device instance.
+ * @param   pPciDev             The PCI device structure. Cannot be NULL.
+ * @param   cPages              Number of pages to lock.
+ * @param   paGCPhysPages       The guest physical address of the pages that
+ *                              should be mapped (@a cPages entries).
+ * @param   fFlags              Flags reserved for future use, MBZ.
+ * @param   papvPages           Where to store the ring-3 mapping addresses
+ *                              corresponding to @a paGCPhysPages.
+ * @param   paLocks             Where to store the locking information that
+ *                              pfnPhysBulkReleasePageMappingLock needs (@a cPages
+ *                              in length).
+ */
+int pdmR3IommuMemAccessBulkReadCCPtr(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
+                                     uint32_t fFlags, const void **papvPages, PPGMPAGEMAPLOCK paLocks)
+{
+    PPDMIOMMU  pIommu       = PDMDEVINS_TO_IOMMU(pDevIns);
+    PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns);
+    if (pDevInsIommu)
+    {
+        if (pDevInsIommu != pDevIns)
+        { /* likely */ }
+        else
+            return VERR_IOMMU_CANNOT_CALL_SELF;
+
+        /* Allocate space for translated addresses. */
+        size_t const cbIovas  = cPages * sizeof(uint64_t);
+        PRTGCPHYS paGCPhysOut = (PRTGCPHYS)RTMemAllocZ(cbIovas);
+        if (paGCPhysOut)
+        { /* likely */ }
+        else
+        {
+            LogFunc(("caller='%s'/%d: returns %Rrc - Failed to alloc %zu bytes for IOVA addresses\n",
+                     pDevIns->pReg->szName, pDevIns->iInstance, VERR_NO_MEMORY, cbIovas));
+            return VERR_NO_MEMORY;
+        }
+
+        /* Ask the IOMMU for corresponding translated physical addresses. */
+        uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev);
+        AssertCompile(sizeof(RTGCPHYS) == sizeof(uint64_t));
+        int rc = pIommu->pfnMemBulkAccess(pDevInsIommu, idDevice, cPages, (uint64_t const *)paGCPhysPages, PDMIOMMU_MEM_F_READ,
+                                          paGCPhysOut);
+        if (RT_SUCCESS(rc))
+        {
+            /* Perform the bulk mapping but with the translated addresses. */
+            rc = pDevIns->pHlpR3->pfnPhysBulkGCPhys2CCPtrReadOnly(pDevIns, cPages, paGCPhysOut, fFlags, papvPages, paLocks);
+            if (RT_FAILURE(rc))
+                LogFunc(("Bulk mapping for read access failed. cPages=%zu fFlags=%#x rc=%Rrc\n", rc, cPages, fFlags));
+        }
+        else
+            LogFunc(("Bulk translation for read access failed. idDevice=%#x cPages=%zu rc=%Rrc\n", idDevice, cPages, rc));
+
+        RTMemFree(paGCPhysOut);
+        return rc;
+    }
+    return VERR_IOMMU_NOT_PRESENT;
+}
+
+
+/**
+ * Requests the mapping of multiple guest pages into ring-3 in prepartion for a bus
+ * master physical memory write operation.
+ *
+ * Refer pfnPhysBulkGCPhys2CCPtr() for further details.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_IOMMU_NOT_PRESENT if an IOMMU is not present.
+ *
+ * @param   pDevIns             The device instance.
+ * @param   pPciDev             The PCI device structure. Cannot be NULL.
+ * @param   cPages              Number of pages to lock.
+ * @param   paGCPhysPages       The guest physical address of the pages that
+ *                              should be mapped (@a cPages entries).
+ * @param   fFlags              Flags reserved for future use, MBZ.
+ * @param   papvPages           Where to store the ring-3 mapping addresses
+ *                              corresponding to @a paGCPhysPages.
+ * @param   paLocks             Where to store the locking information that
+ *                              pfnPhysBulkReleasePageMappingLock needs (@a cPages
+ *                              in length).
+ */
+int pdmR3IommuMemAccessBulkWriteCCPtr(PPDMDEVINS pDevIns, PPDMPCIDEV pPciDev, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
+                                      uint32_t fFlags, void **papvPages, PPGMPAGEMAPLOCK paLocks)
+{
+    PPDMIOMMU  pIommu       = PDMDEVINS_TO_IOMMU(pDevIns);
+    PPDMDEVINS pDevInsIommu = pIommu->CTX_SUFF(pDevIns);
+    if (pDevInsIommu)
+    {
+        if (pDevInsIommu != pDevIns)
+        { /* likely */ }
+        else
+            return VERR_IOMMU_CANNOT_CALL_SELF;
+
+        /* Allocate space for translated addresses. */
+        size_t const cbIovas  = cPages * sizeof(uint64_t);
+        PRTGCPHYS paGCPhysOut = (PRTGCPHYS)RTMemAllocZ(cbIovas);
+        if (paGCPhysOut)
+        { /* likely */ }
+        else
+        {
+            LogFunc(("caller='%s'/%d: returns %Rrc - Failed to alloc %zu bytes for IOVA addresses\n",
+                     pDevIns->pReg->szName, pDevIns->iInstance, VERR_NO_MEMORY, cbIovas));
+            return VERR_NO_MEMORY;
+        }
+
+        /* Ask the IOMMU for corresponding translated physical addresses. */
+        uint16_t const idDevice = pdmIommuGetPciDeviceId(pDevIns, pPciDev);
+        AssertCompile(sizeof(RTGCPHYS) == sizeof(uint64_t));
+        int rc = pIommu->pfnMemBulkAccess(pDevInsIommu, idDevice, cPages, (uint64_t const *)paGCPhysPages, PDMIOMMU_MEM_F_WRITE,
+                                          paGCPhysOut);
+        if (RT_SUCCESS(rc))
+        {
+            /* Perform the bulk mapping but with the translated addresses. */
+            rc = pDevIns->pHlpR3->pfnPhysBulkGCPhys2CCPtr(pDevIns, cPages, paGCPhysOut, fFlags, papvPages, paLocks);
+            if (RT_FAILURE(rc))
+                LogFunc(("Bulk mapping of addresses failed. cPages=%zu fFlags=%#x rc=%Rrc\n", rc, cPages, fFlags));
+        }
+        else
+            LogFunc(("IOMMU bulk translation failed. idDevice=%#x cPages=%zu rc=%Rrc\n", idDevice, cPages, rc));
+
+        RTMemFree(paGCPhysOut);
+        return rc;
+    }
+    return VERR_IOMMU_NOT_PRESENT;
+}
+#endif /* IN_RING3 */
+
diff --git a/src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp b/src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp
new file mode 100644
index 00000000..d1b56dd5
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllNetShaper.cpp
@@ -0,0 +1,139 @@
+/* $Id: PDMAllNetShaper.cpp $ */
+/** @file
+ * PDM Network Shaper - Limit network traffic according to bandwidth group settings.
+ */
+
+/*
+ * Copyright (C) 2011-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_NET_SHAPER
+#include <VBox/vmm/pdmnetshaper.h>
+#include "PDMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/log.h>
+#include <iprt/time.h>
+#include <iprt/asm-math.h>
+
+
+/**
+ * Obtain bandwidth in a bandwidth group.
+ *
+ * @returns True if bandwidth was allocated, false if not.
+ * @param   pVM             The cross context VM structure.
+ * @param   pFilter         Pointer to the filter that allocates bandwidth.
+ * @param   cbTransfer      Number of bytes to allocate.
+ */
+VMM_INT_DECL(bool) PDMNetShaperAllocateBandwidth(PVMCC pVM, PPDMNSFILTER pFilter, size_t cbTransfer)
+{
+    AssertPtrReturn(pFilter, true);
+
+    /*
+     * If we haven't got a valid bandwidth group, we always allow the traffic.
+     */
+    bool fAllowed = true;
+    uint32_t iGroup = ASMAtomicUoReadU32(&pFilter->iGroup);
+    if (iGroup != 0)
+    {
+        if (iGroup <= RT_MIN(pVM->pdm.s.cNsGroups, RT_ELEMENTS(pVM->pdm.s.aNsGroups)))
+        {
+            PPDMNSBWGROUP pGroup = &pVM->pdm.s.aNsGroups[iGroup - 1];
+            int rc = PDMCritSectEnter(pVM, &pGroup->Lock, VINF_TRY_AGAIN);
+            if (rc == VINF_SUCCESS)
+            {
+                uint64_t const cbPerSecMax = pGroup->cbPerSecMax;
+                if (cbPerSecMax > 0)
+                {
+                    /*
+                     * Re-fill the bucket first
+                     *
+                     * Note! We limit the cTokensAdded calculation to 1 second, since it's really
+                     *       pointless to calculate much beyond PDM_NETSHAPER_MAX_LATENCY (100ms)
+                     *       let alone 1 sec.  This makes it possible to use ASMMultU64ByU32DivByU32
+                     *       as the cNsDelta is less than 30 bits wide now, which means we don't get
+                     *       into overflow issues when multiplying two 64-bit values.
+                     */
+                    uint64_t const nsNow        = RTTimeSystemNanoTS();
+                    uint64_t const cNsDelta     = nsNow - pGroup->tsUpdatedLast;
+                    uint64_t const cTokensAdded = cNsDelta < RT_NS_1SEC
+                                                ? ASMMultU64ByU32DivByU32(cbPerSecMax, (uint32_t)cNsDelta, RT_NS_1SEC)
+                                                : cbPerSecMax;
+                    uint32_t const cbBucket     = pGroup->cbBucket;
+                    uint32_t const cbTokensLast = pGroup->cbTokensLast;
+                    uint32_t const cTokens      = (uint32_t)RT_MIN(cbBucket, cTokensAdded + cbTokensLast);
+
+                    /*
+                     * Allowed?
+                     */
+                    if (cbTransfer <= cTokens)
+                    {
+                        pGroup->cbTokensLast  = cTokens - (uint32_t)cbTransfer;
+                        pGroup->tsUpdatedLast = nsNow;
+                        Log2(("pdmNsAllocateBandwidth/%s: allowed - cbTransfer=%#zx cTokens=%#x cTokensAdded=%#x\n",
+                              pGroup->szName, cbTransfer, cTokens, cTokensAdded));
+                    }
+                    else
+                    {
+                        /*
+                         * No, we're choked.  Arm the unchoke timer for the next period.
+                         * Just do this on a simple PDM_NETSHAPER_MAX_LATENCY clock granularity.
+                         * ASSUMES the timer uses millisecond resolution clock.
+                         */
+                        ASMAtomicWriteBool(&pFilter->fChoked, true);
+                        if (ASMAtomicCmpXchgBool(&pVM->pdm.s.fNsUnchokeTimerArmed, true, false))
+                        {
+                            Assert(TMTimerGetFreq(pVM, pVM->pdm.s.hNsUnchokeTimer) == RT_MS_1SEC);
+                            uint64_t const msNow    = TMTimerGet(pVM, pVM->pdm.s.hNsUnchokeTimer);
+                            uint64_t const msExpire = (msNow / PDM_NETSHAPER_MAX_LATENCY + 1) * PDM_NETSHAPER_MAX_LATENCY;
+                            rc = TMTimerSet(pVM, pVM->pdm.s.hNsUnchokeTimer, msExpire);
+                            AssertRC(rc);
+
+                            Log2(("pdmNsAllocateBandwidth/%s: refused - cbTransfer=%#zx cTokens=%#x cTokensAdded=%#x cMsExpire=%u\n",
+                                  pGroup->szName, cbTransfer, cTokens, cTokensAdded, msExpire - msNow));
+                        }
+                        else
+                            Log2(("pdmNsAllocateBandwidth/%s: refused - cbTransfer=%#zx cTokens=%#x cTokensAdded=%#x\n",
+                                  pGroup->szName, cbTransfer, cTokens, cTokensAdded));
+                        ASMAtomicIncU64(&pGroup->cTotalChokings);
+                        fAllowed = false;
+                    }
+                }
+                else
+                    Log2(("pdmNsAllocateBandwidth/%s: disabled\n", pGroup->szName));
+
+                rc = PDMCritSectLeave(pVM, &pGroup->Lock);
+                AssertRCSuccess(rc);
+            }
+            else if (rc == VINF_TRY_AGAIN) /* (accounted for by the critsect stats) */
+                Log2(("pdmNsAllocateBandwidth/%s: allowed - lock contention\n", pGroup->szName));
+            else
+                PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pGroup->Lock, rc);
+        }
+        else
+            AssertMsgFailed(("Invalid iGroup=%d\n", iGroup));
+    }
+    return fAllowed;
+}
diff --git a/src/VBox/VMM/VMMAll/PDMAllQueue.cpp b/src/VBox/VMM/VMMAll/PDMAllQueue.cpp
new file mode 100644
index 00000000..f64a198b
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllQueue.cpp
@@ -0,0 +1,313 @@
+/* $Id: PDMAllQueue.cpp $ */
+/** @file
+ * PDM Queue - Transport data and tasks to EMT and R3.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_QUEUE
+#include "PDMInternal.h"
+#include <VBox/vmm/pdm.h>
+#ifndef IN_RC
+# include <VBox/vmm/mm.h>
+#endif
+#include <VBox/vmm/vmcc.h>
+#include <iprt/errcore.h>
+#include <VBox/log.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/*
+ * Macros for thoroughly validating a queue handle and ownership.
+ */
+#define PDMQUEUE_HANDLE_TO_VARS_RETURN_COMMON(a_cbMax, a_cbTotalMax) \
+    AssertReturn(cbItem >= sizeof(PDMQUEUEITEMCORE), pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \
+    AssertReturn(cbItem <= (a_cbMax), pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \
+    \
+    /* paranoia^3: */ \
+    AssertReturn(cItems > 0, pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \
+    AssertReturn(cItems <= PDMQUEUE_MAX_ITEMS, pQueue->rcOkay = VERR_INTERNAL_ERROR_4); \
+    AssertReturn(cbItem * cItems <= (a_cbTotalMax), pQueue->rcOkay = VERR_INTERNAL_ERROR_4)
+
+#ifdef IN_RING0
+# define PDMQUEUE_HANDLE_TO_VARS_RETURN(a_pVM, a_hQueue, a_pvOwner) \
+    AssertPtrReturn((a_pvOwner), VERR_INVALID_PARAMETER); \
+    \
+    AssertCompile(RT_ELEMENTS((a_pVM)->pdm.s.apRing0Queues) == RT_ELEMENTS((a_pVM)->pdmr0.s.aQueues)); \
+    AssertReturn((a_hQueue) < RT_ELEMENTS((a_pVM)->pdmr0.s.aQueues), VERR_INVALID_HANDLE); \
+    AssertReturn((a_hQueue) < (a_pVM)->pdmr0.s.cQueues, VERR_INVALID_HANDLE); \
+    AssertReturn((a_pVM)->pdmr0.s.aQueues[(a_hQueue)].pvOwner == (a_pvOwner), VERR_INVALID_HANDLE); \
+    PPDMQUEUE pQueue = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].pQueue; \
+    AssertPtrReturn(pQueue, VERR_INVALID_HANDLE); \
+    AssertReturn(pQueue->u32Magic == PDMQUEUE_MAGIC, VERR_INVALID_HANDLE); \
+    AssertReturn(pQueue->rcOkay == VINF_SUCCESS, pQueue->rcOkay); \
+    \
+    uint32_t const cbItem   = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].cbItem; \
+    uint32_t const cItems   = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].cItems; \
+    uint32_t const offItems = (a_pVM)->pdmr0.s.aQueues[(a_hQueue)].offItems; \
+    \
+    /* paranoia^2: */ \
+    AssertReturn(pQueue->cbItem == cbItem, pQueue->rcOkay = VERR_INTERNAL_ERROR_3); \
+    AssertReturn(pQueue->cItems == cItems, pQueue->rcOkay = VERR_INTERNAL_ERROR_3); \
+    AssertReturn(pQueue->offItems == offItems, pQueue->rcOkay = VERR_INTERNAL_ERROR_3); \
+    \
+    PDMQUEUE_HANDLE_TO_VARS_RETURN_COMMON(PDMQUEUE_MAX_ITEM_SIZE, PDMQUEUE_MAX_TOTAL_SIZE_R0)
+
+#else
+# define PDMQUEUE_HANDLE_TO_VARS_RETURN(a_pVM, a_hQueue, a_pvOwner) \
+    AssertPtrReturn((a_pvOwner), VERR_INVALID_PARAMETER); \
+    \
+    PPDMQUEUE pQueue; \
+    if ((a_hQueue) < RT_ELEMENTS((a_pVM)->pdm.s.apRing0Queues)) \
+        pQueue = (a_pVM)->pdm.s.apRing0Queues[(a_hQueue)]; \
+    else \
+    { \
+        (a_hQueue) -= RT_ELEMENTS((a_pVM)->pdm.s.apRing0Queues); \
+        AssertReturn((a_pVM)->pdm.s.cRing3Queues, VERR_INVALID_HANDLE); \
+        pQueue = (a_pVM)->pdm.s.papRing3Queues[(a_hQueue)]; \
+    } \
+    AssertPtrReturn(pQueue, VERR_INVALID_HANDLE); \
+    AssertReturn(pQueue->u32Magic == PDMQUEUE_MAGIC, VERR_INVALID_HANDLE); \
+    AssertReturn(pQueue->u.Gen.pvOwner == (a_pvOwner), VERR_INVALID_HANDLE); \
+    AssertReturn(pQueue->rcOkay == VINF_SUCCESS, pQueue->rcOkay); \
+    \
+    uint32_t const cbItem   = pQueue->cbItem; \
+    uint32_t const cItems   = pQueue->cItems; \
+    uint32_t const offItems = pQueue->offItems; \
+    \
+    PDMQUEUE_HANDLE_TO_VARS_RETURN_COMMON(PDMQUEUE_MAX_ITEM_SIZE, PDMQUEUE_MAX_TOTAL_SIZE_R3)
+
+#endif
+
+
+/**
+ * Commmon function for initializing the shared queue structure.
+ */
+void pdmQueueInit(PPDMQUEUE pQueue, uint32_t cbBitmap, uint32_t cbItem, uint32_t cItems,
+                  const char *pszName, PDMQUEUETYPE enmType, RTR3PTR pfnCallback, RTR3PTR pvOwner)
+{
+    Assert(cbBitmap * 8 >= cItems);
+
+    pQueue->u32Magic            = PDMQUEUE_MAGIC;
+    pQueue->cbItem              = cbItem;
+    pQueue->cItems              = cItems;
+    pQueue->offItems            = RT_UOFFSETOF(PDMQUEUE, bmAlloc) + cbBitmap;
+    pQueue->rcOkay              = VINF_SUCCESS;
+    pQueue->u32Padding          = 0;
+    pQueue->hTimer              = NIL_TMTIMERHANDLE;
+    pQueue->cMilliesInterval    = 0;
+    pQueue->enmType             = enmType;
+    pQueue->u.Gen.pfnCallback   = pfnCallback;
+    pQueue->u.Gen.pvOwner       = pvOwner;
+    RTStrCopy(pQueue->szName, sizeof(pQueue->szName), pszName);
+    pQueue->iPending            = UINT32_MAX;
+    RT_BZERO(pQueue->bmAlloc, cbBitmap);
+    ASMBitSetRange(pQueue->bmAlloc, 0, cItems);
+
+    uint8_t *pbItem = (uint8_t *)&pQueue->bmAlloc[0] + cbBitmap;
+    while (cItems-- > 0)
+    {
+        ((PPDMQUEUEITEMCORE)pbItem)->u64View = UINT64_C(0xfeedfeedfeedfeed);
+
+        /* next */
+        pbItem += cbItem;
+    }
+}
+
+
+/**
+ * Allocate an item from a queue, extended version.
+ *
+ * The allocated item must be handed on to PDMR3QueueInsert() after the
+ * data have been filled in.
+ *
+ * @returns VBox status code.
+ * @param   pVM         Pointer to the cross context VM structure w/ ring-0.
+ * @param   hQueue      The queue handle.
+ * @param   pvOwner     The queue owner.
+ * @param   ppNew       Where to return the item pointer on success.
+ * @thread  Any thread.
+ */
+VMMDECL(int) PDMQueueAllocEx(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner, PPDMQUEUEITEMCORE *ppNew)
+{
+    /*
+     * Validate and translate input.
+     */
+    *ppNew = NULL;
+    PDMQUEUE_HANDLE_TO_VARS_RETURN(pVM, hQueue, pvOwner);
+
+    /*
+     * Do the allocation.
+     */
+    uint32_t cEmptyScans = 0;
+    for (;;)
+    {
+        int32_t iBit = ASMBitFirstSet(pQueue->bmAlloc, cItems);
+        if (iBit >= 0)
+        {
+            if (ASMAtomicBitTestAndClear(pQueue->bmAlloc, iBit))
+            {
+                PPDMQUEUEITEMCORE pNew = (PPDMQUEUEITEMCORE)&((uint8_t *)pQueue)[offItems + iBit * cbItem];
+                pNew->u64View = UINT64_C(0xbeefbeefbeefbeef);
+                *ppNew = pNew;
+                return VINF_SUCCESS;
+            }
+            cEmptyScans = 0;
+        }
+        else if (++cEmptyScans < 16)
+            ASMNopPause();
+        else
+        {
+            STAM_REL_COUNTER_INC(&pQueue->StatAllocFailures);
+            return VERR_OUT_OF_RESOURCES;
+        }
+    }
+}
+
+
+/**
+ * Allocate an item from a queue.
+ *
+ * The allocated item must be handed on to PDMR3QueueInsert() after the
+ * data have been filled in.
+ *
+ * @returns Pointer to the new item on success, NULL on failure.
+ * @param   pVM         Pointer to the cross context VM structure w/ ring-0.
+ * @param   hQueue      The queue handle.
+ * @param   pvOwner     The queue owner.
+ * @thread  Any thread.
+ */
+VMMDECL(PPDMQUEUEITEMCORE) PDMQueueAlloc(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner)
+{
+    PPDMQUEUEITEMCORE pNew = NULL;
+    int rc = PDMQueueAllocEx(pVM, hQueue, pvOwner, &pNew);
+    if (RT_SUCCESS(rc))
+        return pNew;
+    return NULL;
+}
+
+
+/**
+ * Sets the FFs and fQueueFlushed.
+ *
+ * @param   pVM         Pointer to the cross context VM structure w/ ring-0.
+ */
+static void pdmQueueSetFF(PVMCC pVM)
+{
+    Log2(("PDMQueueInsert: VM_FF_PDM_QUEUES %d -> 1\n", VM_FF_IS_SET(pVM, VM_FF_PDM_QUEUES)));
+    VM_FF_SET(pVM, VM_FF_PDM_QUEUES);
+    ASMAtomicBitSet(&pVM->pdm.s.fQueueFlushing, PDM_QUEUE_FLUSH_FLAG_PENDING_BIT);
+#ifdef IN_RING3
+    VMR3NotifyGlobalFFU(pVM->pUVM, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+}
+
+
+/**
+ * Queue an item.
+ *
+ * The item must have been obtained using PDMQueueAlloc(). Once the item
+ * have been passed to this function it must not be touched!
+ *
+ * @returns VBox status code.
+ * @param   pVM         Pointer to the cross context VM structure w/ ring-0.
+ * @param   hQueue      The queue handle.
+ * @param   pvOwner     The queue owner.
+ * @param   pInsert     The item to insert.
+ * @thread  Any thread.
+ */
+VMMDECL(int) PDMQueueInsert(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner, PPDMQUEUEITEMCORE pInsert)
+{
+    /*
+     * Validate and translate input.
+     */
+    PDMQUEUE_HANDLE_TO_VARS_RETURN(pVM, hQueue, pvOwner);
+
+    uint8_t * const pbItems   = (uint8_t *)pQueue + offItems;
+    uintptr_t const offInsert = (uintptr_t)pInsert - (uintptr_t)pbItems;
+    uintptr_t const iInsert   = offInsert / cbItem;
+    AssertReturn(iInsert < cItems, VERR_INVALID_PARAMETER);
+    AssertReturn(iInsert * cbItem == offInsert, VERR_INVALID_PARAMETER);
+
+    AssertReturn(ASMBitTest(pQueue->bmAlloc, iInsert) == false, VERR_INVALID_PARAMETER);
+
+    /*
+     * Append the item to the pending list.
+     */
+    for (;;)
+    {
+        uint32_t const iOldPending = ASMAtomicUoReadU32(&pQueue->iPending);
+        pInsert->iNext = iOldPending;
+        if (ASMAtomicCmpXchgU32(&pQueue->iPending, iInsert, iOldPending))
+            break;
+        ASMNopPause();
+    }
+
+    if (pQueue->hTimer == NIL_TMTIMERHANDLE)
+        pdmQueueSetFF(pVM);
+    STAM_REL_COUNTER_INC(&pQueue->StatInsert);
+    STAM_STATS({ ASMAtomicIncU32(&pQueue->cStatPending); });
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Schedule the queue for flushing (processing) if necessary.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if a flush was necessary.
+ * @retval  VINF_NO_CHANGE if no flushing needed.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pvOwner     The alleged queue owner.
+ * @param   hQueue      The queueu to maybe flush.
+ */
+VMMDECL(int) PDMQueueFlushIfNecessary(PVMCC pVM, PDMQUEUEHANDLE hQueue, void *pvOwner)
+{
+    /*
+     * Validate input.
+     */
+    PDMQUEUE_HANDLE_TO_VARS_RETURN(pVM, hQueue, pvOwner);
+    RT_NOREF(offItems);
+
+    /*
+     * Check and maybe flush.
+     */
+    if (ASMAtomicUoReadU32(&pQueue->iPending) != UINT32_MAX)
+    {
+        pdmQueueSetFF(pVM);
+        return VINF_SUCCESS;
+    }
+    return VINF_NO_CHANGE;
+}
+
diff --git a/src/VBox/VMM/VMMAll/PDMAllTask.cpp b/src/VBox/VMM/VMMAll/PDMAllTask.cpp
new file mode 100644
index 00000000..72bce3d5
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PDMAllTask.cpp
@@ -0,0 +1,124 @@
+/* $Id: PDMAllTask.cpp $ */
+/** @file
+ * PDM Task - Asynchronous user mode tasks, all context code.
+ */
+
+/*
+ * Copyright (C) 2019-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PDM_TASK
+#include "PDMInternal.h"
+#include <VBox/vmm/pdmtask.h>
+#include <VBox/vmm/gvm.h>
+#include <VBox/err.h>
+
+#include <VBox/log.h>
+#include <VBox/sup.h>
+#include <iprt/asm.h>
+#include <iprt/assert.h>
+#include <iprt/semaphore.h>
+
+
+
+/**
+ * Triggers a task.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_ALREADY_POSTED is the task is already pending.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   enmType     The task owner type.
+ * @param   pvOwner     The task owner.
+ * @param   hTask       The task to trigger.
+ * @thread  Any
+ */
+VMM_INT_DECL(int)   PDMTaskTrigger(PVMCC pVM, PDMTASKTYPE enmType, RTR3PTR pvOwner, PDMTASKHANDLE hTask)
+{
+    /*
+     * Validate input and translate the handle to a task.
+     */
+    AssertReturn(pvOwner, VERR_NOT_OWNER);
+    AssertReturn(enmType >= PDMTASKTYPE_DEV && enmType <= PDMTASKTYPE_INTERNAL, VERR_NOT_OWNER);
+
+    size_t const iTask    = hTask % RT_ELEMENTS(pVM->pdm.s.aTaskSets[0].aTasks);
+    size_t const iTaskSet = hTask / RT_ELEMENTS(pVM->pdm.s.aTaskSets[0].aTasks);
+#ifdef IN_RING3
+    AssertReturn(iTaskSet < RT_ELEMENTS(pVM->pdm.s.apTaskSets), VERR_INVALID_HANDLE);
+    PPDMTASKSET pTaskSet  = pVM->pdm.s.apTaskSets[iTaskSet];
+    AssertReturn(pTaskSet, VERR_INVALID_HANDLE);
+#else
+    AssertReturn(iTaskSet < RT_ELEMENTS(pVM->pdm.s.aTaskSets),
+                 iTaskSet < RT_ELEMENTS(pVM->pdm.s.apTaskSets) ? VERR_INVALID_CONTEXT : VERR_INVALID_HANDLE);
+    PPDMTASKSET pTaskSet  = &pVM->pdm.s.aTaskSets[iTaskSet];
+#endif
+    AssertReturn(pTaskSet->u32Magic == PDMTASKSET_MAGIC, VERR_INVALID_MAGIC);
+    PPDMTASK pTask = &pTaskSet->aTasks[iTask];
+
+    /*
+     * Check task ownership.
+     */
+    AssertReturn(pvOwner == pTask->pvOwner, VERR_NOT_OWNER);
+    AssertReturn(enmType == pTask->enmType, VERR_NOT_OWNER);
+
+    /*
+     * Trigger the task, wake up the thread if the task isn't triggered yet.
+     */
+    if (!ASMAtomicBitTestAndSet(&pTaskSet->fTriggered, (uint32_t)iTask))
+    {
+        Log(("PDMTaskTrigger: Triggered %RU64 (%s)\n", hTask, R3STRING(pTask->pszName)));
+#ifdef IN_RING3
+        if (pTaskSet->hEventR3 != NIL_RTSEMEVENT)
+        {
+            int rc = RTSemEventSignal(pTaskSet->hEventR3);
+            AssertLogRelRCReturn(rc, rc);
+        }
+        else
+#endif
+        {
+            int rc = SUPSemEventSignal(pVM->pSession, pTaskSet->hEventR0);
+            AssertLogRelRCReturn(rc, rc);
+        }
+        return VINF_SUCCESS;
+    }
+    ASMAtomicIncU32(&pTask->cAlreadyTrigged);
+    Log(("PDMTaskTrigger: %RU64 (%s) was already triggered\n", hTask, R3STRING(pTask->pszName)));
+    return VINF_ALREADY_POSTED;
+}
+
+
+/**
+ * Triggers an internal task.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTask       The task to trigger.
+ * @thread  Any
+ */
+VMM_INT_DECL(int)   PDMTaskTriggerInternal(PVMCC pVM, PDMTASKHANDLE hTask)
+{
+    return PDMTaskTrigger(pVM, PDMTASKTYPE_INTERNAL, pVM->pVMR3, hTask);
+}
+
diff --git a/src/VBox/VMM/VMMAll/PGMAll.cpp b/src/VBox/VMM/VMMAll/PGMAll.cpp
new file mode 100644
index 00000000..0074071a
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAll.cpp
@@ -0,0 +1,4077 @@
+/* $Id: PGMAll.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor - All context code.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/cpum.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/sup.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/hm_vmx.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include "PGMInline.h"
+#include <iprt/assert.h>
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+# include <iprt/asm-amd64-x86.h>
+#endif
+#include <iprt/string.h>
+#include <VBox/log.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPT *ppPdpt, PX86PDPAE *ppPD);
+DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
+DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested, PPGMPTWALK pWalk,
+                          PPGMPTWALKGST pGstWalk);
+static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3);
+static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT *ppPdpt, PEPTPD *ppPD,
+                                   PPGMPTWALKGST pGstWalkAll);
+#endif
+static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
+static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT *ppPdpt, PEPTPD *ppPD);
+
+
+/*
+ * Second level transation - EPT.
+ */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+# define PGM_SLAT_TYPE               PGM_SLAT_TYPE_EPT
+# include "PGMSlatDefs.h"
+# include "PGMAllGstSlatEpt.cpp.h"
+# undef PGM_SLAT_TYPE
+#endif
+
+
+/*
+ * Shadow - 32-bit mode
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_32BIT
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_32BIT(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_REAL(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllGst.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_PROT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllGst.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_32BIT_32BIT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_32BIT_PD
+#include "PGMGstDefs.h"
+#include "PGMAllGst.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - PAE mode
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_PAE
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_REAL(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_REAL(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_PROT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_32BIT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT_FOR_32BIT
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_PAE_PAE(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+#define BTH_PGMPOOLKIND_PT_FOR_BIG  PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
+#define BTH_PGMPOOLKIND_ROOT        PGMPOOLKIND_PAE_PDPT
+#include "PGMGstDefs.h"
+#include "PGMAllGst.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_BIG
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - AMD64 mode
+ */
+#define PGM_SHW_TYPE               PGM_TYPE_AMD64
+#define PGM_SHW_NAME(name)         PGM_SHW_NAME_AMD64(name)
+#include "PGMAllShw.h"
+
+/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
+/** @todo retire this hack. */
+#define PGM_GST_TYPE               PGM_TYPE_PROT
+#define PGM_GST_NAME(name)         PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)         PGM_BTH_NAME_AMD64_PROT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT  PGMPOOLKIND_PAE_PT_FOR_PHYS
+#define BTH_PGMPOOLKIND_ROOT       PGMPOOLKIND_PAE_PD_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef BTH_PGMPOOLKIND_ROOT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE              PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)        PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)        PGM_BTH_NAME_AMD64_AMD64(name)
+# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
+# define BTH_PGMPOOLKIND_ROOT      PGMPOOLKIND_64BIT_PML4
+# include "PGMGstDefs.h"
+# include "PGMAllGst.h"
+# include "PGMAllBth.h"
+# undef BTH_PGMPOOLKIND_PT_FOR_BIG
+# undef BTH_PGMPOOLKIND_PT_FOR_PT
+# undef BTH_PGMPOOLKIND_ROOT
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - 32-bit nested paging mode.
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_NESTED_32BIT
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_NESTED_32BIT(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_32BIT_REAL(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_32BIT_PROT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_32BIT_PAE(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
+# include "PGMGstDefs.h"
+# include "PGMAllBth.h"
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - PAE nested paging mode.
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_NESTED_PAE
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_NESTED_PAE(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PAE_REAL(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PAE_PROT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PAE_32BIT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_PAE_PAE(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_NESTED_PAE_AMD64(name)
+# include "PGMGstDefs.h"
+# include "PGMAllBth.h"
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - AMD64 nested paging mode.
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_NESTED_AMD64
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_NESTED_AMD64(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_AMD64_REAL(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_AMD64_PROT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NESTED_AMD64_PAE(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
+# include "PGMGstDefs.h"
+# include "PGMAllBth.h"
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - EPT.
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_EPT
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_EPT(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_REAL(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_EPT_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_PROT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_EPT_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_32BIT(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_EPT_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE                PGM_TYPE_PAE
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_EPT_PAE(name)
+#define BTH_PGMPOOLKIND_PT_FOR_PT   PGMPOOLKIND_EPT_PT_FOR_PHYS
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef BTH_PGMPOOLKIND_PT_FOR_PT
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_EPT_AMD64(name)
+# define BTH_PGMPOOLKIND_PT_FOR_PT  PGMPOOLKIND_EPT_PT_FOR_PHYS
+# include "PGMGstDefs.h"
+# include "PGMAllBth.h"
+# undef BTH_PGMPOOLKIND_PT_FOR_PT
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+/*
+ * Shadow - NEM / None.
+ */
+#define PGM_SHW_TYPE                PGM_TYPE_NONE
+#define PGM_SHW_NAME(name)          PGM_SHW_NAME_NONE(name)
+#include "PGMAllShw.h"
+
+/* Guest - real mode */
+#define PGM_GST_TYPE                PGM_TYPE_REAL
+#define PGM_GST_NAME(name)          PGM_GST_NAME_REAL(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NONE_REAL(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - protected mode */
+#define PGM_GST_TYPE                PGM_TYPE_PROT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_PROT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NONE_PROT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - 32-bit mode */
+#define PGM_GST_TYPE                PGM_TYPE_32BIT
+#define PGM_GST_NAME(name)          PGM_GST_NAME_32BIT(name)
+#define PGM_BTH_NAME(name)          PGM_BTH_NAME_NONE_32BIT(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+/* Guest - PAE mode */
+#define PGM_GST_TYPE               PGM_TYPE_PAE
+#define PGM_GST_NAME(name)         PGM_GST_NAME_PAE(name)
+#define PGM_BTH_NAME(name)         PGM_BTH_NAME_NONE_PAE(name)
+#include "PGMGstDefs.h"
+#include "PGMAllBth.h"
+#undef PGM_BTH_NAME
+#undef PGM_GST_TYPE
+#undef PGM_GST_NAME
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+/* Guest - AMD64 mode */
+# define PGM_GST_TYPE               PGM_TYPE_AMD64
+# define PGM_GST_NAME(name)         PGM_GST_NAME_AMD64(name)
+# define PGM_BTH_NAME(name)         PGM_BTH_NAME_NONE_AMD64(name)
+# include "PGMGstDefs.h"
+# include "PGMAllBth.h"
+# undef PGM_BTH_NAME
+# undef PGM_GST_TYPE
+# undef PGM_GST_NAME
+#endif /* VBOX_WITH_64_BITS_GUESTS */
+
+#undef PGM_SHW_TYPE
+#undef PGM_SHW_NAME
+
+
+
+/**
+ * Guest mode data array.
+ */
+PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
+{
+    { UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */
+    {
+        PGM_TYPE_REAL,
+        PGM_GST_NAME_REAL(GetPage),
+        PGM_GST_NAME_REAL(ModifyPage),
+        PGM_GST_NAME_REAL(Enter),
+        PGM_GST_NAME_REAL(Exit),
+#ifdef IN_RING3
+        PGM_GST_NAME_REAL(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_PROT,
+        PGM_GST_NAME_PROT(GetPage),
+        PGM_GST_NAME_PROT(ModifyPage),
+        PGM_GST_NAME_PROT(Enter),
+        PGM_GST_NAME_PROT(Exit),
+#ifdef IN_RING3
+        PGM_GST_NAME_PROT(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_32BIT,
+        PGM_GST_NAME_32BIT(GetPage),
+        PGM_GST_NAME_32BIT(ModifyPage),
+        PGM_GST_NAME_32BIT(Enter),
+        PGM_GST_NAME_32BIT(Exit),
+#ifdef IN_RING3
+        PGM_GST_NAME_32BIT(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_PAE,
+        PGM_GST_NAME_PAE(GetPage),
+        PGM_GST_NAME_PAE(ModifyPage),
+        PGM_GST_NAME_PAE(Enter),
+        PGM_GST_NAME_PAE(Exit),
+#ifdef IN_RING3
+        PGM_GST_NAME_PAE(Relocate),
+#endif
+    },
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    {
+        PGM_TYPE_AMD64,
+        PGM_GST_NAME_AMD64(GetPage),
+        PGM_GST_NAME_AMD64(ModifyPage),
+        PGM_GST_NAME_AMD64(Enter),
+        PGM_GST_NAME_AMD64(Exit),
+# ifdef IN_RING3
+        PGM_GST_NAME_AMD64(Relocate),
+# endif
+    },
+#endif
+};
+
+
+/**
+ * The shadow mode data array.
+ */
+PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
+{
+    { UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
+    { UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
+    { UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
+    {
+        PGM_TYPE_32BIT,
+        PGM_SHW_NAME_32BIT(GetPage),
+        PGM_SHW_NAME_32BIT(ModifyPage),
+        PGM_SHW_NAME_32BIT(Enter),
+        PGM_SHW_NAME_32BIT(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_32BIT(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_PAE,
+        PGM_SHW_NAME_PAE(GetPage),
+        PGM_SHW_NAME_PAE(ModifyPage),
+        PGM_SHW_NAME_PAE(Enter),
+        PGM_SHW_NAME_PAE(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_PAE(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_AMD64,
+        PGM_SHW_NAME_AMD64(GetPage),
+        PGM_SHW_NAME_AMD64(ModifyPage),
+        PGM_SHW_NAME_AMD64(Enter),
+        PGM_SHW_NAME_AMD64(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_AMD64(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_NESTED_32BIT,
+        PGM_SHW_NAME_NESTED_32BIT(GetPage),
+        PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
+        PGM_SHW_NAME_NESTED_32BIT(Enter),
+        PGM_SHW_NAME_NESTED_32BIT(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_NESTED_32BIT(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_NESTED_PAE,
+        PGM_SHW_NAME_NESTED_PAE(GetPage),
+        PGM_SHW_NAME_NESTED_PAE(ModifyPage),
+        PGM_SHW_NAME_NESTED_PAE(Enter),
+        PGM_SHW_NAME_NESTED_PAE(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_NESTED_PAE(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_NESTED_AMD64,
+        PGM_SHW_NAME_NESTED_AMD64(GetPage),
+        PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
+        PGM_SHW_NAME_NESTED_AMD64(Enter),
+        PGM_SHW_NAME_NESTED_AMD64(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_NESTED_AMD64(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_EPT,
+        PGM_SHW_NAME_EPT(GetPage),
+        PGM_SHW_NAME_EPT(ModifyPage),
+        PGM_SHW_NAME_EPT(Enter),
+        PGM_SHW_NAME_EPT(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_EPT(Relocate),
+#endif
+    },
+    {
+        PGM_TYPE_NONE,
+        PGM_SHW_NAME_NONE(GetPage),
+        PGM_SHW_NAME_NONE(ModifyPage),
+        PGM_SHW_NAME_NONE(Enter),
+        PGM_SHW_NAME_NONE(Exit),
+#ifdef IN_RING3
+        PGM_SHW_NAME_NONE(Relocate),
+#endif
+    },
+};
+
+
+/**
+ * The guest+shadow mode data array.
+ */
+PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
+{
+#if   !defined(IN_RING3) && !defined(VBOX_STRICT)
+# define PGMMODEDATABTH_NULL_ENTRY()    { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
+# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
+    { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler) }
+
+#elif !defined(IN_RING3) && defined(VBOX_STRICT)
+# define PGMMODEDATABTH_NULL_ENTRY()    { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
+# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
+    { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(NestedTrap0eHandler), Nm(AssertCR3) }
+
+#elif defined(IN_RING3) && !defined(VBOX_STRICT)
+# define PGMMODEDATABTH_NULL_ENTRY()    { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
+# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
+    { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
+
+#elif defined(IN_RING3) && defined(VBOX_STRICT)
+# define PGMMODEDATABTH_NULL_ENTRY()    { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
+# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
+    { uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
+
+#else
+# error "Misconfig."
+#endif
+
+    /* 32-bit shadow paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL,  PGM_BTH_NAME_32BIT_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT,  PGM_BTH_NAME_32BIT_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64        - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE         - illegal */
+
+    /* PAE shadow paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL,  PGM_BTH_NAME_PAE_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT,  PGM_BTH_NAME_PAE_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE,   PGM_BTH_NAME_PAE_PAE),
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64        - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE         - illegal */
+
+    /* AMD64 shadow paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL,  PGM_BTH_NAME_AMD64_REAL),
+    PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT,  PGM_BTH_NAME_AMD64_PROT),
+    PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
+    PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE,   PGM_BTH_NAME_AMD64_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE         - illegal */
+
+    /* 32-bit nested paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL,  PGM_BTH_NAME_NESTED_32BIT_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT,  PGM_BTH_NAME_NESTED_32BIT_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE,   PGM_BTH_NAME_NESTED_32BIT_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE         - illegal */
+
+    /* PAE nested paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL,  PGM_BTH_NAME_NESTED_PAE_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT,  PGM_BTH_NAME_NESTED_PAE_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE,   PGM_BTH_NAME_NESTED_PAE_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE         - illegal */
+
+    /* AMD64 nested paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL,  PGM_BTH_NAME_NESTED_AMD64_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT,  PGM_BTH_NAME_NESTED_AMD64_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE,   PGM_BTH_NAME_NESTED_AMD64_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE         - illegal */
+
+    /* EPT nested paging mode: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL,  PGM_BTH_NAME_EPT_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT,  PGM_BTH_NAME_EPT_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE,   PGM_BTH_NAME_EPT_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE         - illegal */
+
+    /* NONE / NEM: */
+    PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL,  PGM_BTH_NAME_EPT_REAL),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT,  PGM_BTH_NAME_EPT_PROT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE,   PGM_BTH_NAME_EPT_PAE),
+#ifdef VBOX_WITH_64_BITS_GUESTS
+    PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
+#else
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64        - illegal */
+#endif
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE   - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT          - illegal */
+    PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE         - illegal */
+
+
+#undef PGMMODEDATABTH_ENTRY
+#undef PGMMODEDATABTH_NULL_ENTRY
+};
+
+
+/** Mask array used by pgmGetCr3MaskForMode.
+ * X86_CR3_AMD64_PAGE_MASK is used for modes that doesn't have a CR3 or EPTP. */
+static uint64_t const g_auCr3MaskForMode[PGMMODE_MAX] =
+{
+    /* [PGMMODE_INVALID] = */           X86_CR3_AMD64_PAGE_MASK,
+    /* [PGMMODE_REAL] = */              X86_CR3_AMD64_PAGE_MASK,
+    /* [PGMMODE_PROTECTED] = */         X86_CR3_AMD64_PAGE_MASK,
+    /* [PGMMODE_32_BIT] = */            X86_CR3_PAGE_MASK,
+    /* [PGMMODE_PAE] = */               X86_CR3_PAE_PAGE_MASK,
+    /* [PGMMODE_PAE_NX] = */            X86_CR3_PAE_PAGE_MASK,
+    /* [PGMMODE_AMD64] = */             X86_CR3_AMD64_PAGE_MASK,
+    /* [PGMMODE_AMD64_NX] = */          X86_CR3_AMD64_PAGE_MASK,
+    /* [PGMMODE_NESTED_32BIT = */       X86_CR3_PAGE_MASK,
+    /* [PGMMODE_NESTED_PAE] = */        X86_CR3_PAE_PAGE_MASK,
+    /* [PGMMODE_NESTED_AMD64] = */      X86_CR3_AMD64_PAGE_MASK,
+    /* [PGMMODE_EPT] = */               X86_CR3_EPT_PAGE_MASK,
+    /* [PGMMODE_NONE] = */              X86_CR3_AMD64_PAGE_MASK,
+};
+
+
+/**
+ * Gets the physical address mask for CR3 in the given paging mode.
+ *
+ * The mask is for eliminating flags and other stuff in CR3/EPTP when
+ * extracting the physical address.  It is not for validating whether there are
+ * reserved bits set.  PGM ASSUMES that whoever loaded the CR3 value and passed
+ * it to PGM checked for reserved bits, including reserved physical address
+ * bits.
+ *
+ * @returns The CR3 mask.
+ * @param   enmMode         The paging mode.
+ * @param   enmSlatMode     The second-level address translation mode.
+ */
+DECLINLINE(uint64_t) pgmGetCr3MaskForMode(PGMMODE enmMode, PGMSLAT enmSlatMode)
+{
+    if (enmSlatMode == PGMSLAT_DIRECT)
+    {
+        Assert(enmMode != PGMMODE_EPT);
+        return g_auCr3MaskForMode[(unsigned)enmMode < (unsigned)PGMMODE_MAX ? enmMode : 0];
+    }
+    Assert(enmSlatMode == PGMSLAT_EPT);
+    return X86_CR3_EPT_PAGE_MASK;
+}
+
+
+/**
+ * Gets the masked CR3 value according to the current guest paging mode.
+ *
+ * See disclaimer in pgmGetCr3MaskForMode.
+ *
+ * @returns The masked PGM CR3 value.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   uCr3    The raw guest CR3 value.
+ */
+DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
+{
+    uint64_t const fCr3Mask  = pgmGetCr3MaskForMode(pVCpu->pgm.s.enmGuestMode, pVCpu->pgm.s.enmGuestSlatMode);
+    RTGCPHYS       GCPhysCR3 = (RTGCPHYS)(uCr3 & fCr3Mask);
+    PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
+    return GCPhysCR3;
+}
+
+
+#ifdef IN_RING0
+/**
+ * #PF Handler.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uErr        The trap error code.
+ * @param   pCtx        Pointer to the register context for the CPU.
+ * @param   pvFault     The fault address.
+ */
+VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a);
+    STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
+
+
+# ifdef VBOX_WITH_STATISTICS
+    /*
+     * Error code stats.
+     */
+    if (uErr & X86_TRAP_PF_US)
+    {
+        if (!(uErr & X86_TRAP_PF_P))
+        {
+            if (uErr & X86_TRAP_PF_RW)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
+            else
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
+        }
+        else if (uErr & X86_TRAP_PF_RW)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
+        else if (uErr & X86_TRAP_PF_RSVD)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
+        else if (uErr & X86_TRAP_PF_ID)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
+        else
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
+    }
+    else
+    {   /* Supervisor */
+        if (!(uErr & X86_TRAP_PF_P))
+        {
+            if (uErr & X86_TRAP_PF_RW)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
+            else
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
+        }
+        else if (uErr & X86_TRAP_PF_RW)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
+        else if (uErr & X86_TRAP_PF_ID)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
+        else if (uErr & X86_TRAP_PF_RSVD)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
+    }
+# endif /* VBOX_WITH_STATISTICS */
+
+    /*
+     * Call the worker.
+     */
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
+    bool fLockTaken = false;
+    int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pCtx, pvFault, &fLockTaken);
+    if (fLockTaken)
+    {
+        PGM_LOCK_ASSERT_OWNER(pVM);
+        PGM_UNLOCK(pVM);
+    }
+    LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
+
+    /*
+     * Return code tweaks.
+     */
+    if (rc != VINF_SUCCESS)
+    {
+        if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
+            rc = VINF_SUCCESS;
+
+        /* Note: hack alert for difficult to reproduce problem. */
+        if (    rc == VERR_PAGE_NOT_PRESENT                 /* SMP only ; disassembly might fail. */
+            ||  rc == VERR_PAGE_TABLE_NOT_PRESENT           /* seen with UNI & SMP */
+            ||  rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT   /* seen with SMP */
+            ||  rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)     /* precaution */
+        {
+            Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pCtx->rip));
+            /* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
+            rc = VINF_SUCCESS;
+        }
+    }
+
+    STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); });
+    STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0)
+                    pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
+    STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
+    return rc;
+}
+#endif /* IN_RING0 */
+
+
+/**
+ * Prefetch a page
+ *
+ * Typically used to sync commonly used pages before entering raw mode
+ * after a CR3 reload.
+ *
+ * @returns VBox status code suitable for scheduling.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   Page to invalidate.
+ */
+VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
+
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
+    int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
+    AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * Emulation of the invlpg instruction (HC only actually).
+ *
+ * @returns Strict VBox status code, special care required.
+ * @retval  VINF_PGM_SYNC_CR3 - handled.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
+ * @retval  VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   Page to invalidate.
+ *
+ * @remark  ASSUMES the page table entry or page directory is valid. Fairly
+ *          safe, but there could be edge cases!
+ *
+ * @todo    Flush page or page directory only if necessary!
+ * @todo    VBOXSTRICTRC
+ */
+VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    int rc;
+    Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
+
+    IEMTlbInvalidatePage(pVCpu, GCPtrPage);
+
+    /*
+     * Call paging mode specific worker.
+     */
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
+    PGM_LOCK_VOID(pVM);
+
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
+    AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
+    rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
+
+    PGM_UNLOCK(pVM);
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
+
+    /* Ignore all irrelevant error codes. */
+    if (    rc == VERR_PAGE_NOT_PRESENT
+        ||  rc == VERR_PAGE_TABLE_NOT_PRESENT
+        ||  rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
+        ||  rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
+        rc = VINF_SUCCESS;
+
+    return rc;
+}
+
+
+/**
+ * Executes an instruction using the interpreter.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pvFault     Fault address.
+ */
+VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCPUCC pVCpu, RTGCPTR pvFault)
+{
+    RT_NOREF(pvFault);
+    VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu);
+    if (rc == VERR_EM_INTERPRETER)
+        rc = VINF_EM_RAW_EMULATE_INSTR;
+    if (rc != VINF_SUCCESS)
+        Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
+    return rc;
+}
+
+
+/**
+ * Gets effective page information (from the VMM page directory).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Guest Context virtual address of the page.
+ * @param   pfFlags     Where to store the flags. These are X86_PTE_*.
+ * @param   pHCPhys     Where to store the HC physical address of the page.
+ *                      This is page aligned.
+ * @remark  You should use PGMMapGetPage() for pages in a mapping.
+ */
+VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
+
+    uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
+    AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
+    int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
+
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+
+/**
+ * Modify page flags for a range of pages in the shadow context.
+ *
+ * The existing flags are ANDed with the fMask and ORed with the fFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   fFlags      The OR  mask - page flags X86_PTE_*, excluding the page mask of course.
+ * @param   fMask       The AND mask - page flags X86_PTE_*.
+ *                      Be very CAREFUL when ~'ing constants which could be 32-bit!
+ * @param   fOpFlags    A combination of the PGM_MK_PK_XXX flags.
+ * @remark  You must use PGMMapModifyPage() for pages in a mapping.
+ */
+DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
+{
+    AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
+    Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 | PGM_MK_PG_IS_WRITE_FAULT)));
+
+    GCPtr &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK; /** @todo this ain't necessary, right... */
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
+
+    uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
+    AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
+    int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, GUEST_PAGE_SIZE, fFlags, fMask, fOpFlags);
+
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+
+/**
+ * Changing the page flags for a single page in the shadow page tables so as to
+ * make it read-only.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   fOpFlags    A combination of the PGM_MK_PK_XXX flags.
+ */
+VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
+{
+    return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
+}
+
+
+/**
+ * Changing the page flags for a single page in the shadow page tables so as to
+ * make it writable.
+ *
+ * The call must know with 101% certainty that the guest page tables maps this
+ * as writable too.  This function will deal shared, zero and write monitored
+ * pages.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   fOpFlags    A combination of the PGM_MK_PK_XXX flags.
+ */
+VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
+{
+    if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
+        return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Changing the page flags for a single page in the shadow page tables so as to
+ * make it not present.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   fOpFlags    A combination of the PGM_MK_PG_XXX flags.
+ */
+VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
+{
+    return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
+}
+
+
+/**
+ * Changing the page flags for a single page in the shadow page tables so as to
+ * make it supervisor and writable.
+ *
+ * This if for dealing with CR0.WP=0 and readonly user pages.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   fBigPage    Whether or not this is a big page. If it is, we have to
+ *                      change the shadow PDE as well.  If it isn't, the caller
+ *                      has checked that the shadow PDE doesn't need changing.
+ *                      We ASSUME 4KB pages backing the big page here!
+ * @param   fOpFlags    A combination of the PGM_MK_PG_XXX flags.
+ */
+int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
+{
+    int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
+    if (rc == VINF_SUCCESS && fBigPage)
+    {
+        /* this is a bit ugly... */
+        switch (pVCpu->pgm.s.enmShadowMode)
+        {
+            case PGMMODE_32_BIT:
+            {
+                PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
+                AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
+                Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
+                pPde->u |= X86_PDE_RW;
+                Log(("-> PDE=%#llx (32)\n", pPde->u));
+                break;
+            }
+            case PGMMODE_PAE:
+            case PGMMODE_PAE_NX:
+            {
+                PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
+                AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
+                Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
+                pPde->u |= X86_PDE_RW;
+                Log(("-> PDE=%#llx (PAE)\n", pPde->u));
+                break;
+            }
+            default:
+                AssertFailedReturn(VERR_INTERNAL_ERROR_4);
+        }
+    }
+    return rc;
+}
+
+
+/**
+ * Gets the shadow page directory for the specified address, PAE.
+ *
+ * @returns Pointer to the shadow PD.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @param   uGstPdpe    Guest PDPT entry. Valid.
+ * @param   ppPD        Receives address of page directory
+ */
+int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
+{
+    PVMCC          pVM      = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL       pPool    = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPAGE   pShwPage;
+    int            rc;
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+
+    /* Allocate page directory if not present. */
+    const unsigned iPdPt     = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+    PX86PDPT       pPdpt     = pgmShwGetPaePDPTPtr(pVCpu);
+    PX86PDPE       pPdpe     = &pPdpt->a[iPdPt];
+    X86PGPAEUINT const uPdpe = pPdpe->u;
+    if (uPdpe & (X86_PDPE_P | X86_PDPE_PG_MASK))
+    {
+        pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
+        AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+        Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
+
+        pgmPoolCacheUsed(pPool, pShwPage);
+
+        /* Update the entry if necessary. */
+        X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key | (uGstPdpe & (X86_PDPE_P | X86_PDPE_A)) | (uPdpe & PGM_PDPT_FLAGS);
+        if (uPdpeNew == uPdpe)
+        { /* likely */ }
+        else
+            ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
+    }
+    else
+    {
+        RTGCPTR64   GCPdPt;
+        PGMPOOLKIND enmKind;
+        if (pVM->pgm.s.fNestedPaging || !CPUMIsGuestPagingEnabled(pVCpu))
+        {
+            /* AMD-V nested paging or real/protected mode without paging. */
+            GCPdPt  = GCPtr & ~(RT_BIT_64(X86_PDPT_SHIFT) - 1);
+            enmKind = PGMPOOLKIND_PAE_PD_PHYS;
+        }
+        else if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
+        {
+            if (uGstPdpe & X86_PDPE_P)
+            {
+                GCPdPt  = uGstPdpe & X86_PDPE_PG_MASK;
+                enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
+            }
+            else
+            {
+                /* PD not present; guest must reload CR3 to change it.
+                 * No need to monitor anything in this case. */
+                /** @todo r=bird: WTF is hit?!?   */
+                /*Assert(VM_IS_RAW_MODE_ENABLED(pVM)); - ??? */
+                GCPdPt  = uGstPdpe & X86_PDPE_PG_MASK;
+                enmKind = PGMPOOLKIND_PAE_PD_PHYS;
+                Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */
+            }
+        }
+        else
+        {
+            GCPdPt  = CPUMGetGuestCR3(pVCpu);
+            enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
+        }
+
+        /* Create a reference back to the PDPT by using the index in its shadow page. */
+        rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /*fLockPage*/,
+                          &pShwPage);
+        AssertRCReturn(rc, rc);
+
+        /* Hook it up. */
+        ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key | (uGstPdpe & (X86_PDPE_P | X86_PDPE_A)) | (uPdpe & PGM_PDPT_FLAGS));
+    }
+    PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
+
+    *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the pointer to the shadow page directory entry for an address, PAE.
+ *
+ * @returns Pointer to the PDE.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       The address.
+ * @param   ppShwPde    Receives the address of the pgm pool page for the shadow page directory
+ */
+DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
+{
+    PVMCC pVM   = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    PX86PDPT        pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
+    AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT);    /* can't happen */
+    const unsigned  iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
+    X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
+    if (!(uPdpe & X86_PDPE_P))
+    {
+        LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe));
+        return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
+    }
+    AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+    *ppShwPde = pShwPde;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Syncs the SHADOW page directory pointer for the specified address.
+ *
+ * Allocates backing pages in case the PDPT or PML4 entry is missing.
+ *
+ * The caller is responsible for making sure the guest has a valid PD before
+ * calling this function.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @param   uGstPml4e   Guest PML4 entry (valid).
+ * @param   uGstPdpe    Guest PDPT entry (valid).
+ * @param   ppPD        Receives address of page directory
+ */
+static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
+{
+    PVMCC          pVM           = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL       pPool         = pVM->pgm.s.CTX_SUFF(pPool);
+    bool const     fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging || !CPUMIsGuestPagingEnabled(pVCpu);
+    int            rc;
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * PML4.
+     */
+    PPGMPOOLPAGE pShwPage;
+    {
+        const unsigned     iPml4  = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+        PX86PML4E          pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
+        X86PGPAEUINT const uPml4e = pPml4e->u;
+
+        /* Allocate page directory pointer table if not present. */
+        if (uPml4e & (X86_PML4E_P | X86_PML4E_PG_MASK))
+        {
+            pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
+            AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+            pgmPoolCacheUsed(pPool, pShwPage);
+
+            /* Update the entry if needed. */
+            X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key | (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
+                                         | (uPml4e & PGM_PML4_FLAGS);
+            if (uPml4e == uPml4eNew)
+            { /* likely */ }
+            else
+                ASMAtomicWriteU64(&pPml4e->u, uPml4eNew);
+        }
+        else
+        {
+            Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+
+            RTGCPTR64   GCPml4;
+            PGMPOOLKIND enmKind;
+            if (fNestedPagingOrNoGstPaging)
+            {
+                /* AMD-V nested paging or real/protected mode without paging */
+                GCPml4  = (RTGCPTR64)iPml4 << X86_PML4_SHIFT; /** @todo bogus calculation for PML5 */
+                enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
+            }
+            else
+            {
+                GCPml4  = uGstPml4e & X86_PML4E_PG_MASK;
+                enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
+            }
+
+            /* Create a reference back to the PDPT by using the index in its shadow page. */
+            rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                              pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /*fLockPage*/,
+                              &pShwPage);
+            AssertRCReturn(rc, rc);
+
+            /* Hook it up. */
+            ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
+                                        | (uPml4e & PGM_PML4_FLAGS));
+        }
+    }
+
+    /*
+     * PDPT.
+     */
+    const unsigned     iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    PX86PDPT           pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    PX86PDPE           pPdpe = &pPdpt->a[iPdPt];
+    X86PGPAEUINT const uPdpe = pPdpe->u;
+
+    /* Allocate page directory if not present. */
+    if (uPdpe & (X86_PDPE_P | X86_PDPE_PG_MASK))
+    {
+        pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
+        AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+        pgmPoolCacheUsed(pPool, pShwPage);
+
+        /* Update the entry if needed. */
+        X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key | (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask)
+                                     | (uPdpe & PGM_PDPT_FLAGS);
+        if (uPdpe == uPdpeNew)
+        { /* likely */ }
+        else
+            ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
+    }
+    else
+    {
+        RTGCPTR64   GCPdPt;
+        PGMPOOLKIND enmKind;
+        if (fNestedPagingOrNoGstPaging)
+        {
+            /* AMD-V nested paging or real/protected mode without paging */
+            GCPdPt  = GCPtr & ~(RT_BIT_64(iPdPt << X86_PDPT_SHIFT) - 1);
+            enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
+        }
+        else
+        {
+            GCPdPt  = uGstPdpe & X86_PDPE_PG_MASK;
+            enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
+        }
+
+        /* Create a reference back to the PDPT by using the index in its shadow page. */
+        rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          pShwPage->idx, iPdPt, false /*fLockPage*/,
+                          &pShwPage);
+        AssertRCReturn(rc, rc);
+
+        /* Hook it up. */
+        ASMAtomicWriteU64(&pPdpe->u,
+                          pShwPage->Core.Key | (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) | (uPdpe & PGM_PDPT_FLAGS));
+    }
+
+    *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the SHADOW page directory pointer for the specified address (long mode).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @param   ppPml4e     Receives the address of the page map level 4 entry.
+ * @param   ppPdpt      Receives the address of the page directory pointer table.
+ * @param   ppPD        Receives the address of the page directory.
+ */
+DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E *ppPml4e, PX86PDPT *ppPdpt, PX86PDPAE *ppPD)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * PML4
+     */
+    const unsigned  iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+    PCX86PML4E      pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
+    AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
+    if (ppPml4e)
+        *ppPml4e = (PX86PML4E)pPml4e;
+    X86PGPAEUINT const uPml4e = pPml4e->u;
+    Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e));
+    if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */
+        return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
+
+    PPGMPOOL        pPool    = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
+    AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+    /*
+     * PDPT
+     */
+    const unsigned      iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    PCX86PDPT           pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    X86PGPAEUINT const  uPdpe = pPdpt->a[iPdPt].u;
+    if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */
+        return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
+
+    pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+    *ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    Log4(("pgmShwGetLongModePDPtr %RGv -> *ppPD=%p PDE=%p/%RX64\n", GCPtr, *ppPD, &(*ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (*ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
+ * backing pages in case the PDPT or PML4 entry is missing.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The address.
+ * @param   ppPdpt      Receives address of pdpt
+ * @param   ppPD        Receives address of page directory
+ */
+static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT *ppPdpt, PEPTPD *ppPD)
+{
+    PVMCC          pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL       pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    int            rc;
+
+    Assert(pVM->pgm.s.fNestedPaging);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * PML4 level.
+     */
+    PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+    Assert(pPml4);
+
+    /* Allocate page directory pointer table if not present. */
+    PPGMPOOLPAGE pShwPage;
+    {
+        const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
+        PEPTPML4E pPml4e = &pPml4->a[iPml4];
+        EPTPML4E Pml4e;
+        Pml4e.u = pPml4e->u;
+        if (!(Pml4e.u & (EPT_E_PG_MASK | EPT_E_READ)))
+        {
+            RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
+            rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                              pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /*fLockPage*/,
+                              &pShwPage);
+            AssertRCReturn(rc, rc);
+
+            /* Hook up the new PDPT now. */
+            ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE);
+        }
+        else
+        {
+            pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
+            AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+            pgmPoolCacheUsed(pPool, pShwPage);
+
+            /* Hook up the cached PDPT if needed (probably not given 512*512 PTs to sync). */
+            if (Pml4e.u == (pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE))
+            { }
+            else
+                ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE);
+        }
+    }
+
+    /*
+     * PDPT level.
+     */
+    const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
+    PEPTPDPT  pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
+
+    if (ppPdpt)
+        *ppPdpt = pPdpt;
+
+    /* Allocate page directory if not present. */
+    EPTPDPTE Pdpe;
+    Pdpe.u = pPdpe->u;
+    if (!(Pdpe.u & (EPT_E_PG_MASK | EPT_E_READ)))
+    {
+        RTGCPTR64 const GCPdPt = GCPtr & ~(RT_BIT_64(EPT_PDPT_SHIFT) - 1);
+        rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          pShwPage->idx, iPdPt, false /*fLockPage*/,
+                          &pShwPage);
+        AssertRCReturn(rc, rc);
+
+        /* Hook up the new PD now. */
+        ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE);
+    }
+    else
+    {
+        pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
+        AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+        pgmPoolCacheUsed(pPool, pShwPage);
+
+        /* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
+        if (Pdpe.u == (pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE))
+        { }
+        else
+            ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE);
+    }
+
+    *ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    return VINF_SUCCESS;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * Syncs the SHADOW nested-guest page directory pointer for the specified address.
+ * Allocates backing pages in case the PDPT or PML4 entry is missing.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhysNested    The nested-guest physical address.
+ * @param   ppPdpt          Where to store the PDPT. Optional, can be NULL.
+ * @param   ppPD            Where to store the PD. Optional, can be NULL.
+ * @param   pGstWalkAll     The guest walk info.
+ */
+static int pgmShwGetNestedEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPhysNested, PEPTPDPT *ppPdpt, PEPTPD *ppPD,
+                                   PPGMPTWALKGST pGstWalkAll)
+{
+    PVMCC    pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    int      rc;
+
+    PPGMPOOLPAGE pShwPage;
+    Assert(pVM->pgm.s.fNestedPaging);
+    Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * PML4 level.
+     */
+    {
+        PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+        Assert(pPml4);
+
+        /* Allocate page directory pointer table if not present. */
+        {
+            uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pml4e.u & pVCpu->pgm.s.fGstEptShadowedPml4eMask;
+            const unsigned iPml4e    = (GCPhysNested >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
+            PEPTPML4E      pPml4e    = &pPml4->a[iPml4e];
+
+            if (!(pPml4e->u & (EPT_E_PG_MASK | EPT_PRESENT_MASK)))
+            {
+                RTGCPHYS const GCPhysPdpt = pGstWalkAll->u.Ept.Pml4e.u & EPT_PML4E_PG_MASK;
+                rc = pgmPoolAlloc(pVM, GCPhysPdpt, PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT, PGMPOOLACCESS_DONTCARE,
+                                  PGM_A20_IS_ENABLED(pVCpu), pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4e, false /*fLockPage*/,
+                                  &pShwPage);
+                AssertRCReturn(rc, rc);
+
+                /* Hook up the new PDPT now. */
+                ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | fShwFlags);
+            }
+            else
+            {
+                pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
+                AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+                pgmPoolCacheUsed(pPool, pShwPage);
+
+                /* Hook up the cached PDPT if needed (probably not given 512*512 PTs to sync). */
+                if (pPml4e->u != (pShwPage->Core.Key | fShwFlags))
+                    ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key | fShwFlags);
+            }
+            Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
+            Log7Func(("GstPml4e=%RX64 ShwPml4e=%RX64 iPml4e=%u\n", pGstWalkAll->u.Ept.Pml4e.u, pPml4e->u, iPml4e));
+        }
+    }
+
+    /*
+     * PDPT level.
+     */
+    {
+        AssertReturn(!(pGstWalkAll->u.Ept.Pdpte.u & EPT_E_LEAF), VERR_NOT_SUPPORTED); /* shadowing 1GB pages not supported yet. */
+
+        PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+        if (ppPdpt)
+            *ppPdpt = pPdpt;
+
+        uint64_t const fShwFlags = pGstWalkAll->u.Ept.Pdpte.u & pVCpu->pgm.s.fGstEptShadowedPdpteMask;
+        const unsigned iPdPte    = (GCPhysNested >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
+        PEPTPDPTE      pPdpte    = &pPdpt->a[iPdPte];
+
+        if (!(pPdpte->u & (EPT_E_PG_MASK | EPT_PRESENT_MASK)))
+        {
+            RTGCPHYS const GCPhysPd = pGstWalkAll->u.Ept.Pdpte.u & EPT_PDPTE_PG_MASK;
+            rc = pgmPoolAlloc(pVM, GCPhysPd, PGMPOOLKIND_EPT_PD_FOR_EPT_PD, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                              pShwPage->idx, iPdPte, false /*fLockPage*/, &pShwPage);
+            AssertRCReturn(rc, rc);
+
+            /* Hook up the new PD now. */
+            ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key | fShwFlags);
+        }
+        else
+        {
+            pShwPage = pgmPoolGetPage(pPool, pPdpte->u & EPT_PDPTE_PG_MASK);
+            AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+            pgmPoolCacheUsed(pPool, pShwPage);
+
+            /* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
+            if (pPdpte->u != (pShwPage->Core.Key | fShwFlags))
+                ASMAtomicWriteU64(&pPdpte->u, pShwPage->Core.Key | fShwFlags);
+        }
+        Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
+        Log7Func(("GstPdpte=%RX64 ShwPdpte=%RX64 iPdPte=%u \n", pGstWalkAll->u.Ept.Pdpte.u, pPdpte->u, iPdPte));
+
+        *ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    }
+
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+
+#ifdef IN_RING0
+/**
+ * Synchronizes a range of nested page table entries.
+ *
+ * The caller must own the PGM lock.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhys              Where to start.
+ * @param   cPages              How many pages which entries should be synced.
+ * @param   enmShwPagingMode    The shadow paging mode (PGMMODE_EPT for VT-x,
+ *                              host paging mode for AMD-V).
+ */
+int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
+{
+    PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
+
+/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
+    int rc;
+    switch (enmShwPagingMode)
+    {
+        case PGMMODE_32_BIT:
+        {
+            X86PDE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
+            rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
+            break;
+        }
+
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:
+        {
+            X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
+            rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
+            break;
+        }
+
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+        {
+            X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
+            rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
+            break;
+        }
+
+        case PGMMODE_EPT:
+        {
+            X86PDEPAE PdeDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
+            rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /*uErr*/);
+            break;
+        }
+
+        default:
+            AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+    return rc;
+}
+#endif /* IN_RING0 */
+
+
+/**
+ * Gets effective Guest OS page information.
+ *
+ * When GCPtr is in a big page, the function will return as if it was a normal
+ * 4KB page. If the need for distinguishing between big and normal page becomes
+ * necessary at a later point, a PGMGstGetPage() will be created for that
+ * purpose.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       Guest Context virtual address of the page.
+ * @param   pWalk       Where to store the page walk information.
+ */
+VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    Assert(pWalk);
+    uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
+    AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
+    return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pWalk);
+}
+
+
+/**
+ * Maps the guest CR3.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhysCr3       The guest CR3 value.
+ * @param   pHCPtrGuestCr3  Where to store the mapped memory.
+ */
+DECLINLINE(int) pgmGstMapCr3(PVMCPUCC pVCpu, RTGCPHYS GCPhysCr3, PRTHCPTR pHCPtrGuestCr3)
+{
+    /** @todo this needs some reworking wrt. locking?  */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
+    PPGMPAGE pPageCr3 = pgmPhysGetPage(pVM, GCPhysCr3);
+    AssertReturnStmt(pPageCr3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR);
+
+    RTHCPTR HCPtrGuestCr3;
+    int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCr3, GCPhysCr3, (void **)&HCPtrGuestCr3);
+    PGM_UNLOCK(pVM);
+
+    *pHCPtrGuestCr3 = HCPtrGuestCr3;
+    return rc;
+}
+
+
+/**
+ * Unmaps the guest CR3.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(int) pgmGstUnmapCr3(PVMCPUCC pVCpu)
+{
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
+    return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
+}
+
+
+/**
+ * Performs a guest page table walk.
+ *
+ * The guest should be in paged protect mode or long mode when making a call to
+ * this function.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT on failure.  Check pWalk for details.
+ * @retval  VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
+ *          not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       The guest virtual address to walk by.
+ * @param   pWalk       Where to return the walk result. This is valid for some
+ *                      error codes as well.
+ * @param   pGstWalk    The guest mode specific page walk information.
+ */
+int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    switch (pVCpu->pgm.s.enmGuestMode)
+    {
+        case PGMMODE_32_BIT:
+            pGstWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
+            return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Legacy);
+
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:
+            pGstWalk->enmType = PGMPTWALKGSTTYPE_PAE;
+            return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Pae);
+
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+            pGstWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
+            return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, pWalk, &pGstWalk->u.Amd64);
+
+        case PGMMODE_REAL:
+        case PGMMODE_PROTECTED:
+            pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
+            return VERR_PGM_NOT_USED_IN_MODE;
+
+        case PGMMODE_EPT:
+        case PGMMODE_NESTED_32BIT:
+        case PGMMODE_NESTED_PAE:
+        case PGMMODE_NESTED_AMD64:
+        default:
+            AssertFailed();
+            pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
+            return VERR_PGM_NOT_USED_IN_MODE;
+    }
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * Performs a guest second-level address translation (SLAT).
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT on failure.  Check pWalk for details.
+ * @retval  VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
+ *          not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhysNested        The nested-guest physical address being translated.
+ * @param   fIsLinearAddrValid  Whether the linear address in @a GCPtrNested is the
+ *                              cause for this translation.
+ * @param   GCPtrNested         The nested-guest virtual address that initiated the
+ *                              SLAT. If none, pass 0 (and not NIL_RTGCPTR).
+ * @param   pWalk               Where to return the walk result. This is updated for
+ *                              all error codes other than
+ *                              VERR_PGM_NOT_USED_IN_MODE.
+ * @param   pGstWalk            Where to store the second-level paging-mode specific
+ *                              walk info.
+ */
+static int pgmGstSlatWalk(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested,
+                          PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
+{
+    /* SLAT mode must be valid at this point as this should only be used -after- we have determined SLAT mode. */
+    Assert(   pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_DIRECT
+           && pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_INVALID);
+    AssertPtr(pWalk);
+    AssertPtr(pGstWalk);
+    switch (pVCpu->pgm.s.enmGuestSlatMode)
+    {
+        case PGMSLAT_EPT:
+            pGstWalk->enmType = PGMPTWALKGSTTYPE_EPT;
+            return PGM_GST_SLAT_NAME_EPT(Walk)(pVCpu, GCPhysNested, fIsLinearAddrValid, GCPtrNested, pWalk, &pGstWalk->u.Ept);
+
+        default:
+            AssertFailed();
+            pGstWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
+            return VERR_PGM_NOT_USED_IN_MODE;
+    }
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+
+/**
+ * Tries to continue the previous walk.
+ *
+ * @note    Requires the caller to hold the PGM lock from the first
+ *          pgmGstPtWalk() call to the last pgmGstPtWalkNext() call.  Otherwise
+ *          we cannot use the pointers.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT on failure.  Check pWalk for details.
+ * @retval  VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
+ *          not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       The guest virtual address to walk by.
+ * @param   pWalk       Pointer to the previous walk result and where to return
+ *                      the result of this walk.  This is valid for some error
+ *                      codes as well.
+ * @param   pGstWalk    The guest-mode specific walk information.
+ */
+int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PPGMPTWALKGST pGstWalk)
+{
+    /*
+     * We can only handle successfully walks.
+     * We also limit ourselves to the next page.
+     */
+    if (   pWalk->fSucceeded
+        && GCPtr - pWalk->GCPtr == GUEST_PAGE_SIZE)
+    {
+        Assert(pWalk->uLevel == 0);
+        if (pGstWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
+        {
+            /*
+             * AMD64
+             */
+            if (!pWalk->fGigantPage && !pWalk->fBigPage)
+            {
+                /*
+                 * We fall back to full walk if the PDE table changes, if any
+                 * reserved bits are set, or if the effective page access changes.
+                 */
+                const uint64_t fPteSame = X86_PTE_P   | X86_PTE_RW | X86_PTE_US     | X86_PTE_PWT
+                                        | X86_PTE_PCD | X86_PTE_A  | X86_PTE_PAE_NX;
+                const uint64_t fPdeSame = X86_PDE_P   | X86_PDE_RW | X86_PDE_US     | X86_PDE_PWT
+                                        | X86_PDE_PCD | X86_PDE_A  | X86_PDE_PAE_NX | X86_PDE_PS;
+
+                if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->GCPtr >> X86_PD_PAE_SHIFT))
+                {
+                    if (pGstWalk->u.Amd64.pPte)
+                    {
+                        X86PTEPAE Pte;
+                        Pte.u = pGstWalk->u.Amd64.pPte[1].u;
+                        if (   (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
+                            && !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
+                        {
+                            pWalk->GCPtr  = GCPtr;
+                            pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
+                            pGstWalk->u.Amd64.Pte.u = Pte.u;
+                            pGstWalk->u.Amd64.pPte++;
+                            return VINF_SUCCESS;
+                        }
+                    }
+                }
+                else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->GCPtr >> X86_PDPT_SHIFT))
+                {
+                    Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
+                    if (pGstWalk->u.Amd64.pPde)
+                    {
+                        X86PDEPAE Pde;
+                        Pde.u = pGstWalk->u.Amd64.pPde[1].u;
+                        if (   (Pde.u & fPdeSame) == (pGstWalk->u.Amd64.Pde.u & fPdeSame)
+                            && !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
+                        {
+                            /* Get the new PTE and check out the first entry. */
+                            int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
+                                                               &pGstWalk->u.Amd64.pPt);
+                            if (RT_SUCCESS(rc))
+                            {
+                                pGstWalk->u.Amd64.pPte = &pGstWalk->u.Amd64.pPt->a[0];
+                                X86PTEPAE Pte;
+                                Pte.u = pGstWalk->u.Amd64.pPte->u;
+                                if (   (Pte.u & fPteSame) == (pGstWalk->u.Amd64.Pte.u & fPteSame)
+                                    && !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
+                                {
+                                    pWalk->GCPtr  = GCPtr;
+                                    pWalk->GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
+                                    pGstWalk->u.Amd64.Pte.u = Pte.u;
+                                    pGstWalk->u.Amd64.Pde.u = Pde.u;
+                                    pGstWalk->u.Amd64.pPde++;
+                                    return VINF_SUCCESS;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            else if (!pWalk->fGigantPage)
+            {
+                if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_2M_BASE_MASK))
+                {
+                    pWalk->GCPtr   = GCPtr;
+                    pWalk->GCPhys += GUEST_PAGE_SIZE;
+                    return VINF_SUCCESS;
+                }
+            }
+            else
+            {
+                if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->GCPtr & X86_PAGE_1G_BASE_MASK))
+                {
+                    pWalk->GCPtr   = GCPtr;
+                    pWalk->GCPhys += GUEST_PAGE_SIZE;
+                    return VINF_SUCCESS;
+                }
+            }
+        }
+    }
+    /* Case we don't handle.  Do full walk. */
+    return pgmGstPtWalk(pVCpu, GCPtr, pWalk, pGstWalk);
+}
+
+
+/**
+ * Modify page flags for a range of pages in the guest's tables
+ *
+ * The existing flags are ANDed with the fMask and ORed with the fFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range.
+ * @param   cb          Size (in bytes) of the range to apply the modification to.
+ * @param   fFlags      The OR  mask - page flags X86_PTE_*, excluding the page mask of course.
+ * @param   fMask       The AND mask - page flags X86_PTE_*, excluding the page mask of course.
+ *                      Be very CAREFUL when ~'ing constants which could be 32-bit!
+ */
+VMMDECL(int)  PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Validate input.
+     */
+    AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
+    Assert(cb);
+
+    LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
+
+    /*
+     * Adjust input.
+     */
+    cb     += GCPtr & GUEST_PAGE_OFFSET_MASK;
+    cb      = RT_ALIGN_Z(cb, GUEST_PAGE_SIZE);
+    GCPtr  &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
+
+    /*
+     * Call worker.
+     */
+    uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
+    AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
+    int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
+    return rc;
+}
+
+
+/**
+ * Checks whether the given PAE PDPEs are potentially valid for the guest.
+ *
+ * @returns @c true if the PDPE is valid, @c false otherwise.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   paPaePdpes  The PAE PDPEs to validate.
+ *
+ * @remarks This function -only- checks the reserved bits in the PDPE entries.
+ */
+VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
+{
+    Assert(paPaePdpes);
+    for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
+    {
+        X86PDPE const PaePdpe = paPaePdpes[i];
+        if (   !(PaePdpe.u & X86_PDPE_P)
+            || !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask))
+        { /* likely */ }
+        else
+            return false;
+    }
+    return true;
+}
+
+
+/**
+ * Performs the lazy mapping of the 32-bit guest PD.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   ppPd        Where to return the pointer to the mapping.  This is
+ *                      always set.
+ */
+int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
+{
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
+
+    Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
+
+    RTGCPHYS    GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
+    PPGMPAGE    pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
+        if (RT_SUCCESS(rc))
+        {
+# ifdef IN_RING3
+            pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
+            pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
+# else
+            pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
+            pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
+# endif
+            PGM_UNLOCK(pVM);
+            return VINF_SUCCESS;
+        }
+        AssertRC(rc);
+    }
+    PGM_UNLOCK(pVM);
+
+    *ppPd = NULL;
+    return rc;
+}
+
+
+/**
+ * Performs the lazy mapping of the PAE guest PDPT.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   ppPdpt      Where to return the pointer to the mapping.  This is
+ *                      always set.
+ */
+int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
+{
+    Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
+
+    RTGCPHYS    GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
+    PPGMPAGE    pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
+        if (RT_SUCCESS(rc))
+        {
+# ifdef IN_RING3
+            pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
+            pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
+# else
+            pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
+            pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
+# endif
+            PGM_UNLOCK(pVM);
+            return VINF_SUCCESS;
+        }
+        AssertRC(rc);
+    }
+
+    PGM_UNLOCK(pVM);
+    *ppPdpt = NULL;
+    return rc;
+}
+
+
+/**
+ * Performs the lazy mapping / updating of a PAE guest PD.
+ *
+ * @returns Pointer to the mapping.
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   iPdpt       Which PD entry to map (0..3).
+ * @param   ppPd        Where to return the pointer to the mapping.  This is
+ *                      always set.
+ */
+int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
+{
+    PVMCC           pVM         = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
+
+    PX86PDPT        pGuestPDPT  = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
+    Assert(pGuestPDPT);
+    Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P);
+    RTGCPHYS        GCPhys      = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
+    bool const      fChanged    = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
+
+    PPGMPAGE        pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
+        AssertRC(rc);
+        if (RT_SUCCESS(rc))
+        {
+# ifdef IN_RING3
+            pVCpu->pgm.s.apGstPaePDsR0[iPdpt]          = NIL_RTR0PTR;
+            pVCpu->pgm.s.apGstPaePDsR3[iPdpt]          = *ppPd;
+# else
+            pVCpu->pgm.s.apGstPaePDsR3[iPdpt]          = NIL_RTR3PTR;
+            pVCpu->pgm.s.apGstPaePDsR0[iPdpt]          = *ppPd;
+# endif
+            if (fChanged)
+                pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt]   = GCPhys;
+            PGM_UNLOCK(pVM);
+            return VINF_SUCCESS;
+        }
+    }
+
+    /* Invalid page or some failure, invalidate the entry. */
+    pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt]   = NIL_RTGCPHYS;
+    pVCpu->pgm.s.apGstPaePDsR3[iPdpt]      = NIL_RTR3PTR;
+    pVCpu->pgm.s.apGstPaePDsR0[iPdpt]      = NIL_RTR0PTR;
+
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+
+/**
+ * Performs the lazy mapping of the 32-bit guest PD.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   ppPml4      Where to return the pointer to the mapping.  This will
+ *                      always be set.
+ */
+int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
+{
+    Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
+
+    RTGCPHYS    GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, pVCpu->pgm.s.GCPhysCR3);
+    PPGMPAGE    pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
+        if (RT_SUCCESS(rc))
+        {
+# ifdef IN_RING3
+            pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
+            pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
+# else
+            pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
+            pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
+# endif
+            PGM_UNLOCK(pVM);
+            return VINF_SUCCESS;
+        }
+    }
+
+    PGM_UNLOCK(pVM);
+    *ppPml4 = NULL;
+    return rc;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+ /**
+ * Performs the lazy mapping of the guest PML4 table when using EPT paging.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   ppEptPml4   Where to return the pointer to the mapping.  This will
+ *                      always be set.
+ */
+int pgmGstLazyMapEptPml4(PVMCPUCC pVCpu, PEPTPML4 *ppEptPml4)
+{
+    Assert(!pVCpu->pgm.s.CTX_SUFF(pGstEptPml4));
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
+
+    RTGCPHYS const GCPhysEpt = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK;
+    PPGMPAGE       pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhysEpt, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysEpt, (void **)ppEptPml4);
+        if (RT_SUCCESS(rc))
+        {
+# ifdef IN_RING3
+            pVCpu->pgm.s.pGstEptPml4R0 = NIL_RTR0PTR;
+            pVCpu->pgm.s.pGstEptPml4R3 = *ppEptPml4;
+# else
+            pVCpu->pgm.s.pGstEptPml4R3 = NIL_RTR3PTR;
+            pVCpu->pgm.s.pGstEptPml4R0 = *ppEptPml4;
+# endif
+            PGM_UNLOCK(pVM);
+            return VINF_SUCCESS;
+        }
+    }
+
+    PGM_UNLOCK(pVM);
+    *ppEptPml4 = NULL;
+    return rc;
+}
+#endif
+
+
+/**
+ * Gets the current CR3 register value for the shadow memory context.
+ * @returns CR3 value.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
+{
+    PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
+    return pPoolPage->Core.Key;
+}
+
+
+/**
+ * Forces lazy remapping of the guest's PAE page-directory structures.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void pgmGstFlushPaePdpes(PVMCPU pVCpu)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++)
+    {
+        pVCpu->pgm.s.apGstPaePDsR3[i]     = 0;
+        pVCpu->pgm.s.apGstPaePDsR0[i]     = 0;
+        pVCpu->pgm.s.aGCPhysGstPaePDs[i]  = NIL_RTGCPHYS;
+    }
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * Performs second-level address translation for the given CR3 and updates the
+ * nested-guest CR3 when successful.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uCr3        The masked nested-guest CR3 value.
+ * @param   pGCPhysCR3  Where to store the translated CR3.
+ *
+ * @warning This updates PGMCPU::GCPhysNstGstCR3 when the translation succeeds. Be
+ *          mindful of this in code that's hyper sensitive to the order of
+ *          operations.
+ */
+static int pgmGstSlatTranslateCr3(PVMCPUCC pVCpu, uint64_t uCr3, PRTGCPHYS pGCPhysCr3)
+{
+    if (uCr3 != pVCpu->pgm.s.GCPhysNstGstCR3)
+    {
+        PGMPTWALK    Walk;
+        PGMPTWALKGST GstWalk;
+        int const rc = pgmGstSlatWalk(pVCpu, uCr3,  false /* fIsLinearAddrValid */,  0 /* GCPtrNested */, &Walk, &GstWalk);
+        if (RT_SUCCESS(rc))
+        {
+            /* Update nested-guest CR3. */
+            pVCpu->pgm.s.GCPhysNstGstCR3 = uCr3;
+
+            /* Pass back the translated result. */
+            *pGCPhysCr3 = Walk.GCPhys;
+            return VINF_SUCCESS;
+        }
+
+        /* Translation failed. */
+        *pGCPhysCr3 = NIL_RTGCPHYS;
+        return rc;
+    }
+
+    /*
+     * If the nested-guest CR3 has not changed, then the previously
+     * translated CR3 result (i.e. GCPhysCR3) is passed back.
+     */
+    *pGCPhysCr3 = pVCpu->pgm.s.GCPhysCR3;
+    return VINF_SUCCESS;
+}
+#endif
+
+
+/**
+ * Performs and schedules necessary updates following a CR3 load or reload.
+ *
+ * This will normally involve mapping the guest PD or nPDPT
+ *
+ * @returns VBox status code.
+ * @retval  VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
+ *          safely be ignored and overridden since the FF will be set too then.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cr3             The new cr3.
+ * @param   fGlobal         Indicates whether this is a global flush or not.
+ */
+VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Always flag the necessary updates; necessary for hardware acceleration
+     */
+    /** @todo optimize this, it shouldn't always be necessary. */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+    if (fGlobal)
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+    /*
+     * Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
+     */
+    RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
+    RTGCPHYS       GCPhysCR3    = pgmGetGuestMaskedCr3(pVCpu, cr3);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    if (   pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT
+        && PGMMODE_WITH_PAGING(pVCpu->pgm.s.enmGuestMode))
+    {
+        RTGCPHYS GCPhysOut;
+        int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
+        if (RT_SUCCESS(rc))
+            GCPhysCR3 = GCPhysOut;
+        else
+        {
+            /* CR3 SLAT translation failed but we try to pretend it
+               succeeded for the reasons mentioned in PGMHCChangeMode(). */
+            AssertMsgFailed(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
+            int const rc2 = pgmGstUnmapCr3(pVCpu);
+            pVCpu->pgm.s.GCPhysCR3       = NIL_RTGCPHYS;
+            pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
+            return rc2;
+        }
+    }
+#endif
+
+    LogFlowFunc(("cr3=%RX64 old=%RX64 fGlobal=%d\n", cr3, GCPhysOldCR3, fGlobal));
+    int rc = VINF_SUCCESS;
+    if (GCPhysOldCR3 != GCPhysCR3)
+    {
+        uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+        AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+        AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+
+        pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
+        rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
+        if (RT_LIKELY(rc == VINF_SUCCESS))
+        { }
+        else
+        {
+            AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
+            Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_PGM_SYNC_CR3));
+            pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped) = false;
+            pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
+            pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
+            pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_MAP_CR3;
+        }
+
+        if (fGlobal)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
+        else
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3));
+    }
+    else
+    {
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+        if (pPool->cDirtyPages)
+        {
+            PGM_LOCK_VOID(pVM);
+            pgmPoolResetDirtyPages(pVM);
+            PGM_UNLOCK(pVM);
+        }
+#endif
+        if (fGlobal)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
+        else
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3));
+
+        /*
+         * Flush PAE PDPTEs.
+         */
+        if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
+            pgmGstFlushPaePdpes(pVCpu);
+    }
+
+    IEMTlbInvalidateAll(pVCpu);
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
+    return rc;
+}
+
+
+/**
+ * Performs and schedules necessary updates following a CR3 load or reload when
+ * using nested or extended paging.
+ *
+ * This API is an alternative to PGMFlushTLB that avoids actually flushing the
+ * TLB and triggering a SyncCR3.
+ *
+ * This will normally involve mapping the guest PD or nPDPT
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS.
+ * @retval  VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
+ *          paging modes).  This can safely be ignored and overridden since the
+ *          FF will be set too then.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   cr3             The new CR3.
+ */
+VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /* We assume we're only called in nested paging mode. */
+    Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging || pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
+
+    /*
+     * Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
+     */
+    RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
+    RTGCPHYS       GCPhysCR3    = pgmGetGuestMaskedCr3(pVCpu, cr3);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
+    {
+        RTGCPHYS GCPhysOut;
+        int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
+        if (RT_SUCCESS(rc))
+            GCPhysCR3 = GCPhysOut;
+        else
+        {
+            /* CR3 SLAT translation failed but we try to pretend it
+               succeeded for the reasons mentioned in PGMHCChangeMode(). */
+            Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", cr3, rc));
+            int const rc2 = pgmGstUnmapCr3(pVCpu);
+            pVCpu->pgm.s.GCPhysCR3       = NIL_RTGCPHYS;
+            pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
+            return rc2;
+        }
+    }
+#endif
+
+    LogFlowFunc(("cr3=%RX64 old=%RX64\n", cr3, GCPhysOldCR3));
+    int rc = VINF_SUCCESS;
+    if (GCPhysOldCR3 != GCPhysCR3)
+    {
+        uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+        AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+        AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+
+        pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
+        rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
+
+        AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. */ /** @todo this isn't true for the mac, but we need hw to test/fix this. */
+    }
+    /*
+     * Flush PAE PDPTEs.
+     */
+    else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
+        pgmGstFlushPaePdpes(pVCpu);
+
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
+    return rc;
+}
+
+
+/**
+ * Synchronize the paging structures.
+ *
+ * This function is called in response to the VM_FF_PGM_SYNC_CR3 and
+ * VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
+ * in several places, most importantly whenever the CR3 is loaded.
+ *
+ * @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
+ * @retval  VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
+ *          the VMM into guest context.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cr0         Guest context CR0 register
+ * @param   cr3         Guest context CR3 register
+ * @param   cr4         Guest context CR4 register
+ * @param   fGlobal     Including global page directories or not
+ */
+VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
+{
+    int rc;
+
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * The pool may have pending stuff and even require a return to ring-3 to
+     * clear the whole thing.
+     */
+    rc = pgmPoolSyncCR3(pVCpu);
+    if (rc != VINF_SUCCESS)
+        return rc;
+
+    /*
+     * We might be called when we shouldn't.
+     *
+     * The mode switching will ensure that the PD is resynced after every mode
+     * switch.  So, if we find ourselves here when in protected or real mode
+     * we can safely clear the FF and return immediately.
+     */
+    if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
+    {
+        Assert((cr0 & (X86_CR0_PG | X86_CR0_PE)) != (X86_CR0_PG | X86_CR0_PE));
+        Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+        return VINF_SUCCESS;
+    }
+
+    /* If global pages are not supported, then all flushes are global. */
+    if (!(cr4 & X86_CR4_PGE))
+        fGlobal = true;
+    LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
+             VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
+
+    /*
+     * Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
+     * This should be done before SyncCR3.
+     */
+    if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
+    {
+        pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
+
+        RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
+        RTGCPHYS       GCPhysCR3    = pgmGetGuestMaskedCr3(pVCpu, cr3);
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
+        {
+            RTGCPHYS GCPhysOut;
+            int rc2 = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
+            if (RT_SUCCESS(rc2))
+                GCPhysCR3 = GCPhysOut;
+            else
+            {
+                /* CR3 SLAT translation failed but we try to pretend it
+                   succeeded for the reasons mentioned in PGMHCChangeMode(). */
+                AssertMsgFailed(("Failed to translate CR3 %#RX64. rc=%Rrc\n", cr3, rc2));
+                pVCpu->pgm.s.GCPhysCR3       = NIL_RTGCPHYS;
+                pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
+                return rc2;
+            }
+        }
+#endif
+        Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
+        if (GCPhysOldCR3 != GCPhysCR3)
+        {
+            uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+            AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+            AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+            pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
+            rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
+        }
+
+        /* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
+        if (    rc == VINF_PGM_SYNC_CR3
+            ||  (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
+        {
+            Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
+#ifdef IN_RING3
+            rc = pgmPoolSyncCR3(pVCpu);
+#else
+            if (rc == VINF_PGM_SYNC_CR3)
+                pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
+            return VINF_PGM_SYNC_CR3;
+#endif
+        }
+        AssertRCReturn(rc, rc);
+        AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
+    }
+
+    /*
+     * Let the 'Bth' function do the work and we'll just keep track of the flags.
+     */
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
+
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
+    rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
+    AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("rc=%Rrc\n", rc));
+    if (rc == VINF_SUCCESS)
+    {
+        if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
+        {
+            /* Go back to ring 3 if a pgm pool sync is again pending. */
+            return VINF_PGM_SYNC_CR3;
+        }
+
+        if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
+        {
+            Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+            VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+        }
+    }
+
+    /*
+     * Now flush the CR3 (guest context).
+     */
+    if (rc == VINF_SUCCESS)
+        PGM_INVL_VCPU_TLBS(pVCpu);
+    return rc;
+}
+
+
+/**
+ * Maps all the PAE PDPE entries.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   paPaePdpes  The new PAE PDPE values.
+ *
+ * @remarks This function may be invoked during the process of changing the guest
+ *          paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not
+ *          reflect PAE paging just yet.
+ */
+VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
+{
+    Assert(paPaePdpes);
+    for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
+    {
+        X86PDPE const PaePdpe = paPaePdpes[i];
+
+        /*
+         * In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs
+         * are deferred.[1] Also, different situations require different handling of invalid
+         * PDPE entries. Here we assume the caller has already validated or doesn't require
+         * validation of the PDPEs.
+         *
+         * In the case of nested EPT (i.e. for nested-guests), the PAE PDPEs have been
+         * validated by the VMX transition.
+         *
+         * [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
+         */
+        if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask | X86_PDPE_P)) == X86_PDPE_P)
+        {
+            PVMCC   pVM = pVCpu->CTX_SUFF(pVM);
+            RTHCPTR HCPtr;
+
+            RTGCPHYS GCPhys;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
+            {
+                PGMPTWALK      Walk;
+                PGMPTWALKGST   GstWalk;
+                RTGCPHYS const GCPhysNested = PaePdpe.u & X86_PDPE_PG_MASK;
+                int const rc = pgmGstSlatWalk(pVCpu, GCPhysNested, false /* fIsLinearAddrValid */, 0 /* GCPtrNested */,
+                                              &Walk, &GstWalk);
+                if (RT_SUCCESS(rc))
+                    GCPhys = Walk.GCPhys;
+                else
+                {
+                    /*
+                     * Second-level address translation of the PAE PDPE has failed but we must -NOT-
+                     * abort and return a failure now. This is because we're called from a Mov CRx
+                     * instruction (or similar operation). Let's just pretend success but flag that
+                     * we need to map this PDPE lazily later.
+                     *
+                     * See Intel spec. 25.3 "Changes to instruction behavior in VMX non-root operation".
+                     * See Intel spec. 28.3.1 "EPT Overview".
+                     */
+                    pVCpu->pgm.s.apGstPaePDsR3[i]    = 0;
+                    pVCpu->pgm.s.apGstPaePDsR0[i]    = 0;
+                    pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
+                    continue;
+                }
+            }
+            else
+#endif
+            {
+                GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK);
+            }
+
+            PGM_LOCK_VOID(pVM);
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+            AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR);
+            int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
+            PGM_UNLOCK(pVM);
+            if (RT_SUCCESS(rc))
+            {
+#ifdef IN_RING3
+                pVCpu->pgm.s.apGstPaePDsR3[i]    = (PX86PDPAE)HCPtr;
+                pVCpu->pgm.s.apGstPaePDsR0[i]    = NIL_RTR0PTR;
+#else
+                pVCpu->pgm.s.apGstPaePDsR3[i]    = NIL_RTR3PTR;
+                pVCpu->pgm.s.apGstPaePDsR0[i]    = (PX86PDPAE)HCPtr;
+#endif
+                pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
+                continue;
+            }
+            AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i));
+        }
+        pVCpu->pgm.s.apGstPaePDsR3[i]    = 0;
+        pVCpu->pgm.s.apGstPaePDsR0[i]    = 0;
+        pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates and maps the PDPT and PAE PDPEs referenced by the given CR3.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   cr3     The guest CR3 value.
+ *
+ * @remarks This function may be invoked during the process of changing the guest
+ *          paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect
+ *          PAE paging just yet.
+ */
+VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3)
+{
+    /*
+     * Read the page-directory-pointer table (PDPT) at CR3.
+     */
+    RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK);
+    PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
+    {
+        RTGCPHYS GCPhysOut;
+        int const rc = pgmGstSlatTranslateCr3(pVCpu, GCPhysCR3, &GCPhysOut);
+        if (RT_SUCCESS(rc))
+            GCPhysCR3 = GCPhysOut;
+        else
+        {
+            Log(("Failed to load CR3 at %#RX64. rc=%Rrc\n", GCPhysCR3, rc));
+            return rc;
+        }
+    }
+#endif
+
+    RTHCPTR HCPtrGuestCr3;
+    int rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCr3);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Validate the page-directory-pointer table entries (PDPE).
+         */
+        X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
+        memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes));
+        if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0]))
+        {
+            /*
+             * Map the PDPT.
+             * We deliberately don't update PGM's GCPhysCR3 here as it's expected
+             * that PGMFlushTLB will be called soon and only a change to CR3 then
+             * will cause the shadow page tables to be updated.
+             */
+#ifdef IN_RING3
+            pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3;
+            pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
+#else
+            pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
+            pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3;
+#endif
+
+            /*
+             * Update CPUM and map the 4 PAE PDPEs.
+             */
+            CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
+            rc = PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
+            if (RT_SUCCESS(rc))
+            {
+#ifdef IN_RING3
+                pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
+                pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
+#else
+                pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
+                pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
+#endif
+                pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
+            }
+        }
+        else
+            rc = VERR_PGM_PAE_PDPE_RSVD;
+    }
+    return rc;
+}
+
+
+/**
+ * Called whenever CR0 or CR4 in a way which may affect the paging mode.
+ *
+ * @returns VBox status code, with the following informational code for
+ *          VM scheduling.
+ * @retval  VINF_SUCCESS if the was no change, or it was successfully dealt with.
+ * @retval  VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cr0         The new cr0.
+ * @param   cr4         The new cr4.
+ * @param   efer        The new extended feature enable register.
+ * @param   fForce      Whether to force a mode change.
+ */
+VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer, bool fForce)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Calc the new guest mode.
+     *
+     * Note! We check PG before PE and without requiring PE because of the
+     *       special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
+     */
+    PGMMODE enmGuestMode;
+    if (cr0 & X86_CR0_PG)
+    {
+        if (!(cr4 & X86_CR4_PAE))
+        {
+            bool const fPse = !!(cr4 & X86_CR4_PSE);
+            if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
+                Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
+            pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
+            enmGuestMode = PGMMODE_32_BIT;
+        }
+        else if (!(efer & MSR_K6_EFER_LME))
+        {
+            if (!(efer & MSR_K6_EFER_NXE))
+                enmGuestMode = PGMMODE_PAE;
+            else
+                enmGuestMode = PGMMODE_PAE_NX;
+        }
+        else
+        {
+            if (!(efer & MSR_K6_EFER_NXE))
+                enmGuestMode = PGMMODE_AMD64;
+            else
+                enmGuestMode = PGMMODE_AMD64_NX;
+        }
+    }
+    else if (!(cr0 & X86_CR0_PE))
+        enmGuestMode = PGMMODE_REAL;
+    else
+        enmGuestMode = PGMMODE_PROTECTED;
+
+    /*
+     * Did it change?
+     */
+    if (   !fForce
+        && pVCpu->pgm.s.enmGuestMode == enmGuestMode)
+        return VINF_SUCCESS;
+
+    /* Flush the TLB */
+    PGM_INVL_VCPU_TLBS(pVCpu);
+    return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode, fForce);
+}
+
+
+/**
+ * Converts a PGMMODE value to a PGM_TYPE_* \#define.
+ *
+ * @returns PGM_TYPE_*.
+ * @param   pgmMode     The mode value to convert.
+ */
+DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
+{
+    switch (pgmMode)
+    {
+        case PGMMODE_REAL:          return PGM_TYPE_REAL;
+        case PGMMODE_PROTECTED:     return PGM_TYPE_PROT;
+        case PGMMODE_32_BIT:        return PGM_TYPE_32BIT;
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX:        return PGM_TYPE_PAE;
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:      return PGM_TYPE_AMD64;
+        case PGMMODE_NESTED_32BIT:  return PGM_TYPE_NESTED_32BIT;
+        case PGMMODE_NESTED_PAE:    return PGM_TYPE_NESTED_PAE;
+        case PGMMODE_NESTED_AMD64:  return PGM_TYPE_NESTED_AMD64;
+        case PGMMODE_EPT:           return PGM_TYPE_EPT;
+        case PGMMODE_NONE:          return PGM_TYPE_NONE;
+        default:
+            AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
+    }
+}
+
+
+/**
+ * Calculates the shadow paging mode.
+ *
+ * @returns The shadow paging mode.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmGuestMode    The guest mode.
+ * @param   enmHostMode     The host mode.
+ * @param   enmShadowMode   The current shadow mode.
+ */
+static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
+{
+    switch (enmGuestMode)
+    {
+        case PGMMODE_REAL:
+        case PGMMODE_PROTECTED:
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_32_BIT;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                default:
+                    AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
+            }
+            break;
+
+        case PGMMODE_32_BIT:
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_32_BIT;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                default:
+                    AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
+            }
+            break;
+
+        case PGMMODE_PAE:
+        case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_PAE;
+                    break;
+
+                default:
+                    AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
+            }
+            break;
+
+        case PGMMODE_AMD64:
+        case PGMMODE_AMD64_NX:
+            switch (enmHostMode)
+            {
+                case SUPPAGINGMODE_32_BIT:
+                case SUPPAGINGMODE_32_BIT_GLOBAL:
+                    enmShadowMode = PGMMODE_AMD64;
+                    break;
+
+                case SUPPAGINGMODE_PAE:
+                case SUPPAGINGMODE_PAE_NX:
+                case SUPPAGINGMODE_PAE_GLOBAL:
+                case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_AMD64;
+                    break;
+
+                case SUPPAGINGMODE_AMD64:
+                case SUPPAGINGMODE_AMD64_GLOBAL:
+                case SUPPAGINGMODE_AMD64_NX:
+                case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                    enmShadowMode = PGMMODE_AMD64;
+                    break;
+
+                default:
+                    AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
+            }
+            break;
+
+        default:
+            AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
+    }
+
+    /*
+     * Override the shadow mode when NEM, IEM or nested paging is active.
+     */
+    if (!VM_IS_HM_ENABLED(pVM))
+    {
+        Assert(VM_IS_NEM_ENABLED(pVM) || VM_IS_EXEC_ENGINE_IEM(pVM));
+        pVM->pgm.s.fNestedPaging = true;
+        enmShadowMode = PGMMODE_NONE;
+    }
+    else
+    {
+        bool fNestedPaging = HMIsNestedPagingActive(pVM);
+        pVM->pgm.s.fNestedPaging = fNestedPaging;
+        if (fNestedPaging)
+        {
+            if (HMIsVmxActive(pVM))
+                enmShadowMode = PGMMODE_EPT;
+            else
+            {
+                /* The nested SVM paging depends on the host one. */
+                Assert(HMIsSvmActive(pVM));
+                if (   enmGuestMode == PGMMODE_AMD64
+                    || enmGuestMode == PGMMODE_AMD64_NX)
+                    enmShadowMode = PGMMODE_NESTED_AMD64;
+                else
+                    switch (pVM->pgm.s.enmHostMode)
+                    {
+                        case SUPPAGINGMODE_32_BIT:
+                        case SUPPAGINGMODE_32_BIT_GLOBAL:
+                            enmShadowMode = PGMMODE_NESTED_32BIT;
+                            break;
+
+                        case SUPPAGINGMODE_PAE:
+                        case SUPPAGINGMODE_PAE_GLOBAL:
+                        case SUPPAGINGMODE_PAE_NX:
+                        case SUPPAGINGMODE_PAE_GLOBAL_NX:
+                            enmShadowMode = PGMMODE_NESTED_PAE;
+                            break;
+
+                        case SUPPAGINGMODE_AMD64:
+                        case SUPPAGINGMODE_AMD64_GLOBAL:
+                        case SUPPAGINGMODE_AMD64_NX:
+                        case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+                            enmShadowMode = PGMMODE_NESTED_AMD64;
+                            break;
+
+                        default:
+                            AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
+                    }
+            }
+        }
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        else
+        {
+            /* Nested paging is a requirement for nested VT-x. */
+            AssertLogRelMsgReturn(enmGuestMode != PGMMODE_EPT, ("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
+        }
+#endif
+    }
+
+    return enmShadowMode;
+}
+
+
+/**
+ * Performs the actual mode change.
+ * This is called by PGMChangeMode and pgmR3InitPaging().
+ *
+ * @returns VBox status code. May suspend or power off the VM on error, but this
+ *          will trigger using FFs and not informational status codes.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   enmGuestMode    The new guest mode. This is assumed to be different from
+ *                          the current mode.
+ * @param   fForce          Whether to force a shadow paging mode change.
+ */
+VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode, bool fForce)
+{
+    Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
+    STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
+
+    /*
+     * Calc the shadow mode and switcher.
+     */
+    PGMMODE const enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
+    bool const fShadowModeChanged = enmShadowMode != pVCpu->pgm.s.enmShadowMode || fForce;
+
+    /*
+     * Exit old mode(s).
+     */
+    /* shadow */
+    if (fShadowModeChanged)
+    {
+        LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
+        uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
+        if (   idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
+            && g_aPgmShadowModeData[idxOldShw].pfnExit)
+        {
+            int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
+            AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
+        }
+    }
+    else
+        LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n",  PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
+
+    /* guest */
+    uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
+    if (   idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
+        && g_aPgmGuestModeData[idxOldGst].pfnExit)
+    {
+        int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
+        AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
+    }
+    pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
+    pVCpu->pgm.s.GCPhysNstGstCR3 = NIL_RTGCPHYS;
+    pVCpu->pgm.s.GCPhysPaeCR3 = NIL_RTGCPHYS;
+    Assert(!pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped));
+
+    /*
+     * Change the paging mode data indexes.
+     */
+    uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
+    AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
+#ifdef IN_RING3
+    AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
+#endif
+
+    uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
+    AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
+#ifdef IN_RING3
+    AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
+#endif
+
+    uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
+    AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
+#ifdef VBOX_STRICT
+    AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
+#endif
+
+    /*
+     * Determine SLAT mode -before- entering the new shadow mode!
+     */
+    pVCpu->pgm.s.enmGuestSlatMode = !CPUMIsGuestVmxEptPagingEnabled(pVCpu) ? PGMSLAT_DIRECT : PGMSLAT_EPT;
+
+    /*
+     * Enter new shadow mode (if changed).
+     */
+    if (fShadowModeChanged)
+    {
+        pVCpu->pgm.s.enmShadowMode = enmShadowMode;
+        int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu);
+        AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
+                                    pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
+    }
+
+    /*
+     * Always flag the necessary updates
+     */
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+
+    /*
+     * Enter the new guest and shadow+guest modes.
+     */
+    /* Calc the new CR3 value. */
+    RTGCPHYS GCPhysCR3;
+    switch (enmGuestMode)
+    {
+        case PGMMODE_REAL:
+        case PGMMODE_PROTECTED:
+            GCPhysCR3 = NIL_RTGCPHYS;
+            break;
+
+        case PGMMODE_32_BIT:
+            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
+            break;
+
+        case PGMMODE_PAE_NX:
+        case PGMMODE_PAE:
+            if (!pVM->cpum.ro.GuestFeatures.fPae)
+#ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */
+                return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
+                                         N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
+#else
+                AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE);
+
+#endif
+            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
+            break;
+
+#ifdef VBOX_WITH_64_BITS_GUESTS
+        case PGMMODE_AMD64_NX:
+        case PGMMODE_AMD64:
+            GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
+            break;
+#endif
+        default:
+            AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    /*
+     * If a nested-guest is using EPT paging:
+     *   - Update the second-level address translation (SLAT) mode.
+     *   - Indicate that the CR3 is nested-guest physical address.
+     */
+    if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
+    {
+        if (PGMMODE_WITH_PAGING(enmGuestMode))
+        {
+            /*
+             * Translate CR3 to its guest-physical address.
+             * We don't use pgmGstSlatTranslateCr3() here as we want to update GCPhysNstGstCR3 -after-
+             * switching modes to keep it consistent with how GCPhysCR3 is updated.
+             */
+            PGMPTWALK    Walk;
+            PGMPTWALKGST GstWalk;
+            int const rc = pgmGstSlatWalk(pVCpu, GCPhysCR3, false /* fIsLinearAddrValid */, 0 /* GCPtrNested */, &Walk,
+                                          &GstWalk);
+            if (RT_SUCCESS(rc))
+            { /* likely */ }
+            else
+            {
+                /*
+                 * SLAT failed but we avoid reporting this to the caller because the caller
+                 * is not supposed to fail. The only time the caller needs to indicate a
+                 * failure to software is when PAE paging is used by the nested-guest, but
+                 * we handle the PAE case separately (e.g., see VMX transition in IEM).
+                 * In all other cases, the failure will be indicated when CR3 tries to be
+                 * translated on the next linear-address memory access.
+                 * See Intel spec. 27.2.1 "EPT Overview".
+                 */
+                Log(("SLAT failed for CR3 %#RX64 rc=%Rrc\n", GCPhysCR3, rc));
+
+                /* Trying to coax PGM to succeed for the time being... */
+                Assert(pVCpu->pgm.s.GCPhysCR3 == NIL_RTGCPHYS);
+                pVCpu->pgm.s.GCPhysNstGstCR3  = GCPhysCR3;
+                pVCpu->pgm.s.enmGuestMode     = enmGuestMode;
+                HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
+                return VINF_SUCCESS;
+            }
+            pVCpu->pgm.s.GCPhysNstGstCR3  = GCPhysCR3;
+            GCPhysCR3 = Walk.GCPhys & X86_CR3_EPT_PAGE_MASK;
+        }
+    }
+    else
+        Assert(pVCpu->pgm.s.GCPhysNstGstCR3 == NIL_RTGCPHYS);
+#endif
+
+    /*
+     * Enter the new guest mode.
+     */
+    pVCpu->pgm.s.enmGuestMode = enmGuestMode;
+    int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
+    int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
+
+    /* Set the new guest CR3 (and nested-guest CR3). */
+    pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
+
+    /* status codes. */
+    AssertRC(rc);
+    AssertRC(rc2);
+    if (RT_SUCCESS(rc))
+    {
+        rc = rc2;
+        if (RT_SUCCESS(rc)) /* no informational status codes. */
+            rc = VINF_SUCCESS;
+    }
+
+    /*
+     * Notify HM.
+     */
+    HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
+    return rc;
+}
+
+
+/**
+ * Called by CPUM or REM when CR0.WP changes to 1.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @thread  EMT
+ */
+VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
+{
+    /*
+     * Netware WP0+RO+US hack cleanup when WP0 -> WP1.
+     *
+     * Use the counter to judge whether there might be pool pages with active
+     * hacks in them.  If there are, we will be running the risk of messing up
+     * the guest by allowing it to write to read-only pages.  Thus, we have to
+     * clear the page pool ASAP if there is the slightest chance.
+     */
+    if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
+    {
+        Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
+
+        Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
+        pVCpu->pgm.s.cNetwareWp0Hacks = 0;
+        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    }
+}
+
+
+/**
+ * Gets the current guest paging mode.
+ *
+ * If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
+ *
+ * @returns The current paging mode.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
+{
+    return pVCpu->pgm.s.enmGuestMode;
+}
+
+
+/**
+ * Gets the current shadow paging mode.
+ *
+ * @returns The current paging mode.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
+{
+    return pVCpu->pgm.s.enmShadowMode;
+}
+
+
+/**
+ * Gets the current host paging mode.
+ *
+ * @returns The current paging mode.
+ * @param   pVM             The cross context VM structure.
+ */
+VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
+{
+    switch (pVM->pgm.s.enmHostMode)
+    {
+        case SUPPAGINGMODE_32_BIT:
+        case SUPPAGINGMODE_32_BIT_GLOBAL:
+            return PGMMODE_32_BIT;
+
+        case SUPPAGINGMODE_PAE:
+        case SUPPAGINGMODE_PAE_GLOBAL:
+            return PGMMODE_PAE;
+
+        case SUPPAGINGMODE_PAE_NX:
+        case SUPPAGINGMODE_PAE_GLOBAL_NX:
+            return PGMMODE_PAE_NX;
+
+        case SUPPAGINGMODE_AMD64:
+        case SUPPAGINGMODE_AMD64_GLOBAL:
+            return PGMMODE_AMD64;
+
+        case SUPPAGINGMODE_AMD64_NX:
+        case SUPPAGINGMODE_AMD64_GLOBAL_NX:
+            return PGMMODE_AMD64_NX;
+
+        default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
+    }
+
+    return PGMMODE_INVALID;
+}
+
+
+/**
+ * Get mode name.
+ *
+ * @returns read-only name string.
+ * @param   enmMode     The mode which name is desired.
+ */
+VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
+{
+    switch (enmMode)
+    {
+        case PGMMODE_REAL:          return "Real";
+        case PGMMODE_PROTECTED:     return "Protected";
+        case PGMMODE_32_BIT:        return "32-bit";
+        case PGMMODE_PAE:           return "PAE";
+        case PGMMODE_PAE_NX:        return "PAE+NX";
+        case PGMMODE_AMD64:         return "AMD64";
+        case PGMMODE_AMD64_NX:      return "AMD64+NX";
+        case PGMMODE_NESTED_32BIT:  return "Nested-32";
+        case PGMMODE_NESTED_PAE:    return "Nested-PAE";
+        case PGMMODE_NESTED_AMD64:  return "Nested-AMD64";
+        case PGMMODE_EPT:           return "EPT";
+        case PGMMODE_NONE:          return "None";
+        default:                    return "unknown mode value";
+    }
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * Gets the SLAT mode name.
+ *
+ * @returns The read-only SLAT mode descriptive string.
+ * @param   enmSlatMode     The SLAT mode value.
+ */
+VMM_INT_DECL(const char *) PGMGetSlatModeName(PGMSLAT enmSlatMode)
+{
+    switch (enmSlatMode)
+    {
+        case PGMSLAT_DIRECT:        return "Direct";
+        case PGMSLAT_EPT:           return "EPT";
+        case PGMSLAT_32BIT:         return "32-bit";
+        case PGMSLAT_PAE:           return "PAE";
+        case PGMSLAT_AMD64:         return "AMD64";
+        default:                    return "Unknown";
+    }
+}
+#endif  /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+
+/**
+ * Gets the physical address represented in the guest CR3 as PGM sees it.
+ *
+ * This is mainly for logging and debugging.
+ *
+ * @returns PGM's guest CR3 value.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
+{
+    return pVCpu->pgm.s.GCPhysCR3;
+}
+
+
+
+/**
+ * Notification from CPUM that the EFER.NXE bit has changed.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the CPU for
+ *                      which EFER changed.
+ * @param   fNxe        The new NXE state.
+ */
+VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
+{
+/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
+    Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
+
+    pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
+    if (fNxe)
+    {
+        /*pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A */
+        pVCpu->pgm.s.fGstPaeMbzPteMask       &= ~X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGstPaeMbzPdeMask       &= ~X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGstPaeMbzBigPdeMask    &= ~X86_PDE2M_PAE_NX;
+        /*pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A */
+        pVCpu->pgm.s.fGstAmd64MbzPteMask     &= ~X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdeMask     &= ~X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdeMask  &= ~X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdpeMask    &= ~X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPml4eMask   &= ~X86_PML4E_NX;
+
+        pVCpu->pgm.s.fGst64ShadowedPteMask        |= X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedPdeMask        |= X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedBigPdeMask     |= X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask |= X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask    |= X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask   |= X86_PML4E_NX;
+    }
+    else
+    {
+        /*pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A */
+        pVCpu->pgm.s.fGstPaeMbzPteMask       |= X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGstPaeMbzPdeMask       |= X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGstPaeMbzBigPdeMask    |= X86_PDE2M_PAE_NX;
+        /*pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A */
+        pVCpu->pgm.s.fGstAmd64MbzPteMask     |= X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdeMask     |= X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdeMask  |= X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPdpeMask    |= X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask |= X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64MbzPml4eMask   |= X86_PML4E_NX;
+
+        pVCpu->pgm.s.fGst64ShadowedPteMask        &= ~X86_PTE_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedPdeMask        &= ~X86_PDE_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedBigPdeMask     &= ~X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
+        pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask    &= ~X86_PDPE_LM_NX;
+        pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask   &= ~X86_PML4E_NX;
+    }
+}
+
+
+/**
+ * Check if any pgm pool pages are marked dirty (not monitored)
+ *
+ * @returns bool locked/not locked
+ * @param   pVM         The cross context VM structure.
+ */
+VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
+{
+    return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
+}
+
+
+/**
+ * Check if this VCPU currently owns the PGM lock.
+ *
+ * @returns bool owner/not owner
+ * @param   pVM         The cross context VM structure.
+ */
+VMMDECL(bool) PGMIsLockOwner(PVMCC pVM)
+{
+    return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX);
+}
+
+
+/**
+ * Enable or disable large page usage
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   fUseLargePages  Use/not use large pages
+ */
+VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
+
+    pVM->pgm.s.fUseLargePages = fUseLargePages;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Acquire the PGM lock.
+ *
+ * @returns VBox status code
+ * @param   pVM         The cross context VM structure.
+ * @param   fVoid       Set if the caller cannot handle failure returns.
+ * @param   SRC_POS     The source position of the caller (RT_SRC_POS).
+ */
+#if defined(VBOX_STRICT) || defined(DOXYGEN_RUNNING)
+int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL)
+#else
+int pgmLock(PVMCC pVM, bool fVoid)
+#endif
+{
+#if defined(VBOX_STRICT)
+    int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
+#else
+    int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS);
+#endif
+    if (RT_SUCCESS(rc))
+        return rc;
+    if (fVoid)
+        PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc);
+    else
+        AssertRC(rc);
+    return rc;
+}
+
+
+/**
+ * Release the PGM lock.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+void pgmUnlock(PVMCC pVM)
+{
+    uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
+    pVM->pgm.s.cDeprecatedPageLocks = 0;
+    int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX);
+    if (rc == VINF_SEM_NESTED)
+        pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
+}
+
+
+#if !defined(IN_R0) || defined(LOG_ENABLED)
+
+/** Format handler for PGMPAGE.
+ * @copydoc FNRTSTRFORMATTYPE */
+static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
+                                                     const char *pszType, void const *pvValue,
+                                                     int cchWidth, int cchPrecision, unsigned fFlags,
+                                                     void *pvUser)
+{
+    size_t    cch;
+    PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
+    if (RT_VALID_PTR(pPage))
+    {
+        char szTmp[64+80];
+
+        cch = 0;
+
+        /* The single char state stuff. */
+        static const char s_achPageStates[4]    = { 'Z', 'A', 'W', 'S' };
+        szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
+
+# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) || cchPrecision == (lvl) || cchPrecision >= (lvl)+10 )
+        if (IS_PART_INCLUDED(5))
+        {
+            static const char s_achHandlerStates[4*2] = { '-', 't', 'w', 'a' , '_', 'T', 'W', 'A' };
+            szTmp[cch++] = s_achHandlerStates[  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)
+                                              | ((uint8_t)PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage) << 2)];
+        }
+
+        /* The type. */
+        if (IS_PART_INCLUDED(4))
+        {
+            szTmp[cch++] = ':';
+            static const char s_achPageTypes[8][4]  = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
+            szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
+            szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
+            szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
+        }
+
+        /* The numbers. */
+        if (IS_PART_INCLUDED(3))
+        {
+            szTmp[cch++] = ':';
+            cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD | RTSTR_F_64BIT);
+        }
+
+        if (IS_PART_INCLUDED(2))
+        {
+            szTmp[cch++] = ':';
+            cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD | RTSTR_F_32BIT);
+        }
+
+        if (IS_PART_INCLUDED(6))
+        {
+            szTmp[cch++] = ':';
+            static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
+            szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
+            cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD | RTSTR_F_16BIT);
+        }
+# undef IS_PART_INCLUDED
+
+        cch = pfnOutput(pvArgOutput, szTmp, cch);
+    }
+    else
+        cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
+    NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
+    return cch;
+}
+
+
+/** Format handler for PGMRAMRANGE.
+ * @copydoc FNRTSTRFORMATTYPE */
+static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
+                                                         const char *pszType, void const *pvValue,
+                                                         int cchWidth, int cchPrecision, unsigned fFlags,
+                                                         void *pvUser)
+{
+    size_t              cch;
+    PGMRAMRANGE const  *pRam = (PGMRAMRANGE const *)pvValue;
+    if (RT_VALID_PTR(pRam))
+    {
+        char szTmp[80];
+        cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
+        cch = pfnOutput(pvArgOutput, szTmp, cch);
+    }
+    else
+        cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
+    NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
+    return cch;
+}
+
+/** Format type andlers to be registered/deregistered. */
+static const struct
+{
+    char                szType[24];
+    PFNRTSTRFORMATTYPE  pfnHandler;
+} g_aPgmFormatTypes[] =
+{
+    { "pgmpage",        pgmFormatTypeHandlerPage },
+    { "pgmramrange",    pgmFormatTypeHandlerRamRange }
+};
+
+#endif /* !IN_R0 || LOG_ENABLED */
+
+/**
+ * Registers the global string format types.
+ *
+ * This should be called at module load time or in some other manner that ensure
+ * that it's called exactly one time.
+ *
+ * @returns IPRT status code on RTStrFormatTypeRegister failure.
+ */
+VMMDECL(int) PGMRegisterStringFormatTypes(void)
+{
+#if !defined(IN_R0) || defined(LOG_ENABLED)
+    int         rc = VINF_SUCCESS;
+    unsigned    i;
+    for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
+    {
+        rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
+# ifdef IN_RING0
+        if (rc == VERR_ALREADY_EXISTS)
+        {
+            /* in case of cleanup failure in ring-0 */
+            RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
+            rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
+        }
+# endif
+    }
+    if (RT_FAILURE(rc))
+        while (i-- > 0)
+            RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
+
+    return rc;
+#else
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Deregisters the global string format types.
+ *
+ * This should be called at module unload time or in some other manner that
+ * ensure that it's called exactly one time.
+ */
+VMMDECL(void) PGMDeregisterStringFormatTypes(void)
+{
+#if !defined(IN_R0) || defined(LOG_ENABLED)
+    for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
+        RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
+#endif
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * Asserts that everything related to the guest CR3 is correctly shadowed.
+ *
+ * This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
+ * and assert the correctness of the guest CR3 mapping before asserting that the
+ * shadow page tables is in sync with the guest page tables.
+ *
+ * @returns Number of conflicts.
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   cr3     The current guest CR3 register value.
+ * @param   cr4     The current guest CR4 register value.
+ */
+VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
+{
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
+
+    uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
+
+    PGM_LOCK_VOID(pVM);
+    unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
+    PGM_UNLOCK(pVM);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
+    return cErrors;
+}
+#endif /* VBOX_STRICT */
+
+
+/**
+ * Updates PGM's copy of the guest's EPT pointer.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uEptPtr     The EPT pointer.
+ *
+ * @remarks This can be called as part of VM-entry so we might be in the midst of
+ *          switching to VMX non-root mode.
+ */
+VMM_INT_DECL(void) PGMSetGuestEptPtr(PVMCPUCC pVCpu, uint64_t uEptPtr)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
+    pVCpu->pgm.s.uEptPtr = uEptPtr;
+    pVCpu->pgm.s.pGstEptPml4R3 = 0;
+    pVCpu->pgm.s.pGstEptPml4R0 = 0;
+    PGM_UNLOCK(pVM);
+}
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllBth.h b/src/VBox/VMM/VMMAll/PGMAllBth.h
new file mode 100644
index 00000000..50b7a30e
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllBth.h
@@ -0,0 +1,5268 @@
+/* $Id: PGMAllBth.h $ */
+/** @file
+ * VBox - Page Manager, Shadow+Guest Paging Template - All context code.
+ *
+ * @remarks Extended page tables (intel) are built with PGM_GST_TYPE set to
+ *          PGM_TYPE_PROT (and PGM_SHW_TYPE set to PGM_TYPE_EPT).
+ *          bird: WTF does this mean these days?  Looking at PGMAll.cpp it's
+ *
+ * @remarks This file is one big \#ifdef-orgy!
+ *
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#ifdef _MSC_VER
+/** @todo we're generating unnecessary code in nested/ept shadow mode and for
+ *        real/prot-guest+RC mode. */
+# pragma warning(disable: 4505)
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
+#ifndef IN_RING3
+PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken);
+PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
+                                       bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken);
+# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
+static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
+                                               unsigned iPte, PCPGMPTWALKGST pGstWalkAll);
+static int  PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
+                                         uint32_t uErr, PPGMPTWALKGST pGstWalkAll);
+static int  PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll);
+# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+#endif
+PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
+static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr);
+static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc, RTGCPTR GCPtrPage);
+static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPD, PGSTPD pPDSrc, RTGCPTR GCPtrPage);
+#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
+#else
+static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage, unsigned iPTDst);
+#endif
+PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR Addr, unsigned fPage, unsigned uErr);
+PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage);
+PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal);
+#ifdef VBOX_STRICT
+PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr = 0, RTGCPTR cb = ~(RTGCPTR)0);
+#endif
+PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
+PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu);
+
+#ifdef IN_RING3
+PGM_BTH_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
+#endif
+RT_C_DECLS_END
+
+
+
+
+/*
+ * Filter out some illegal combinations of guest and shadow paging, so we can
+ * remove redundant checks inside functions.
+ */
+#if    PGM_GST_TYPE == PGM_TYPE_PAE && PGM_SHW_TYPE != PGM_TYPE_PAE \
+    && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
+# error "Invalid combination; PAE guest implies PAE shadow"
+#endif
+
+#if    (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
+    && !(   PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64 \
+         || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
+# error "Invalid combination; real or protected mode without paging implies 32 bits or PAE shadow paging."
+#endif
+
+#if     (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) \
+    && !(   PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_PAE \
+         || PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE)
+# error "Invalid combination; 32 bits guest paging or PAE implies 32 bits or PAE shadow paging."
+#endif
+
+#if    (PGM_GST_TYPE == PGM_TYPE_AMD64 && PGM_SHW_TYPE != PGM_TYPE_AMD64 && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE) \
+    || (PGM_SHW_TYPE == PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PROT)
+# error "Invalid combination; AMD64 guest implies AMD64 shadow and vice versa"
+#endif
+
+
+/**
+ * Enters the shadow+guest mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPhysCR3   The physical address from the CR3 register.
+ */
+PGM_BTH_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
+{
+    /* Here we deal with allocation of the root shadow page table for real and protected mode during mode switches;
+     * Other modes rely on MapCR3/UnmapCR3 to setup the shadow root page tables.
+     */
+#if  (   (   PGM_SHW_TYPE == PGM_TYPE_32BIT \
+          || PGM_SHW_TYPE == PGM_TYPE_PAE    \
+          || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
+      && (   PGM_GST_TYPE == PGM_TYPE_REAL   \
+          || PGM_GST_TYPE == PGM_TYPE_PROT))
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    Assert(!pVM->pgm.s.fNestedPaging);
+
+    PGM_LOCK_VOID(pVM);
+    /* Note: we only really need shadow paging in real and protected mode for VT-x and AMD-V (excluding nested paging/EPT modes),
+     *       but any calls to GC need a proper shadow page setup as well.
+     */
+    /* Free the previous root mapping if still active. */
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPAGE pOldShwPageCR3 = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    if (pOldShwPageCR3)
+    {
+        Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
+
+        /* Mark the page as unlocked; allow flushing again. */
+        pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+
+        pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
+        pVCpu->pgm.s.pShwPageCR3R3 = NIL_RTR3PTR;
+        pVCpu->pgm.s.pShwPageCR3R0 = NIL_RTR0PTR;
+    }
+
+    /* construct a fake address. */
+    GCPhysCR3 = RT_BIT_64(63);
+    PPGMPOOLPAGE pNewShwPageCR3;
+    int rc = pgmPoolAlloc(pVM, GCPhysCR3, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          NIL_PGMPOOL_IDX, UINT32_MAX, false /*fLockPage*/,
+                          &pNewShwPageCR3);
+    AssertRCReturn(rc, rc);
+
+    pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
+    pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
+
+    /* Mark the page as locked; disallow flushing. */
+    pgmPoolLockPage(pPool, pNewShwPageCR3);
+
+    /* Set the current hypervisor CR3. */
+    CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
+
+    PGM_UNLOCK(pVM);
+    return rc;
+#else
+    NOREF(pVCpu); NOREF(GCPhysCR3);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+#ifndef IN_RING3
+
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+/**
+ * Deal with a guest page fault.
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_EM_RAW_GUEST_TRAP
+ * @retval  VINF_EM_RAW_EMULATE_INSTR
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pWalk           The guest page table walk result.
+ * @param   uErr            The error code.
+ */
+PGM_BTH_DECL(VBOXSTRICTRC, Trap0eHandlerGuestFault)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, RTGCUINT uErr)
+{
+    /*
+     * Calc the error code for the guest trap.
+     */
+    uint32_t uNewErr = GST_IS_NX_ACTIVE(pVCpu)
+                     ? uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID)
+                     : uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US);
+    if (   pWalk->fRsvdError
+        || pWalk->fBadPhysAddr)
+    {
+        uNewErr |= X86_TRAP_PF_RSVD | X86_TRAP_PF_P;
+        Assert(!pWalk->fNotPresent);
+    }
+    else if (!pWalk->fNotPresent)
+        uNewErr |= X86_TRAP_PF_P;
+    TRPMSetErrorCode(pVCpu, uNewErr);
+
+    LogFlow(("Guest trap; cr2=%RGv uErr=%RGv lvl=%d\n", pWalk->GCPtr, uErr, pWalk->uLevel));
+    STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
+    return VINF_EM_RAW_GUEST_TRAP;
+}
+# endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+
+
+#if !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
+/**
+ * Deal with a guest page fault.
+ *
+ * The caller has taken the PGM lock.
+ *
+ * @returns Strict VBox status code.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   uErr            The error code.
+ * @param   pCtx            Pointer to the register context for the CPU.
+ * @param   pvFault         The fault address.
+ * @param   pPage           The guest page at @a pvFault.
+ * @param   pWalk           The guest page table walk result.
+ * @param   pGstWalk        The guest paging-mode specific walk information.
+ * @param   pfLockTaken     PGM lock taken here or not (out).  This is true
+ *                          when we're called.
+ */
+static VBOXSTRICTRC PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
+                                                                RTGCPTR pvFault, PPGMPAGE pPage, bool *pfLockTaken
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
+                                                                , PPGMPTWALK pWalk
+                                                                , PGSTPTWALK pGstWalk
+# endif
+                                                                )
+{
+# if !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    GSTPDE const    PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A };
+# endif
+    PVMCC           pVM         = pVCpu->CTX_SUFF(pVM);
+    VBOXSTRICTRC    rcStrict;
+
+    if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
+    {
+        /*
+         * Physical page access handler.
+         */
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        const RTGCPHYS  GCPhysFault = pWalk->GCPhys;
+# else
+        const RTGCPHYS  GCPhysFault = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault);
+# endif
+        PPGMPHYSHANDLER pCur;
+        rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysFault, &pCur);
+        if (RT_SUCCESS(rcStrict))
+        {
+            PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+
+#  ifdef PGM_SYNC_N_PAGES
+            /*
+             * If the region is write protected and we got a page not present fault, then sync
+             * the pages. If the fault was caused by a read, then restart the instruction.
+             * In case of write access continue to the GC write handler.
+             *
+             * ASSUMES that there is only one handler per page or that they have similar write properties.
+             */
+            if (   !(uErr & X86_TRAP_PF_P)
+                &&  pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
+            {
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+                rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   else
+                rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   endif
+                if (    RT_FAILURE(rcStrict)
+                    || !(uErr & X86_TRAP_PF_RW)
+                    || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
+                {
+                    AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
+                    STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
+                    return rcStrict;
+                }
+            }
+#  endif
+#  ifdef PGM_WITH_MMIO_OPTIMIZATIONS
+            /*
+             * If the access was not thru a #PF(RSVD|...) resync the page.
+             */
+            if (   !(uErr & X86_TRAP_PF_RSVD)
+                && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+                && (pWalk->fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK))
+                                      == PGM_PTATTRS_W_MASK  /** @todo Remove pGstWalk->Core.fEffectiveUS and X86_PTE_US further down in the sync code. */
+#   endif
+               )
+            {
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+                rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   else
+                rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   endif
+                if (    RT_FAILURE(rcStrict)
+                    || rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
+                {
+                    AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
+                    STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
+                    return rcStrict;
+                }
+            }
+#  endif
+
+            AssertMsg(   pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
+                      || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
+                      ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
+                       pvFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
+            if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
+            else
+            {
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
+                if (uErr & X86_TRAP_PF_RSVD) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
+            }
+
+            if (pCurType->pfnPfHandler)
+            {
+                STAM_PROFILE_START(&pCur->Stat, h);
+
+                if (pCurType->fKeepPgmLock)
+                {
+                    rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault,
+                                                      !pCurType->fRing0DevInsIdx ? pCur->uUser
+                                                      : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser));
+
+                    STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
+                }
+                else
+                {
+                    uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
+                                         : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
+                    PGM_UNLOCK(pVM);
+                    *pfLockTaken = false;
+
+                    rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, pvFault, GCPhysFault, uUser);
+
+                    STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
+                }
+            }
+            else
+                rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
+            return rcStrict;
+        }
+        AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
+    }
+
+    /*
+     * There is a handled area of the page, but this fault doesn't belong to it.
+     * We must emulate the instruction.
+     *
+     * To avoid crashing (non-fatal) in the interpreter and go back to the recompiler
+     * we first check if this was a page-not-present fault for a page with only
+     * write access handlers. Restart the instruction if it wasn't a write access.
+     */
+    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersUnhandled);
+
+    if (    !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
+        &&  !(uErr & X86_TRAP_PF_P))
+    {
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, pGstWalk->Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#  else
+        rcStrict = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#  endif
+        if (    RT_FAILURE(rcStrict)
+            ||  rcStrict == VINF_PGM_SYNCPAGE_MODIFIED_PDE
+            ||  !(uErr & X86_TRAP_PF_RW))
+        {
+            AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
+            return rcStrict;
+        }
+    }
+
+    /** @todo This particular case can cause quite a lot of overhead. E.g. early stage of kernel booting in Ubuntu 6.06
+     *        It's writing to an unhandled part of the LDT page several million times.
+     */
+    rcStrict = PGMInterpretInstruction(pVCpu, pvFault);
+    LogFlow(("PGM: PGMInterpretInstruction -> rcStrict=%d pPage=%R[pgmpage]\n", VBOXSTRICTRC_VAL(rcStrict), pPage));
+    STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndUnhandled; });
+    return rcStrict;
+} /* if any kind of handler */
+# endif /* !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE*/
+
+
+/**
+ * \#PF Handler for raw-mode guest execution.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uErr        The trap error code.
+ * @param   pCtx        Pointer to the register context for the CPU.
+ * @param   pvFault     The fault address.
+ * @param   pfLockTaken PGM lock taken here or not (out)
+ */
+PGM_BTH_DECL(int, Trap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPTR pvFault, bool *pfLockTaken)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
+
+    *pfLockTaken = false;
+
+# if  (   PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT \
+       || PGM_GST_TYPE == PGM_TYPE_PAE   || PGM_GST_TYPE == PGM_TYPE_AMD64) \
+    && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
+    && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
+    && PGM_SHW_TYPE != PGM_TYPE_NONE
+    int rc;
+
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    /*
+     * Walk the guest page translation tables and check if it's a guest fault.
+     */
+    PGMPTWALK Walk;
+    GSTPTWALK GstWalk;
+    rc = PGM_GST_NAME(Walk)(pVCpu, pvFault, &Walk, &GstWalk);
+    if (RT_FAILURE_NP(rc))
+        return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
+
+    /* assert some GstWalk sanity. */
+#   if PGM_GST_TYPE == PGM_TYPE_AMD64
+    /*AssertMsg(GstWalk.Pml4e.u == GstWalk.pPml4e->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pml4e.u, (uint64_t)GstWalk.pPml4e->u));  - not always true with SMP guests. */
+#   endif
+#   if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
+    /*AssertMsg(GstWalk.Pdpe.u == GstWalk.pPdpe->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pdpe.u, (uint64_t)GstWalk.pPdpe->u)); - ditto */
+#   endif
+    /*AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); - ditto */
+    /*AssertMsg(GstWalk.Core.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); - ditto */
+    Assert(Walk.fSucceeded);
+    Assert(Walk.fEffective & PGM_PTATTRS_R_MASK);
+
+    if (uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_US | X86_TRAP_PF_ID))
+    {
+        if (    (   (uErr & X86_TRAP_PF_RW)
+                 && !(Walk.fEffective & PGM_PTATTRS_W_MASK)
+                 && (   (uErr & X86_TRAP_PF_US)
+                     || CPUMIsGuestR0WriteProtEnabled(pVCpu)) )
+            ||  ((uErr & X86_TRAP_PF_US) && !(Walk.fEffective & PGM_PTATTRS_US_MASK))
+            ||  ((uErr & X86_TRAP_PF_ID) &&  (Walk.fEffective & PGM_PTATTRS_NX_MASK))
+           )
+            return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerGuestFault)(pVCpu, &Walk, uErr));
+    }
+
+    /* Take the big lock now before we update flags. */
+    *pfLockTaken = true;
+    PGM_LOCK_VOID(pVM);
+
+    /*
+     * Set the accessed and dirty flags.
+     */
+    /** @todo Should probably use cmpxchg logic here as we're potentially racing
+     *        other CPUs in SMP configs. (the lock isn't enough, since we take it
+     *        after walking and the page tables could be stale already) */
+#   if PGM_GST_TYPE == PGM_TYPE_AMD64
+    if (!(GstWalk.Pml4e.u & X86_PML4E_A))
+    {
+        GstWalk.Pml4e.u |= X86_PML4E_A;
+        GST_ATOMIC_OR(&GstWalk.pPml4e->u, X86_PML4E_A);
+    }
+    if (!(GstWalk.Pdpe.u & X86_PDPE_A))
+    {
+        GstWalk.Pdpe.u |= X86_PDPE_A;
+        GST_ATOMIC_OR(&GstWalk.pPdpe->u, X86_PDPE_A);
+    }
+#   endif
+    if (Walk.fBigPage)
+    {
+        Assert(GstWalk.Pde.u & X86_PDE_PS);
+        if (uErr & X86_TRAP_PF_RW)
+        {
+            if ((GstWalk.Pde.u & (X86_PDE4M_A | X86_PDE4M_D)) != (X86_PDE4M_A | X86_PDE4M_D))
+            {
+                GstWalk.Pde.u |= X86_PDE4M_A | X86_PDE4M_D;
+                GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A | X86_PDE4M_D);
+            }
+        }
+        else
+        {
+            if (!(GstWalk.Pde.u & X86_PDE4M_A))
+            {
+                GstWalk.Pde.u |= X86_PDE4M_A;
+                GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE4M_A);
+            }
+        }
+    }
+    else
+    {
+        Assert(!(GstWalk.Pde.u & X86_PDE_PS));
+        if (!(GstWalk.Pde.u & X86_PDE_A))
+        {
+            GstWalk.Pde.u |= X86_PDE_A;
+            GST_ATOMIC_OR(&GstWalk.pPde->u, X86_PDE_A);
+        }
+
+        if (uErr & X86_TRAP_PF_RW)
+        {
+#   ifdef VBOX_WITH_STATISTICS
+            if (GstWalk.Pte.u & X86_PTE_D)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageAlreadyDirty));
+            else
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtiedPage));
+#   endif
+            if ((GstWalk.Pte.u & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D))
+            {
+                GstWalk.Pte.u |= X86_PTE_A | X86_PTE_D;
+                GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A | X86_PTE_D);
+            }
+        }
+        else
+        {
+            if (!(GstWalk.Pte.u & X86_PTE_A))
+            {
+                GstWalk.Pte.u |= X86_PTE_A;
+                GST_ATOMIC_OR(&GstWalk.pPte->u, X86_PTE_A);
+            }
+        }
+        Assert(GstWalk.Pte.u == GstWalk.pPte->u);
+    }
+#if 0
+    /* Disabling this since it's not reliable for SMP, see @bugref{10092#c22}. */
+    AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u,
+              ("%RX64 %RX64 pPte=%p pPde=%p Pte=%RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u, GstWalk.pPte, GstWalk.pPde, (uint64_t)GstWalk.pPte->u));
+#endif
+
+#  else  /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+    GSTPDE const PdeSrcDummy = { X86_PDE_P | X86_PDE_US | X86_PDE_RW | X86_PDE_A}; /** @todo eliminate this */
+
+    /* Take the big lock now. */
+    *pfLockTaken = true;
+    PGM_LOCK_VOID(pVM);
+#  endif /* !PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+
+#  ifdef PGM_WITH_MMIO_OPTIMIZATIONS
+    /*
+     * If it is a reserved bit fault we know that it is an MMIO (access
+     * handler) related fault and can skip some 200 lines of code.
+     */
+    if (uErr & X86_TRAP_PF_RSVD)
+    {
+        Assert(uErr & X86_TRAP_PF_P);
+        PPGMPAGE pPage;
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        rc = pgmPhysGetPageEx(pVM, Walk.GCPhys, &pPage);
+        if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+            return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage,
+                                                                                 pfLockTaken, &Walk, &GstWalk));
+        rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
+#   else
+        rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault), &pPage);
+        if (RT_SUCCESS(rc) && PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+            return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken));
+        rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
+#   endif
+        AssertRC(rc);
+        PGM_INVL_PG(pVCpu, pvFault);
+        return rc; /* Restart with the corrected entry. */
+    }
+#  endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
+
+    /*
+     * Fetch the guest PDE, PDPE and PML4E.
+     */
+#  if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDDst = pvFault >> SHW_PD_SHIFT;
+    PX86PD          pPDDst = pgmShwGet32BitPDPtr(pVCpu);
+
+#  elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    const unsigned  iPDDst = (pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK;   /* pPDDst index, not used with the pool. */
+    PX86PDPAE       pPDDst;
+#   if PGM_GST_TYPE == PGM_TYPE_PAE
+    rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, GstWalk.Pdpe.u, &pPDDst);
+#   else
+    rc = pgmShwSyncPaePDPtr(pVCpu, pvFault, X86_PDPE_P, &pPDDst);       /* RW, US and A are reserved in PAE mode. */
+#   endif
+    AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+
+#  elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PX86PDPAE       pPDDst;
+#   if PGM_GST_TYPE == PGM_TYPE_PROT  /* (AMD-V nested paging) */
+    rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A,
+                                 X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A, &pPDDst);
+#   else
+    rc = pgmShwSyncLongModePDPtr(pVCpu, pvFault, GstWalk.Pml4e.u, GstWalk.Pdpe.u, &pPDDst);
+#   endif
+    AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+
+#  elif PGM_SHW_TYPE == PGM_TYPE_EPT
+    const unsigned  iPDDst = ((pvFault >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PEPTPD          pPDDst;
+    rc = pgmShwGetEPTPDPtr(pVCpu, pvFault, NULL, &pPDDst);
+    AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+#  endif
+    Assert(pPDDst);
+
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    /*
+     * Dirty page handling.
+     *
+     * If we successfully correct the write protection fault due to dirty bit
+     * tracking, then return immediately.
+     */
+    if (uErr & X86_TRAP_PF_RW)  /* write fault? */
+    {
+        STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
+        rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, &pPDDst->a[iPDDst], GstWalk.pPde, pvFault);
+        STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyBitTracking), a);
+        if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
+        {
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0
+                        = rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT
+                          ? &pVCpu->pgm.s.Stats.StatRZTrap0eTime2DirtyAndAccessed
+                          : &pVCpu->pgm.s.Stats.StatRZTrap0eTime2GuestTrap; });
+            Log8(("Trap0eHandler: returns VINF_SUCCESS\n"));
+            return VINF_SUCCESS;
+        }
+#ifdef DEBUG_bird
+        AssertMsg(GstWalk.Pde.u == GstWalk.pPde->u || GstWalk.pPte->u == GstWalk.pPde->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pde.u, (uint64_t)GstWalk.pPde->u)); // - triggers with smp w7 guests.
+        AssertMsg(Walk.fBigPage || GstWalk.Pte.u == GstWalk.pPte->u || pVM->cCpus > 1, ("%RX64 %RX64\n", (uint64_t)GstWalk.Pte.u, (uint64_t)GstWalk.pPte->u)); // - ditto.
+#endif
+    }
+
+#   if 0 /* rarely useful; leave for debugging. */
+    STAM_COUNTER_INC(&pVCpu->pgm.s.StatRZTrap0ePD[iPDSrc]);
+#   endif
+#  endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+
+    /*
+     * A common case is the not-present error caused by lazy page table syncing.
+     *
+     * It is IMPORTANT that we weed out any access to non-present shadow PDEs
+     * here so we can safely assume that the shadow PT is present when calling
+     * SyncPage later.
+     *
+     * On failure, we ASSUME that SyncPT is out of memory or detected some kind
+     * of mapping conflict and defer to SyncCR3 in R3.
+     * (Again, we do NOT support access handlers for non-present guest pages.)
+     *
+     */
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    Assert(GstWalk.Pde.u & X86_PDE_P);
+#  endif
+    if (    !(uErr & X86_TRAP_PF_P) /* not set means page not present instead of page protection violation */
+        &&  !SHW_PDE_IS_P(pPDDst->a[iPDDst]))
+    {
+        STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        LogFlow(("=>SyncPT %04x = %08RX64\n", (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, (uint64_t)GstWalk.Pde.u));
+        rc = PGM_BTH_NAME(SyncPT)(pVCpu, (pvFault >> GST_PD_SHIFT) & GST_PD_MASK, GstWalk.pPd, pvFault);
+#  else
+        LogFlow(("=>SyncPT pvFault=%RGv\n", pvFault));
+        rc = PGM_BTH_NAME(SyncPT)(pVCpu, 0, NULL, pvFault);
+#  endif
+        if (RT_SUCCESS(rc))
+            return rc;
+        Log(("SyncPT: %RGv failed!! rc=%Rrc\n", pvFault, rc));
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
+        return VINF_PGM_SYNC_CR3;
+    }
+
+    /*
+     * Check if this fault address is flagged for special treatment,
+     * which means we'll have to figure out the physical address and
+     * check flags associated with it.
+     *
+     * ASSUME that we can limit any special access handling to pages
+     * in page tables which the guest believes to be present.
+     */
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    RTGCPHYS GCPhys = Walk.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+#  else
+    RTGCPHYS GCPhys = PGM_A20_APPLY(pVCpu, (RTGCPHYS)pvFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
+#  endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) */
+    PPGMPAGE pPage;
+    rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_FAILURE(rc))
+    {
+        /*
+         * When the guest accesses invalid physical memory (e.g. probing
+         * of RAM or accessing a remapped MMIO range), then we'll fall
+         * back to the recompiler to emulate the instruction.
+         */
+        LogFlow(("PGM #PF: pgmPhysGetPageEx(%RGp) failed with %Rrc\n", GCPhys, rc));
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersInvalid);
+        STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2InvalidPhys; });
+        return VINF_EM_RAW_EMULATE_INSTR;
+    }
+
+    /*
+     * Any handlers for this page?
+     */
+    if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken,
+                                                                             &Walk, &GstWalk));
+# else
+        return VBOXSTRICTRC_TODO(PGM_BTH_NAME(Trap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, pvFault, pPage, pfLockTaken));
+# endif
+
+    /*
+     * We are here only if page is present in Guest page tables and
+     * trap is not handled by our handlers.
+     *
+     * Check it for page out-of-sync situation.
+     */
+    if (!(uErr & X86_TRAP_PF_P))
+    {
+        /*
+         * Page is not present in our page tables. Try to sync it!
+         */
+        if (uErr & X86_TRAP_PF_US)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
+        else /* supervisor */
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
+
+        if (PGM_PAGE_IS_BALLOONED(pPage))
+        {
+            /* Emulate reads from ballooned pages as they are not present in
+               our shadow page tables. (Required for e.g. Solaris guests; soft
+               ecc, random nr generator.) */
+            rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault));
+            LogFlow(("PGM: PGMInterpretInstruction balloon -> rc=%d pPage=%R[pgmpage]\n", rc, pPage));
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncBallloon));
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Ballooned; });
+            return rc;
+        }
+
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   else
+        rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, PGM_SYNC_NR_PAGES, uErr);
+#   endif
+        if (RT_SUCCESS(rc))
+        {
+            /* The page was successfully synced, return to the guest. */
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
+            return VINF_SUCCESS;
+        }
+    }
+    else /* uErr & X86_TRAP_PF_P: */
+    {
+        /*
+         * Write protected pages are made writable when the guest makes the
+         * first write to it.  This happens for pages that are shared, write
+         * monitored or not yet allocated.
+         *
+         * We may also end up here when CR0.WP=0 in the guest.
+         *
+         * Also, a side effect of not flushing global PDEs are out of sync
+         * pages due to physical monitored regions, that are no longer valid.
+         * Assume for now it only applies to the read/write flag.
+         */
+        if (uErr & X86_TRAP_PF_RW)
+        {
+            /*
+             * Check if it is a read-only page.
+             */
+            if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+            {
+                Log(("PGM #PF: Make writable: %RGp %R[pgmpage] pvFault=%RGp uErr=%#x\n", GCPhys, pPage, pvFault, uErr));
+                Assert(!PGM_PAGE_IS_ZERO(pPage));
+                AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
+                STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
+
+                rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+                if (rc != VINF_SUCCESS)
+                {
+                    AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
+                    return rc;
+                }
+                if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                    return VINF_EM_NO_MEMORY;
+            }
+
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+            /*
+             * Check to see if we need to emulate the instruction if CR0.WP=0.
+             */
+            if (    !(Walk.fEffective & PGM_PTATTRS_W_MASK)
+                &&  (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
+                &&  CPUMGetGuestCPL(pVCpu) < 3)
+            {
+                Assert((uErr & (X86_TRAP_PF_RW | X86_TRAP_PF_P)) == (X86_TRAP_PF_RW | X86_TRAP_PF_P));
+
+                /*
+                 * The Netware WP0+RO+US hack.
+                 *
+                 * Netware sometimes(/always?) runs with WP0.  It has been observed doing
+                 * excessive write accesses to pages which are mapped with US=1 and RW=0
+                 * while WP=0.  This causes a lot of exits and extremely slow execution.
+                 * To avoid trapping and emulating every write here, we change the shadow
+                 * page table entry to map it as US=0 and RW=1 until user mode tries to
+                 * access it again (see further below).  We count these shadow page table
+                 * changes so we can avoid having to clear the page pool every time the WP
+                 * bit changes to 1 (see PGMCr0WpEnabled()).
+                 */
+#    if (PGM_GST_TYPE == PGM_TYPE_32BIT || PGM_GST_TYPE == PGM_TYPE_PAE) && 1
+                if (   (Walk.fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
+                    && (Walk.fBigPage || (GstWalk.Pde.u & X86_PDE_RW))
+                    && pVM->cCpus == 1 /* Sorry, no go on SMP. Add CFGM option? */)
+                {
+                    Log(("PGM #PF: Netware WP0+RO+US hack: pvFault=%RGp uErr=%#x (big=%d)\n", pvFault, uErr, Walk.fBigPage));
+                    rc = pgmShwMakePageSupervisorAndWritable(pVCpu, pvFault, Walk.fBigPage, PGM_MK_PG_IS_WRITE_FAULT);
+                    if (rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3)
+                    {
+                        PGM_INVL_PG(pVCpu, pvFault);
+                        pVCpu->pgm.s.cNetwareWp0Hacks++;
+                        STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsHack; });
+                        return rc;
+                    }
+                    AssertMsg(RT_FAILURE_NP(rc), ("%Rrc\n", rc));
+                    Log(("pgmShwMakePageSupervisorAndWritable(%RGv) failed with rc=%Rrc - ignored\n", pvFault, rc));
+                }
+#    endif
+
+                /* Interpret the access. */
+                rc = VBOXSTRICTRC_TODO(PGMInterpretInstruction(pVCpu, pvFault));
+                Log(("PGM #PF: WP0 emulation (pvFault=%RGp uErr=%#x cpl=%d fBig=%d fEffUs=%d)\n", pvFault, uErr, CPUMGetGuestCPL(pVCpu), Walk.fBigPage, !!(Walk.fEffective & PGM_PTATTRS_US_MASK)));
+                if (RT_SUCCESS(rc))
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulInRZ);
+                else
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eWPEmulToR3);
+                STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2WPEmulation; });
+                return rc;
+            }
+#   endif
+            /// @todo count the above case; else
+            if (uErr & X86_TRAP_PF_US)
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUserWrite));
+            else /* supervisor */
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
+
+            /*
+             * Sync the page.
+             *
+             * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
+             *       page is not present, which is not true in this case.
+             */
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+            rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
+#   else
+            rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrcDummy, pvFault, 1, uErr);
+#   endif
+            if (RT_SUCCESS(rc))
+            {
+               /*
+                * Page was successfully synced, return to guest but invalidate
+                * the TLB first as the page is very likely to be in it.
+                */
+#   if PGM_SHW_TYPE == PGM_TYPE_EPT
+                HMInvalidatePhysPage(pVM, (RTGCPHYS)pvFault);
+#   else
+                PGM_INVL_PG(pVCpu, pvFault);
+#   endif
+#   ifdef VBOX_STRICT
+                PGMPTWALK GstPageWalk;
+                GstPageWalk.GCPhys = RTGCPHYS_MAX;
+                if (!pVM->pgm.s.fNestedPaging)
+                {
+                    rc = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
+                    AssertMsg(RT_SUCCESS(rc) && ((GstPageWalk.fEffective & X86_PTE_RW) || ((CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG && CPUMGetGuestCPL(pVCpu) < 3)), ("rc=%Rrc fPageGst=%RX64\n", rc, GstPageWalk.fEffective));
+                    LogFlow(("Obsolete physical monitor page out of sync %RGv - phys %RGp flags=%08llx\n", pvFault, GstPageWalk.GCPhys, GstPageWalk.fEffective));
+                }
+#    if 0 /* Bogus! Triggers incorrectly with w7-64 and later for the SyncPage case: "Pde at %RGv changed behind our back?" */
+                uint64_t fPageShw = 0;
+                rc = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
+                AssertMsg((RT_SUCCESS(rc) && (fPageShw & X86_PTE_RW)) || pVM->cCpus > 1 /* new monitor can be installed/page table flushed between the trap exit and PGMTrap0eHandler */,
+                          ("rc=%Rrc fPageShw=%RX64 GCPhys2=%RGp fPageGst=%RX64 pvFault=%RGv\n", rc, fPageShw, GstPageWalk.GCPhys, fPageGst, pvFault));
+#    endif
+#   endif /* VBOX_STRICT */
+                STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
+                return VINF_SUCCESS;
+            }
+        }
+#    if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        /*
+         * Check for Netware WP0+RO+US hack from above and undo it when user
+         * mode accesses the page again.
+         */
+        else if (   (Walk.fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK)) == PGM_PTATTRS_US_MASK
+                 && (Walk.fBigPage || (GstWalk.Pde.u & X86_PDE_RW))
+                 &&  pVCpu->pgm.s.cNetwareWp0Hacks > 0
+                 &&  (CPUMGetGuestCR0(pVCpu) & (X86_CR0_WP | X86_CR0_PG)) == X86_CR0_PG
+                 &&  CPUMGetGuestCPL(pVCpu) == 3
+                 &&  pVM->cCpus == 1
+                )
+        {
+            Log(("PGM #PF: Undo netware WP0+RO+US hack: pvFault=%RGp uErr=%#x\n", pvFault, uErr));
+            rc = PGM_BTH_NAME(SyncPage)(pVCpu, GstWalk.Pde, pvFault, 1, uErr);
+            if (RT_SUCCESS(rc))
+            {
+                PGM_INVL_PG(pVCpu, pvFault);
+                pVCpu->pgm.s.cNetwareWp0Hacks--;
+                STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Wp0RoUsUnhack; });
+                return VINF_SUCCESS;
+            }
+        }
+#    endif /* PGM_WITH_PAGING */
+
+        /** @todo else: why are we here? */
+
+#   if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && defined(VBOX_STRICT)
+        /*
+         * Check for VMM page flags vs. Guest page flags consistency.
+         * Currently only for debug purposes.
+         */
+        if (RT_SUCCESS(rc))
+        {
+            /* Get guest page flags. */
+            PGMPTWALK GstPageWalk;
+            int rc2 = PGMGstGetPage(pVCpu, pvFault, &GstPageWalk);
+            if (RT_SUCCESS(rc2))
+            {
+                uint64_t fPageShw = 0;
+                rc2 = PGMShwGetPage(pVCpu, pvFault, &fPageShw, NULL);
+
+#if 0
+                /*
+                 * Compare page flags.
+                 * Note: we have AVL, A, D bits desynced.
+                 */
+                AssertMsg(      (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
+                             == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK))
+                          || (   pVCpu->pgm.s.cNetwareWp0Hacks > 0
+                              &&    (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
+                                 == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US))
+                              && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW
+                              && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US),
+                          ("Page flags mismatch! pvFault=%RGv uErr=%x GCPhys=%RGp fPageShw=%RX64 fPageGst=%RX64 rc=%d\n",
+                           pvFault, (uint32_t)uErr, GCPhys, fPageShw, fPageGst, rc));
+01:01:15.623511 00:08:43.266063 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
+01:01:15.623511 00:08:43.266064 Location  : e:\vbox\svn\trunk\srcPage flags mismatch! pvFault=fffff801b0d7b000 uErr=11 GCPhys=0000000019b52000 fPageShw=0 fPageGst=77b0000000000121 rc=0
+
+01:01:15.625516 00:08:43.268051 Expression: (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK)) || ( pVCpu->pgm.s.cNetwareWp0Hacks > 0 && (fPageShw & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) == (fPageGst & ~(X86_PTE_A | X86_PTE_D | X86_PTE_AVL_MASK | X86_PTE_RW | X86_PTE_US)) && (fPageShw & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW && (fPageGst & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_US)
+01:01:15.625516 00:08:43.268051 Location  :
+e:\vbox\svn\trunk\srcPage flags mismatch!
+pvFault=fffff801b0d7b000
+                uErr=11     X86_TRAP_PF_ID | X86_TRAP_PF_P
+GCPhys=0000000019b52000
+fPageShw=0
+fPageGst=77b0000000000121
+rc=0
+#endif
+
+            }
+            else
+                AssertMsgFailed(("PGMGstGetPage rc=%Rrc\n", rc));
+        }
+        else
+            AssertMsgFailed(("PGMGCGetPage rc=%Rrc\n", rc));
+#   endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && VBOX_STRICT */
+    }
+
+
+    /*
+     * If we get here it is because something failed above, i.e. most like guru
+     * meditiation time.
+     */
+    LogRel(("%s: returns rc=%Rrc pvFault=%RGv uErr=%RX64 cs:rip=%04x:%08RX64\n",
+            __PRETTY_FUNCTION__, rc, pvFault, (uint64_t)uErr, pCtx->cs.Sel, pCtx->rip));
+    return rc;
+
+# else  /* Nested paging, EPT except PGM_GST_TYPE = PROT, NONE.   */
+    NOREF(uErr); NOREF(pCtx); NOREF(pvFault);
+    AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+# endif
+}
+
+
+# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
+/**
+ * Deals with a nested-guest \#PF fault for a guest-physical page with a handler.
+ *
+ * @returns Strict VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uErr                The error code.
+ * @param   pCtx                Pointer to the register context for the CPU.
+ * @param   GCPhysNestedFault   The nested-guest physical address of the fault.
+ * @param   pPage               The guest page at @a GCPhysNestedFault.
+ * @param   GCPhysFault         The guest-physical address of the fault.
+ * @param   pGstWalkAll         The guest page walk result.
+ * @param   pfLockTaken         Where to store whether the PGM is still held when
+ *                              this function completes.
+ *
+ * @note    The caller has taken the PGM lock.
+ */
+static VBOXSTRICTRC PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx,
+                                                                      RTGCPHYS GCPhysNestedFault, PPGMPAGE pPage,
+                                                                      RTGCPHYS GCPhysFault, PPGMPTWALKGST pGstWalkAll,
+                                                                      bool *pfLockTaken)
+{
+#  if PGM_GST_TYPE == PGM_TYPE_PROT \
+   && PGM_SHW_TYPE == PGM_TYPE_EPT
+
+    /** @todo Assert uErr isn't X86_TRAP_PF_RSVD and remove release checks. */
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysFault);
+    AssertMsgReturn(PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage), ("%RGp %RGp uErr=%u\n", GCPhysNestedFault, GCPhysFault, uErr),
+                    VERR_PGM_HANDLER_IPE_1);
+
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    RTGCPHYS const GCPhysPage       = GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+
+    /*
+     * Physical page access handler.
+     */
+    PPGMPHYSHANDLER pCur;
+    VBOXSTRICTRC rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhysPage, &pCur);
+    AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
+
+    PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+    Assert(pCurType);
+
+    /*
+     * If the region is write protected and we got a page not present fault, then sync
+     * the pages. If the fault was caused by a read, then restart the instruction.
+     * In case of write access continue to the GC write handler.
+     */
+    if (   !(uErr & X86_TRAP_PF_P)
+        &&  pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
+    {
+        Log7Func(("Syncing Monitored: GCPhysNestedPage=%RGp GCPhysPage=%RGp uErr=%#x\n", GCPhysNestedPage, GCPhysPage, uErr));
+        rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll);
+        Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
+        if (    RT_FAILURE(rcStrict)
+            || !(uErr & X86_TRAP_PF_RW))
+        {
+            AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
+            return rcStrict;
+        }
+    }
+    else if (   !(uErr & X86_TRAP_PF_RSVD)
+             && pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE)
+    {
+        /*
+         * If the access was NOT through an EPT misconfig (i.e. RSVD), sync the page.
+         * This can happen for the VMX APIC-access page.
+         */
+        Log7Func(("Syncing MMIO: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
+        rcStrict = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /*cPages*/, uErr, pGstWalkAll);
+        Assert(rcStrict != VINF_PGM_SYNCPAGE_MODIFIED_PDE);
+        if (RT_FAILURE(rcStrict))
+        {
+            AssertMsgRC(rcStrict, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersOutOfSync);
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndPhys; });
+            return rcStrict;
+        }
+    }
+
+    AssertMsg(   pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE
+              || (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE && (uErr & X86_TRAP_PF_RW)),
+              ("Unexpected trap for physical handler: %08X (phys=%08x) pPage=%R[pgmpage] uErr=%X, enmKind=%d\n",
+               GCPhysNestedFault, GCPhysFault, pPage, uErr, pCurType->enmKind));
+    if (pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysWrite);
+    else
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAll);
+        if (uErr & X86_TRAP_PF_RSVD)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eHandlersPhysAllOpt);
+    }
+
+    if (pCurType->pfnPfHandler)
+    {
+        STAM_PROFILE_START(&pCur->Stat, h);
+        uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
+                             : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
+
+        if (pCurType->fKeepPgmLock)
+        {
+            rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser);
+            STAM_PROFILE_STOP(&pCur->Stat, h);
+        }
+        else
+        {
+            PGM_UNLOCK(pVM);
+            *pfLockTaken = false;
+            rcStrict = pCurType->pfnPfHandler(pVM, pVCpu, uErr, pCtx, GCPhysNestedFault, GCPhysFault, uUser);
+            STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("What's going on here!? Fault falls outside handler range!?\n"));
+        rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+    }
+
+    STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2HndPhys; });
+    return rcStrict;
+
+#  else
+    RT_NOREF8(pVCpu, uErr, pCtx, GCPhysNestedFault, pPage, GCPhysFault, pGstWalkAll, pfLockTaken);
+    AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+#  endif
+}
+# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+
+/**
+ * Nested \#PF handler for nested-guest hardware-assisted execution using nested
+ * paging.
+ *
+ * @returns VBox status code (appropriate for trap handling and GC return).
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uErr                The fault error (X86_TRAP_PF_*).
+ * @param   pCtx                Pointer to the register context for the CPU.
+ * @param   GCPhysNestedFault   The nested-guest physical address of the fault.
+ * @param   fIsLinearAddrValid  Whether translation of a nested-guest linear address
+ *                              caused this fault. If @c false, GCPtrNestedFault
+ *                              must be 0.
+ * @param   GCPtrNestedFault    The nested-guest linear address of this fault.
+ * @param   pWalk               The guest page table walk result.
+ * @param   pfLockTaken         Where to store whether the PGM lock is still held
+ *                              when this function completes.
+ */
+PGM_BTH_DECL(int, NestedTrap0eHandler)(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTX pCtx, RTGCPHYS GCPhysNestedFault,
+                                       bool fIsLinearAddrValid, RTGCPTR GCPtrNestedFault, PPGMPTWALK pWalk, bool *pfLockTaken)
+{
+    *pfLockTaken = false;
+# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) \
+    && PGM_GST_TYPE == PGM_TYPE_PROT \
+    && PGM_SHW_TYPE == PGM_TYPE_EPT
+
+    Assert(CPUMIsGuestVmxEptPagingEnabled(pVCpu));
+    Assert(PGM_A20_IS_ENABLED(pVCpu));
+
+    /* We don't support mode-based execute control for EPT yet. */
+    Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt);
+    Assert(!(uErr & X86_TRAP_PF_US));
+
+    /* Take the big lock now. */
+    *pfLockTaken = true;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_VOID(pVM);
+
+    /*
+     * Walk the guest EPT tables and check if it's an EPT violation or misconfiguration.
+     */
+    if (fIsLinearAddrValid)
+        Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x GCPtrNestedFault=%RGv\n",
+                  pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr, GCPtrNestedFault));
+    else
+        Log7Func(("cs:rip=%04x:%#08RX64 GCPhysNestedFault=%RGp uErr=%#x\n",
+                  pCtx->cs.Sel, pCtx->rip, GCPhysNestedFault, uErr));
+    PGMPTWALKGST GstWalkAll;
+    int rc = pgmGstSlatWalk(pVCpu, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk, &GstWalkAll);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(GstWalkAll.enmType == PGMPTWALKGSTTYPE_EPT);
+    Assert(pWalk->fSucceeded);
+    Assert(pWalk->fEffective & (PGM_PTATTRS_EPT_R_MASK | PGM_PTATTRS_EPT_W_MASK | PGM_PTATTRS_EPT_X_SUPER_MASK));
+    Assert(pWalk->fIsSlat);
+
+#  ifdef DEBUG_ramshankar
+    /* Paranoia. */
+    Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_R_MASK)  ==  RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_R_MASK));
+    Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_W_MASK)  ==  RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_W_MASK));
+    Assert(RT_BOOL(pWalk->fEffective & PGM_PTATTRS_NX_MASK) == !RT_BOOL(pWalk->fEffective & PGM_PTATTRS_EPT_X_SUPER_MASK));
+#  endif
+
+    /*
+     * Check page-access permissions.
+     */
+    if (   ((uErr & X86_TRAP_PF_RW) && !(pWalk->fEffective & PGM_PTATTRS_W_MASK))
+        || ((uErr & X86_TRAP_PF_ID) &&  (pWalk->fEffective & PGM_PTATTRS_NX_MASK)))
+    {
+        Log7Func(("Permission failed! GCPtrNested=%RGv GCPhysNested=%RGp uErr=%#x fEffective=%#RX64\n", GCPtrNestedFault,
+                  GCPhysNestedFault, uErr, pWalk->fEffective));
+        pWalk->fFailed = PGM_WALKFAIL_EPT_VIOLATION;
+        return VERR_ACCESS_DENIED;
+    }
+
+    PGM_A20_ASSERT_MASKED(pVCpu, pWalk->GCPhys);
+    RTGCPHYS const GCPhysPage       = pWalk->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    RTGCPHYS const GCPhysNestedPage = GCPhysNestedFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+
+    /*
+     * If we were called via an EPT misconfig, it should've already resulted in a nested-guest VM-exit.
+     */
+    AssertMsgReturn(!(uErr & X86_TRAP_PF_RSVD),
+                    ("Unexpected EPT misconfig VM-exit. GCPhysPage=%RGp GCPhysNestedPage=%RGp\n", GCPhysPage, GCPhysNestedPage),
+                    VERR_PGM_MAPPING_IPE);
+
+    /*
+     * Fetch and sync the nested-guest EPT page directory pointer.
+     */
+    PEPTPD pEptPd;
+    rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL /*ppPdpt*/, &pEptPd, &GstWalkAll);
+    AssertRCReturn(rc, rc);
+    Assert(pEptPd);
+
+    /*
+     * A common case is the not-present error caused by lazy page table syncing.
+     *
+     * It is IMPORTANT that we weed out any access to non-present shadow PDEs
+     * here so we can safely assume that the shadow PT is present when calling
+     * NestedSyncPage later.
+     *
+     * NOTE: It's possible we will be syncing the VMX APIC-access page here.
+     * In that case, we would sync the page but will NOT go ahead with emulating
+     * the APIC-access VM-exit through IEM. However, once the page is mapped in
+     * the shadow tables, subsequent APIC-access VM-exits for the nested-guest
+     * will be triggered by hardware. Maybe calling the IEM #PF handler can be
+     * considered as an optimization later.
+     */
+    unsigned const iPde = (GCPhysNestedPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    if (   !(uErr & X86_TRAP_PF_P)
+        && !(pEptPd->a[iPde].u & EPT_PRESENT_MASK))
+    {
+        STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2SyncPT; });
+        Log7Func(("NestedSyncPT: Lazy. GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
+        rc = PGM_BTH_NAME(NestedSyncPT)(pVCpu, GCPhysNestedPage, GCPhysPage, &GstWalkAll);
+        if (RT_SUCCESS(rc))
+            return rc;
+        AssertMsgFailedReturn(("NestedSyncPT: %RGv failed! rc=%Rrc\n", GCPhysNestedPage, rc), VERR_PGM_MAPPING_IPE);
+    }
+
+    /*
+     * Check if this fault address is flagged for special treatment.
+     * This handles faults on an MMIO or write-monitored page.
+     *
+     * If this happens to be the VMX APIC-access page, we don't treat is as MMIO
+     * but rather sync it further below (as a regular guest page) which lets
+     * hardware-assisted execution trigger the APIC-access VM-exits of the
+     * nested-guest directly.
+     */
+    PPGMPAGE pPage;
+    rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+    AssertRCReturn(rc, rc);
+    if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
+    {
+        Log7Func(("MMIO: Calling NestedTrap0eHandlerDoAccessHandlers for GCPhys %RGp\n", GCPhysPage));
+        return VBOXSTRICTRC_TODO(PGM_BTH_NAME(NestedTrap0eHandlerDoAccessHandlers)(pVCpu, uErr, pCtx, GCPhysNestedFault,
+                                                                                   pPage, pWalk->GCPhys, &GstWalkAll,
+                                                                                   pfLockTaken));
+    }
+
+    /*
+     * We are here only if page is present in nested-guest page tables but the
+     * trap is not handled by our handlers. Check for page out-of-sync situation.
+     */
+    if (!(uErr & X86_TRAP_PF_P))
+    {
+        Assert(!PGM_PAGE_IS_BALLOONED(pPage));
+        Assert(!(uErr & X86_TRAP_PF_US));   /* Mode-based execute not supported yet. */
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
+
+        Log7Func(("SyncPage: Not-Present: GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedFault, GCPhysPage));
+        rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, PGM_SYNC_NR_PAGES, uErr, &GstWalkAll);
+        if (RT_SUCCESS(rc))
+        {
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSync; });
+            return VINF_SUCCESS;
+        }
+    }
+    else if (uErr & X86_TRAP_PF_RW)
+    {
+        /*
+         * Write protected pages are made writable when the guest makes the
+         * first write to it. This happens for pages that are shared, write
+         * monitored or not yet allocated.
+         *
+         * We may also end up here when CR0.WP=0 in the guest.
+         *
+         * Also, a side effect of not flushing global PDEs are out of sync
+         * pages due to physical monitored regions, that are no longer valid.
+         * Assume for now it only applies to the read/write flag.
+         */
+        if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+        {
+            /* This is a read-only page. */
+            AssertMsgFailed(("Failed\n"));
+
+            Assert(!PGM_PAGE_IS_ZERO(pPage));
+            AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhysPage));
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2MakeWritable; });
+
+            Log7Func(("Calling pgmPhysPageMakeWritable for GCPhysPage=%RGp\n", GCPhysPage));
+            rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
+            if (rc != VINF_SUCCESS)
+            {
+                AssertMsg(rc == VINF_PGM_SYNC_CR3 || RT_FAILURE(rc), ("%Rrc\n", rc));
+                return rc;
+            }
+            if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                return VINF_EM_NO_MEMORY;
+        }
+
+        Assert(!(uErr & X86_TRAP_PF_US));   /* Mode-based execute not supported yet. */
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisorWrite));
+
+        /*
+         * Sync the write-protected page.
+         * Note: Do NOT use PGM_SYNC_NR_PAGES here. That only works if the
+         *       page is not present, which is not true in this case.
+         */
+        Log7Func(("SyncPage: RW: cs:rip=%04x:%#RX64 GCPhysNestedPage=%RGp uErr=%#RX32 GCPhysPage=%RGp WalkGCPhys=%RGp\n",
+                  pCtx->cs.Sel, pCtx->rip, GCPhysNestedPage, (uint32_t)uErr, GCPhysPage, pWalk->GCPhys));
+        rc = PGM_BTH_NAME(NestedSyncPage)(pVCpu, GCPhysNestedPage, GCPhysPage, 1 /* cPages */, uErr, &GstWalkAll);
+        if (RT_SUCCESS(rc))
+        {
+            HMInvalidatePhysPage(pVM, GCPhysPage);
+            STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2OutOfSyncHndObs; });
+            return VINF_SUCCESS;
+        }
+    }
+
+    /*
+     * If we get here it is because something failed above => guru meditation time?
+     */
+    LogRelMaxFunc(32, ("rc=%Rrc GCPhysNestedFault=%#RGp (%#RGp) uErr=%#RX32 cs:rip=%04x:%08RX64\n",
+                       rc, GCPhysNestedFault, GCPhysPage, (uint32_t)uErr, pCtx->cs.Sel, pCtx->rip));
+    return VERR_PGM_MAPPING_IPE;
+
+# else /* !VBOX_WITH_NESTED_HWVIRT_VMX_EPT || PGM_GST_TYPE != PGM_TYPE_PROT || PGM_SHW_TYPE != PGM_TYPE_EPT */
+    RT_NOREF7(pVCpu, uErr, pCtx, GCPhysNestedFault, fIsLinearAddrValid, GCPtrNestedFault, pWalk);
+    AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+# endif
+}
+
+#endif /* !IN_RING3 */
+
+
+/**
+ * Emulation of the invlpg instruction.
+ *
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   Page to invalidate.
+ *
+ * @remark  ASSUMES that the guest is updating before invalidating. This order
+ *          isn't required by the CPU, so this is speculative and could cause
+ *          trouble.
+ * @remark  No TLB shootdown is done on any other VCPU as we assume that
+ *          invlpg emulation is the *only* reason for calling this function.
+ *          (The guest has to shoot down TLB entries on other CPUs itself)
+ *          Currently true, but keep in mind!
+ *
+ * @todo    Clean this up! Most of it is (or should be) no longer necessary as we catch all page table accesses.
+ *          Should only be required when PGMPOOL_WITH_OPTIMIZED_DIRTY_PT is active (PAE or AMD64 (for now))
+ */
+PGM_BTH_DECL(int, InvalidatePage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
+{
+#if    PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
+    && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
+    && PGM_SHW_TYPE != PGM_TYPE_NONE
+    int rc;
+    PVMCC    pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    LogFlow(("InvalidatePage %RGv\n", GCPtrPage));
+
+    /*
+     * Get the shadow PD entry and skip out if this PD isn't present.
+     * (Guessing that it is frequent for a shadow PDE to not be present, do this first.)
+     */
+# if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDDst    = (uint32_t)GCPtrPage >> SHW_PD_SHIFT;
+    PX86PDE         pPdeDst   = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+#  ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
+    if (!pShwPde)
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
+        return VINF_SUCCESS;
+    }
+#  else
+    Assert(pShwPde);
+#  endif
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    const unsigned  iPdpt     = (uint32_t)GCPtrPage >> X86_PDPT_SHIFT;
+    PX86PDPT        pPdptDst  = pgmShwGetPaePDPTPtr(pVCpu);
+
+    /* If the shadow PDPE isn't present, then skip the invalidate. */
+#  ifdef IN_RING3 /* Possible we didn't resync yet when called from REM. */
+    if (!pPdptDst || !(pPdptDst->a[iPdpt].u & X86_PDPE_P))
+#  else
+    if (!(pPdptDst->a[iPdpt].u & X86_PDPE_P))
+#  endif
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
+        PGM_INVL_PG(pVCpu, GCPtrPage);
+        return VINF_SUCCESS;
+    }
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
+    AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
+
+    PX86PDPAE       pPDDst  = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+    const unsigned  iPDDst  = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PX86PDEPAE      pPdeDst = &pPDDst->a[iPDDst];
+
+# else /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
+    /* PML4 */
+    /*const unsigned  iPml4     = (GCPtrPage >> X86_PML4_SHIFT) & X86_PML4_MASK;*/
+    const unsigned  iPdpt     = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    const unsigned  iPDDst    = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PX86PDPAE       pPDDst;
+    PX86PDPT        pPdptDst;
+    PX86PML4E       pPml4eDst;
+    rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eDst, &pPdptDst, &pPDDst);
+    if (rc != VINF_SUCCESS)
+    {
+        AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT || rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
+        PGM_INVL_PG(pVCpu, GCPtrPage);
+        return VINF_SUCCESS;
+    }
+    PX86PDEPAE  pPdeDst  = &pPDDst->a[iPDDst];
+    Assert(pPDDst);
+    Assert(pPdptDst->a[iPdpt].u & X86_PDPE_P);
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & SHW_PDPE_PG_MASK);
+    Assert(pShwPde);
+
+# endif /* PGM_SHW_TYPE == PGM_TYPE_AMD64 */
+
+    const SHWPDE PdeDst = *pPdeDst;
+    if (!(PdeDst.u & X86_PDE_P))
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
+        PGM_INVL_PG(pVCpu, GCPtrPage);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Get the guest PD entry and calc big page.
+     */
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+    PGSTPD          pPDSrc      = pgmGstGet32bitPDPtr(pVCpu);
+    const unsigned  iPDSrc      = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
+    GSTPDE          PdeSrc      = pPDSrc->a[iPDSrc];
+# else /* PGM_GST_TYPE != PGM_TYPE_32BIT */
+    unsigned        iPDSrc = 0;
+#  if PGM_GST_TYPE == PGM_TYPE_PAE
+    X86PDPE         PdpeSrcIgn;
+    PX86PDPAE       pPDSrc      = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrcIgn);
+#  else /* AMD64 */
+    PX86PML4E       pPml4eSrcIgn;
+    X86PDPE         PdpeSrcIgn;
+    PX86PDPAE       pPDSrc      = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrcIgn, &PdpeSrcIgn, &iPDSrc);
+#  endif
+    GSTPDE          PdeSrc;
+
+    if (pPDSrc)
+        PdeSrc = pPDSrc->a[iPDSrc];
+    else
+        PdeSrc.u = 0;
+# endif /* PGM_GST_TYPE != PGM_TYPE_32BIT */
+    const bool      fWasBigPage = RT_BOOL(PdeDst.u & PGM_PDFLAGS_BIG_PAGE);
+    const bool      fIsBigPage  = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
+    if (fWasBigPage != fIsBigPage)
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
+
+# ifdef IN_RING3
+    /*
+     * If a CR3 Sync is pending we may ignore the invalidate page operation
+     * depending on the kind of sync and if it's a global page or not.
+     * This doesn't make sense in GC/R0 so we'll skip it entirely there.
+     */
+#  ifdef PGM_SKIP_GLOBAL_PAGEDIRS_ON_NONGLOBAL_FLUSH
+    if (    VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)
+        || (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)
+            && fIsBigPage
+            && (PdeSrc.u & X86_PDE4M_G)
+           )
+       )
+#  else
+    if (VM_FF_IS_ANY_SET(pVM, VM_FF_PGM_SYNC_CR3 | VM_FF_PGM_SYNC_CR3_NON_GLOBAL) )
+#  endif
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePageSkipped));
+        return VINF_SUCCESS;
+    }
+# endif /* IN_RING3 */
+
+    /*
+     * Deal with the Guest PDE.
+     */
+    rc = VINF_SUCCESS;
+    if (PdeSrc.u & X86_PDE_P)
+    {
+        Assert(    (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
+               &&  ((PdeSrc.u & X86_PDE_RW) || !(PdeDst.u & X86_PDE_RW) || pVCpu->pgm.s.cNetwareWp0Hacks > 0));
+        if (!fIsBigPage)
+        {
+            /*
+             * 4KB - page.
+             */
+            PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
+            RTGCPHYS        GCPhys   = GST_GET_PDE_GCPHYS(PdeSrc);
+
+# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+            /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+            GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
+# endif
+            if (pShwPage->GCPhys == GCPhys)
+            {
+                /* Syncing it here isn't 100% safe and it's probably not worth spending time syncing it. */
+                PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+
+                PGSTPT pPTSrc;
+                rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
+                if (RT_SUCCESS(rc))
+                {
+                    const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
+                    GSTPTE PteSrc = pPTSrc->a[iPTSrc];
+                    const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+                    PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
+                    Log2(("SyncPage: 4K  %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
+                            GCPtrPage, PteSrc.u & X86_PTE_P,
+                            (PteSrc.u & PdeSrc.u & X86_PTE_RW),
+                            (PteSrc.u & PdeSrc.u & X86_PTE_US),
+                            (uint64_t)PteSrc.u,
+                            SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+                            SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+                }
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4KBPages));
+                PGM_INVL_PG(pVCpu, GCPtrPage);
+            }
+            else
+            {
+                /*
+                 * The page table address changed.
+                 */
+                LogFlow(("InvalidatePage: Out-of-sync at %RGp PdeSrc=%RX64 PdeDst=%RX64 ShwGCPhys=%RGp iPDDst=%#x\n",
+                         GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, iPDDst));
+                pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
+                SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDOutOfSync));
+                PGM_INVL_VCPU_TLBS(pVCpu);
+            }
+        }
+        else
+        {
+            /*
+             * 2/4MB - page.
+             */
+            /* Before freeing the page, check if anything really changed. */
+            PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
+            RTGCPHYS        GCPhys   = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
+# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+            /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
+            GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
+# endif
+            if (    pShwPage->GCPhys == GCPhys
+                &&  pShwPage->enmKind == BTH_PGMPOOLKIND_PT_FOR_BIG)
+            {
+                /* ASSUMES a the given bits are identical for 4M and normal PDEs */
+                /** @todo This test is wrong as it cannot check the G bit!
+                 *        FIXME */
+                if (        (PdeSrc.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
+                        ==  (PdeDst.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US))
+                    &&  (   (PdeSrc.u & X86_PDE4M_D) /** @todo rainy day: What about read-only 4M pages? not very common, but still... */
+                         || (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)))
+                {
+                    LogFlow(("Skipping flush for big page containing %RGv (PD=%X .u=%RX64)-> nothing has changed!\n", GCPtrPage, iPDSrc, PdeSrc.u));
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPagesSkip));
+                    return VINF_SUCCESS;
+                }
+            }
+
+            /*
+             * Ok, the page table is present and it's been changed in the guest.
+             * If we're in host context, we'll just mark it as not present taking the lazy approach.
+             * We could do this for some flushes in GC too, but we need an algorithm for
+             * deciding which 4MB pages containing code likely to be executed very soon.
+             */
+            LogFlow(("InvalidatePage: Out-of-sync PD at %RGp PdeSrc=%RX64 PdeDst=%RX64\n",
+                     GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+            pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
+            SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage4MBPages));
+            PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
+        }
+    }
+    else
+    {
+        /*
+         * Page directory is not present, mark shadow PDE not present.
+         */
+        pgmPoolFree(pVM, PdeDst.u & SHW_PDE_PG_MASK, pShwPde->idx, iPDDst);
+        SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePagePDNPs));
+        PGM_INVL_PG(pVCpu, GCPtrPage);
+    }
+    return rc;
+
+#else /* guest real and protected mode, nested + ept, none. */
+    /* There's no such thing as InvalidatePage when paging is disabled, so just ignore. */
+    NOREF(pVCpu); NOREF(GCPtrPage);
+    return VINF_SUCCESS;
+#endif
+}
+
+#if PGM_SHW_TYPE != PGM_TYPE_NONE
+
+/**
+ * Update the tracking of shadowed pages.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pShwPage    The shadow page.
+ * @param   HCPhys      The physical page we is being dereferenced.
+ * @param   iPte        Shadow PTE index
+ * @param   GCPhysPage  Guest physical address (only valid if pShwPage->fDirty is set)
+ */
+DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackDeref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, RTHCPHYS HCPhys, uint16_t iPte,
+                                                        RTGCPHYS GCPhysPage)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+# if    defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
+     && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
+     && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
+
+    /* Use the hint we retrieved from the cached guest PT. */
+    if (pShwPage->fDirty)
+    {
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+        Assert(pShwPage->cPresent);
+        Assert(pPool->cPresent);
+        pShwPage->cPresent--;
+        pPool->cPresent--;
+
+        PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysPage);
+        AssertRelease(pPhysPage);
+        pgmTrackDerefGCPhys(pPool, pShwPage, pPhysPage, iPte);
+        return;
+    }
+# else
+  NOREF(GCPhysPage);
+# endif
+
+    STAM_PROFILE_START(&pVM->pgm.s.Stats.StatTrackDeref, a);
+    LogFlow(("SyncPageWorkerTrackDeref: Damn HCPhys=%RHp pShwPage->idx=%#x!!!\n", HCPhys, pShwPage->idx));
+
+    /** @todo If this turns out to be a bottle neck (*very* likely) two things can be done:
+     *      1. have a medium sized HCPhys -> GCPhys TLB (hash?)
+     *      2. write protect all shadowed pages. I.e. implement caching.
+     */
+    /** @todo duplicated in the 2nd half of pgmPoolTracDerefGCPhysHint */
+
+    /*
+     * Find the guest address.
+     */
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        unsigned iPage = pRam->cb >> GUEST_PAGE_SHIFT;
+        while (iPage-- > 0)
+        {
+            if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
+            {
+                PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+                Assert(pShwPage->cPresent);
+                Assert(pPool->cPresent);
+                pShwPage->cPresent--;
+                pPool->cPresent--;
+
+                pgmTrackDerefGCPhys(pPool, pShwPage, &pRam->aPages[iPage], iPte);
+                STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatTrackDeref, a);
+                return;
+            }
+        }
+    }
+
+    for (;;)
+        AssertReleaseMsgFailed(("HCPhys=%RHp wasn't found!\n", HCPhys));
+}
+
+
+/**
+ * Update the tracking of shadowed pages.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pShwPage    The shadow page.
+ * @param   u16         The top 16-bit of the pPage->HCPhys.
+ * @param   pPage       Pointer to the guest page. this will be modified.
+ * @param   iPTDst      The index into the shadow table.
+ */
+DECLINLINE(void) PGM_BTH_NAME(SyncPageWorkerTrackAddref)(PVMCPUCC pVCpu, PPGMPOOLPAGE pShwPage, uint16_t u16,
+                                                         PPGMPAGE pPage, const unsigned iPTDst)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Just deal with the simple first time here.
+     */
+    if (!u16)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackVirgin);
+        u16 = PGMPOOL_TD_MAKE(1, pShwPage->idx);
+        /* Save the page table index. */
+        PGM_PAGE_SET_PTE_INDEX(pVM, pPage, iPTDst);
+    }
+    else
+        u16 = pgmPoolTrackPhysExtAddref(pVM, pPage, u16, pShwPage->idx, iPTDst);
+
+    /* write back */
+    Log2(("SyncPageWorkerTrackAddRef: u16=%#x->%#x  iPTDst=%#x\n", u16, PGM_PAGE_GET_TRACKING(pPage), iPTDst));
+    PGM_PAGE_SET_TRACKING(pVM, pPage, u16);
+
+    /* update statistics. */
+    pVM->pgm.s.CTX_SUFF(pPool)->cPresent++;
+    pShwPage->cPresent++;
+    if (pShwPage->iFirstPresent > iPTDst)
+        pShwPage->iFirstPresent = iPTDst;
+}
+
+
+/**
+ * Modifies a shadow PTE to account for access handlers.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPage       The page in question.
+ * @param   GCPhysPage  The guest-physical address of the page.
+ * @param   fPteSrc     The shadowed flags of the source PTE.  Must include the
+ *                      A (accessed) bit so it can be emulated correctly.
+ * @param   pPteDst     The shadow PTE (output).  This is temporary storage and
+ *                      does not need to be set atomically.
+ */
+DECLINLINE(void) PGM_BTH_NAME(SyncHandlerPte)(PVMCC pVM, PVMCPUCC pVCpu, PCPGMPAGE pPage, RTGCPHYS GCPhysPage, uint64_t fPteSrc,
+                                              PSHWPTE pPteDst)
+{
+    RT_NOREF_PV(pVM); RT_NOREF_PV(fPteSrc); RT_NOREF_PV(pVCpu); RT_NOREF_PV(GCPhysPage);
+
+    /** @todo r=bird: Are we actually handling dirty and access bits for pages with access handlers correctly? No.
+     *  Update: \#PF should deal with this before or after calling the handlers. It has all the info to do the job efficiently. */
+    if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+    {
+        LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark read-only\n", pPage));
+# if PGM_SHW_TYPE == PGM_TYPE_EPT
+        pPteDst->u = PGM_PAGE_GET_HCPHYS(pPage) | EPT_E_READ | EPT_E_EXECUTE | EPT_E_MEMTYPE_WB | EPT_E_IGNORE_PAT;
+# else
+        if (fPteSrc & X86_PTE_A)
+        {
+            SHW_PTE_SET(*pPteDst, fPteSrc | PGM_PAGE_GET_HCPHYS(pPage));
+            SHW_PTE_SET_RO(*pPteDst);
+        }
+        else
+            SHW_PTE_SET(*pPteDst, 0);
+# endif
+    }
+# ifdef PGM_WITH_MMIO_OPTIMIZATIONS
+#  if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
+    else if (   PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
+             && (   BTH_IS_NP_ACTIVE(pVM)
+                 || (fPteSrc & (X86_PTE_RW | X86_PTE_US)) == X86_PTE_RW) /** @todo Remove X86_PTE_US here and pGstWalk->Core.fEffectiveUS before the sync page test. */
+#   if PGM_SHW_TYPE == PGM_TYPE_AMD64
+             && pVM->pgm.s.fLessThan52PhysicalAddressBits
+#   endif
+            )
+    {
+        LogFlow(("SyncHandlerPte: MMIO page -> invalid \n"));
+#   if PGM_SHW_TYPE == PGM_TYPE_EPT
+        /* 25.2.3.1: Reserved physical address bit -> EPT Misconfiguration (exit 49) */
+        pPteDst->u = pVM->pgm.s.HCPhysInvMmioPg
+        /* 25.2.3.1: bits 2:0 = 010b -> EPT Misconfiguration (exit 49) */
+                   | EPT_E_WRITE
+        /* 25.2.3.1: leaf && 2:0 != 0 && u3Emt in {2, 3, 7} -> EPT Misconfiguration */
+                   | EPT_E_MEMTYPE_INVALID_3;
+#   else
+        /* Set high page frame bits that MBZ (bankers on PAE, CPU dependent on AMD64).  */
+        SHW_PTE_SET(*pPteDst, pVM->pgm.s.HCPhysInvMmioPg | X86_PTE_PAE_MBZ_MASK_NO_NX | X86_PTE_P);
+#   endif
+    }
+#  endif
+# endif /* PGM_WITH_MMIO_OPTIMIZATIONS */
+    else
+    {
+        LogFlow(("SyncHandlerPte: monitored page (%R[pgmpage]) -> mark not present\n", pPage));
+        SHW_PTE_SET(*pPteDst, 0);
+    }
+    /** @todo count these kinds of entries. */
+}
+
+
+/**
+ * Creates a 4K shadow page for a guest page.
+ *
+ * For 4M pages the caller must convert the PDE4M to a PTE, this includes adjusting the
+ * physical address.  The PdeSrc argument only the flags are used.  No page
+ * structured will be mapped in this function.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPteDst     Destination page table entry.
+ * @param   PdeSrc      Source page directory entry (i.e. Guest OS page directory entry).
+ *                      Can safely assume that only the flags are being used.
+ * @param   PteSrc      Source page table entry (i.e. Guest OS page table entry).
+ * @param   pShwPage    Pointer to the shadow page.
+ * @param   iPTDst      The index into the shadow table.
+ *
+ * @remark  Not used for 2/4MB pages!
+ */
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) || defined(DOXYGEN_RUNNING)
+static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, GSTPDE PdeSrc, GSTPTE PteSrc,
+                                         PPGMPOOLPAGE pShwPage, unsigned iPTDst)
+# else
+static void PGM_BTH_NAME(SyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPteDst, RTGCPHYS GCPhysPage,
+                                         PPGMPOOLPAGE pShwPage, unsigned iPTDst)
+# endif
+{
+    PVMCC    pVM = pVCpu->CTX_SUFF(pVM);
+    RTGCPHYS GCPhysOldPage = NIL_RTGCPHYS;
+
+# if    defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) \
+     && PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) \
+     && (PGM_GST_TYPE == PGM_TYPE_PAE || PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_PAE /* pae/32bit combo */)
+
+    if (pShwPage->fDirty)
+    {
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+        PGSTPT pGstPT;
+
+        /* Note that iPTDst can be used to index the guest PT even in the pae/32bit combo as we copy only half the table; see pgmPoolAddDirtyPage. */
+        pGstPT = (PGSTPT)&pPool->aDirtyPages[pShwPage->idxDirtyEntry].aPage[0];
+        GCPhysOldPage = GST_GET_PTE_GCPHYS(pGstPT->a[iPTDst]);
+        pGstPT->a[iPTDst].u = PteSrc.u;
+    }
+# else
+    Assert(!pShwPage->fDirty);
+# endif
+
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    if (   (PteSrc.u & X86_PTE_P)
+        && GST_IS_PTE_VALID(pVCpu, PteSrc))
+# endif
+    {
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        RTGCPHYS GCPhysPage = GST_GET_PTE_GCPHYS(PteSrc);
+#  endif
+        PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
+
+        /*
+         * Find the ram range.
+         */
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+        if (RT_SUCCESS(rc))
+        {
+            /* Ignore ballooned pages.
+               Don't return errors or use a fatal assert here as part of a
+               shadow sync range might included ballooned pages. */
+            if (PGM_PAGE_IS_BALLOONED(pPage))
+            {
+                Assert(!SHW_PTE_IS_P(*pPteDst)); /** @todo user tracking needs updating if this triggers. */
+                return;
+            }
+
+# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+            /* Make the page writable if necessary. */
+            if (    PGM_PAGE_GET_TYPE(pPage)  == PGMPAGETYPE_RAM
+                &&  (   PGM_PAGE_IS_ZERO(pPage)
+#  if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+                     || (   (PteSrc.u & X86_PTE_RW)
+#  else
+                     || (   1
+#  endif
+                         && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+#  ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+                         && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
+#  endif
+#  ifdef VBOX_WITH_PAGE_SHARING
+                         && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
+#  endif
+                        )
+                     )
+               )
+            {
+                rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhysPage);
+                AssertRC(rc);
+            }
+# endif
+
+            /*
+             * Make page table entry.
+             */
+            SHWPTE PteDst;
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+            uint64_t fGstShwPteFlags = GST_GET_PTE_SHW_FLAGS(pVCpu, PteSrc);
+# else
+            uint64_t fGstShwPteFlags = X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_A | X86_PTE_D;
+# endif
+            if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
+            {
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+                /*
+                 * If the page or page directory entry is not marked accessed,
+                 * we mark the page not present.
+                 */
+                if (!(PteSrc.u & X86_PTE_A) || !(PdeSrc.u & X86_PDE_A))
+                {
+                    LogFlow(("SyncPageWorker: page and or page directory not accessed -> mark not present\n"));
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,AccessedPage));
+                    SHW_PTE_SET(PteDst, 0);
+                }
+                /*
+                 * If the page is not flagged as dirty and is writable, then make it read-only, so we can set the dirty bit
+                 * when the page is modified.
+                 */
+                else if (!(PteSrc.u & X86_PTE_D) && (PdeSrc.u & PteSrc.u & X86_PTE_RW))
+                {
+                    AssertCompile(X86_PTE_RW == X86_PDE_RW);
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPage));
+                    SHW_PTE_SET(PteDst,
+                                  fGstShwPteFlags
+                                | PGM_PAGE_GET_HCPHYS(pPage)
+                                | PGM_PTFLAGS_TRACK_DIRTY);
+                    SHW_PTE_SET_RO(PteDst);
+                }
+                else
+# endif
+                {
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageSkipped));
+# if PGM_SHW_TYPE == PGM_TYPE_EPT
+                    PteDst.u = PGM_PAGE_GET_HCPHYS(pPage)
+                             | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_MEMTYPE_WB | EPT_E_IGNORE_PAT;
+# else
+                    SHW_PTE_SET(PteDst, fGstShwPteFlags | PGM_PAGE_GET_HCPHYS(pPage));
+# endif
+                }
+
+                /*
+                 * Make sure only allocated pages are mapped writable.
+                 */
+                if (    SHW_PTE_IS_P_RW(PteDst)
+                    &&  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+                {
+                    /* Still applies to shared pages. */
+                    Assert(!PGM_PAGE_IS_ZERO(pPage));
+                    SHW_PTE_SET_RO(PteDst);   /** @todo this isn't quite working yet. Why, isn't it? */
+                    Log3(("SyncPageWorker: write-protecting %RGp pPage=%R[pgmpage]at iPTDst=%d\n", GCPhysPage, pPage, iPTDst));
+                }
+            }
+            else
+                PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhysPage, fGstShwPteFlags, &PteDst);
+
+            /*
+             * Keep user track up to date.
+             */
+            if (SHW_PTE_IS_P(PteDst))
+            {
+                if (!SHW_PTE_IS_P(*pPteDst))
+                    PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
+                else if (SHW_PTE_GET_HCPHYS(*pPteDst) != SHW_PTE_GET_HCPHYS(PteDst))
+                {
+                    Log2(("SyncPageWorker: deref! *pPteDst=%RX64 PteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst), SHW_PTE_LOG64(PteDst)));
+                    PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
+                    PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
+                }
+            }
+            else if (SHW_PTE_IS_P(*pPteDst))
+            {
+                Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
+                PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
+            }
+
+            /*
+             * Update statistics and commit the entry.
+             */
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+            if (!(PteSrc.u & X86_PTE_G))
+                pShwPage->fSeenNonGlobal = true;
+# endif
+            SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
+            return;
+        }
+
+/** @todo count these three different kinds. */
+        Log2(("SyncPageWorker: invalid address in Pte\n"));
+    }
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    else if (!(PteSrc.u & X86_PTE_P))
+        Log2(("SyncPageWorker: page not present in Pte\n"));
+    else
+        Log2(("SyncPageWorker: invalid Pte\n"));
+# endif
+
+    /*
+     * The page is not present or the PTE is bad. Replace the shadow PTE by
+     * an empty entry, making sure to keep the user tracking up to date.
+     */
+    if (SHW_PTE_IS_P(*pPteDst))
+    {
+        Log2(("SyncPageWorker: deref! *pPteDst=%RX64\n", SHW_PTE_LOG64(*pPteDst)));
+        PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPteDst), iPTDst, GCPhysOldPage);
+    }
+    SHW_PTE_ATOMIC_SET(*pPteDst, 0);
+}
+
+
+/**
+ * Syncs a guest OS page.
+ *
+ * There are no conflicts at this point, neither is there any need for
+ * page table allocations.
+ *
+ * When called in PAE or AMD64 guest mode, the guest PDPE shall be valid.
+ * When called in AMD64 guest mode, the guest PML4E shall be valid.
+ *
+ * @returns VBox status code.
+ * @returns VINF_PGM_SYNCPAGE_MODIFIED_PDE if it modifies the PDE in any way.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   PdeSrc      Page directory entry of the guest.
+ * @param   GCPtrPage   Guest context page address.
+ * @param   cPages      Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
+ * @param   uErr        Fault error (X86_TRAP_PF_*).
+ */
+static int PGM_BTH_NAME(SyncPage)(PVMCPUCC pVCpu, GSTPDE PdeSrc, RTGCPTR GCPtrPage, unsigned cPages, unsigned uErr)
+{
+    PVMCC    pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
+    LogFlow(("SyncPage: GCPtrPage=%RGv cPages=%u uErr=%#x\n", GCPtrPage, cPages, uErr));
+    RT_NOREF_PV(uErr); RT_NOREF_PV(cPages); RT_NOREF_PV(GCPtrPage);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+# if   (   PGM_GST_TYPE == PGM_TYPE_32BIT \
+        || PGM_GST_TYPE == PGM_TYPE_PAE \
+        || PGM_GST_TYPE == PGM_TYPE_AMD64) \
+    && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
+
+    /*
+     * Assert preconditions.
+     */
+    Assert(PdeSrc.u & X86_PDE_P);
+    Assert(cPages);
+#  if 0 /* rarely useful; leave for debugging. */
+    STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPagePD[(GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK]);
+#  endif
+
+    /*
+     * Get the shadow PDE, find the shadow page table in the pool.
+     */
+#  if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDDst   = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PX86PDE         pPdeDst  = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    Assert(pShwPde);
+
+#  elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    const unsigned  iPDDst  = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PPGMPOOLPAGE    pShwPde = NULL;
+    PX86PDPAE       pPDDst;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    int rc2 = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
+    AssertRCSuccessReturn(rc2, rc2);
+    Assert(pShwPde);
+
+    pPDDst             = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+    PX86PDEPAE pPdeDst = &pPDDst->a[iPDDst];
+
+#  elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPDDst   = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    const unsigned  iPdpt    = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    PX86PDPAE       pPDDst   = NULL;            /* initialized to shut up gcc */
+    PX86PDPT        pPdptDst = NULL;            /* initialized to shut up gcc */
+
+    int rc2 = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
+    AssertRCSuccessReturn(rc2, rc2);
+    Assert(pPDDst && pPdptDst);
+    PX86PDEPAE      pPdeDst = &pPDDst->a[iPDDst];
+#  endif
+    SHWPDE          PdeDst   = *pPdeDst;
+
+    /*
+     * - In the guest SMP case we could have blocked while another VCPU reused
+     *   this page table.
+     * - With W7-64 we may also take this path when the A bit is cleared on
+     *   higher level tables (PDPE/PML4E).  The guest does not invalidate the
+     *   relevant TLB entries.  If we're write monitoring any page mapped by
+     *   the modified entry, we may end up here with a "stale" TLB entry.
+     */
+    if (!(PdeDst.u & X86_PDE_P))
+    {
+        Log(("CPU%u: SyncPage: Pde at %RGv changed behind our back? (pPdeDst=%p/%RX64) uErr=%#x\n", pVCpu->idCpu, GCPtrPage, pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
+        AssertMsg(pVM->cCpus > 1 || (uErr & (X86_TRAP_PF_P | X86_TRAP_PF_RW)) == (X86_TRAP_PF_P | X86_TRAP_PF_RW),
+                  ("Unexpected missing PDE p=%p/%RX64 uErr=%#x\n", pPdeDst, (uint64_t)PdeDst.u, (uint32_t)uErr));
+        if (uErr & X86_TRAP_PF_P)
+            PGM_INVL_PG(pVCpu, GCPtrPage);
+        return VINF_SUCCESS;    /* force the instruction to be executed again. */
+    }
+
+    PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
+    Assert(pShwPage);
+
+#  if PGM_GST_TYPE == PGM_TYPE_AMD64
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde  = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
+    Assert(pShwPde);
+#  endif
+
+    /*
+     * Check that the page is present and that the shadow PDE isn't out of sync.
+     */
+    const bool      fBigPage  = (PdeSrc.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu);
+    const bool      fPdeValid = !fBigPage ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc);
+    RTGCPHYS        GCPhys;
+    if (!fBigPage)
+    {
+        GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+        /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+        GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
+#  endif
+    }
+    else
+    {
+        GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+        /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
+        GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
+#  endif
+    }
+    /** @todo This doesn't check the G bit of 2/4MB pages. FIXME */
+    if (   fPdeValid
+        && pShwPage->GCPhys == GCPhys
+        && (PdeSrc.u & X86_PDE_P)
+        && (PdeSrc.u & X86_PDE_US) == (PdeDst.u & X86_PDE_US)
+        && ((PdeSrc.u & X86_PDE_RW) == (PdeDst.u & X86_PDE_RW) || !(PdeDst.u & X86_PDE_RW))
+#  if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
+        && ((PdeSrc.u & X86_PDE_PAE_NX) == (PdeDst.u & X86_PDE_PAE_NX) || !GST_IS_NX_ACTIVE(pVCpu))
+#  endif
+       )
+    {
+        /*
+         * Check that the PDE is marked accessed already.
+         * Since we set the accessed bit *before* getting here on a #PF, this
+         * check is only meant for dealing with non-#PF'ing paths.
+         */
+        if (PdeSrc.u & X86_PDE_A)
+        {
+            PSHWPT pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+            if (!fBigPage)
+            {
+                /*
+                 * 4KB Page - Map the guest page table.
+                 */
+                PGSTPT pPTSrc;
+                int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
+                if (RT_SUCCESS(rc))
+                {
+#  ifdef PGM_SYNC_N_PAGES
+                    Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
+                    if (    cPages > 1
+                        &&  !(uErr & X86_TRAP_PF_P)
+                        &&  !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+                    {
+                        /*
+                         * This code path is currently only taken when the caller is PGMTrap0eHandler
+                         * for non-present pages!
+                         *
+                         * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
+                         * deal with locality.
+                         */
+                        unsigned        iPTDst      = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+#   if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+                        /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+                        const unsigned  offPTSrc    = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
+#   else
+                        const unsigned  offPTSrc    = 0;
+#   endif
+                        const unsigned  iPTDstEnd   = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
+                        if (iPTDst < PGM_SYNC_NR_PAGES / 2)
+                            iPTDst = 0;
+                        else
+                            iPTDst -= PGM_SYNC_NR_PAGES / 2;
+
+                        for (; iPTDst < iPTDstEnd; iPTDst++)
+                        {
+                            const PGSTPTE pPteSrc = &pPTSrc->a[offPTSrc + iPTDst];
+
+                            if (   (pPteSrc->u & X86_PTE_P)
+                                && !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
+                            {
+                                RTGCPTR GCPtrCurPage = (GCPtrPage & ~(RTGCPTR)(GST_PT_MASK << GST_PT_SHIFT))
+                                                     | ((offPTSrc + iPTDst) << GUEST_PAGE_SHIFT);
+                                NOREF(GCPtrCurPage);
+                                PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, *pPteSrc, pShwPage, iPTDst);
+                                Log2(("SyncPage: 4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx%s\n",
+                                      GCPtrCurPage, pPteSrc->u & X86_PTE_P,
+                                      !!(pPteSrc->u & PdeSrc.u & X86_PTE_RW),
+                                      !!(pPteSrc->u & PdeSrc.u & X86_PTE_US),
+                                      (uint64_t)pPteSrc->u,
+                                      SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+                                      SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+                            }
+                        }
+                    }
+                    else
+#  endif /* PGM_SYNC_N_PAGES */
+                    {
+                        const unsigned iPTSrc = (GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK;
+                        GSTPTE PteSrc = pPTSrc->a[iPTSrc];
+                        const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+                        PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
+                        Log2(("SyncPage: 4K  %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx} PteDst=%08llx %s\n",
+                              GCPtrPage, PteSrc.u & X86_PTE_P,
+                              !!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
+                              !!(PteSrc.u & PdeSrc.u & X86_PTE_US),
+                              (uint64_t)PteSrc.u,
+                              SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+                              SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+                    }
+                }
+                else /* MMIO or invalid page: emulated in #PF handler. */
+                {
+                    LogFlow(("PGM_GCPHYS_2_PTR %RGp failed with %Rrc\n", GCPhys, rc));
+                    Assert(!SHW_PTE_IS_P(pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK]));
+                }
+            }
+            else
+            {
+                /*
+                 * 4/2MB page - lazy syncing shadow 4K pages.
+                 * (There are many causes of getting here, it's no longer only CSAM.)
+                 */
+                /* Calculate the GC physical address of this 4KB shadow page. */
+                GCPhys = PGM_A20_APPLY(pVCpu, GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc) | (GCPtrPage & GST_BIG_PAGE_OFFSET_MASK));
+                /* Find ram range. */
+                PPGMPAGE pPage;
+                int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+                if (RT_SUCCESS(rc))
+                {
+                    AssertFatalMsg(!PGM_PAGE_IS_BALLOONED(pPage), ("Unexpected ballooned page at %RGp\n", GCPhys));
+
+#  ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+                    /* Try to make the page writable if necessary. */
+                    if (    PGM_PAGE_GET_TYPE(pPage)  == PGMPAGETYPE_RAM
+                        &&  (   PGM_PAGE_IS_ZERO(pPage)
+                             || (   (PdeSrc.u & X86_PDE_RW)
+                                 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+#   ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+                                 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
+#   endif
+#   ifdef VBOX_WITH_PAGE_SHARING
+                                 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
+#   endif
+                                 )
+                             )
+                       )
+                    {
+                        rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+                        AssertRC(rc);
+                    }
+#  endif
+
+                    /*
+                     * Make shadow PTE entry.
+                     */
+                    SHWPTE PteDst;
+                    if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
+                        SHW_PTE_SET(PteDst, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc) | PGM_PAGE_GET_HCPHYS(pPage));
+                    else
+                        PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc), &PteDst);
+
+                    const unsigned iPTDst = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+                    if (    SHW_PTE_IS_P(PteDst)
+                        &&  !SHW_PTE_IS_P(pPTDst->a[iPTDst]))
+                        PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
+
+                    /* Make sure only allocated pages are mapped writable. */
+                    if (    SHW_PTE_IS_P_RW(PteDst)
+                        &&  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+                    {
+                        /* Still applies to shared pages. */
+                        Assert(!PGM_PAGE_IS_ZERO(pPage));
+                        SHW_PTE_SET_RO(PteDst);   /** @todo this isn't quite working yet... */
+                        Log3(("SyncPage: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, GCPtrPage));
+                    }
+
+                    SHW_PTE_ATOMIC_SET2(pPTDst->a[iPTDst], PteDst);
+
+                    /*
+                     * If the page is not flagged as dirty and is writable, then make it read-only
+                     * at PD level, so we can set the dirty bit when the page is modified.
+                     *
+                     * ASSUMES that page access handlers are implemented on page table entry level.
+                     *      Thus we will first catch the dirty access and set PDE.D and restart. If
+                     *      there is an access handler, we'll trap again and let it work on the problem.
+                     */
+                    /** @todo r=bird: figure out why we need this here, SyncPT should've taken care of this already.
+                     * As for invlpg, it simply frees the whole shadow PT.
+                     * ...It's possibly because the guest clears it and the guest doesn't really tell us... */
+                    if ((PdeSrc.u & (X86_PDE4M_D | X86_PDE_RW)) == X86_PDE_RW)
+                    {
+                        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
+                        PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
+                        PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
+                    }
+                    else
+                    {
+                        PdeDst.u &= ~(SHWUINT)(PGM_PDFLAGS_TRACK_DIRTY | X86_PDE_RW);
+                        PdeDst.u |= PdeSrc.u & X86_PDE_RW;
+                    }
+                    SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
+                    Log2(("SyncPage: BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} GCPhys=%RGp%s\n",
+                          GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US),
+                          (uint64_t)PdeSrc.u, GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
+                }
+                else
+                {
+                    LogFlow(("PGM_GCPHYS_2_PTR %RGp (big) failed with %Rrc\n", GCPhys, rc));
+                    /** @todo must wipe the shadow page table entry in this
+                     *        case. */
+                }
+            }
+            PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+            return VINF_SUCCESS;
+        }
+
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDNAs));
+    }
+    else if (fPdeValid)
+    {
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPagePDOutOfSync));
+        Log2(("SyncPage: Out-Of-Sync PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
+              GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
+    }
+    else
+    {
+/// @todo        STAM_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,SyncPagePDOutOfSyncAndInvalid));
+        Log2(("SyncPage: Bad PDE at %RGp PdeSrc=%RX64 PdeDst=%RX64 (GCPhys %RGp vs %RGp)\n",
+              GCPtrPage, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u, pShwPage->GCPhys, GCPhys));
+    }
+
+    /*
+     * Mark the PDE not present.  Restart the instruction and let #PF call SyncPT.
+     * Yea, I'm lazy.
+     */
+    pgmPoolFreeByPage(pPool, pShwPage, pShwPde->idx, iPDDst);
+    SHW_PDE_ATOMIC_SET(*pPdeDst, 0);
+
+    PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+    PGM_INVL_VCPU_TLBS(pVCpu);
+    return VINF_PGM_SYNCPAGE_MODIFIED_PDE;
+
+
+# elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
+    && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
+    && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT)
+    NOREF(PdeSrc);
+
+#  ifdef PGM_SYNC_N_PAGES
+    /*
+     * Get the shadow PDE, find the shadow page table in the pool.
+     */
+#   if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    X86PDE          PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
+
+#   elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    X86PDEPAE       PdeDst = pgmShwGetPaePDE(pVCpu, GCPtrPage);
+
+#   elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPDDst   = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    const unsigned  iPdpt    = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64; NOREF(iPdpt);
+    PX86PDPAE       pPDDst   = NULL;            /* initialized to shut up gcc */
+    X86PDEPAE       PdeDst;
+    PX86PDPT        pPdptDst = NULL;            /* initialized to shut up gcc */
+
+    int rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pPDDst && pPdptDst);
+    PdeDst = pPDDst->a[iPDDst];
+
+#   elif PGM_SHW_TYPE == PGM_TYPE_EPT
+    const unsigned  iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PEPTPD          pPDDst;
+    EPTPDE          PdeDst;
+
+    int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, NULL, &pPDDst);
+    if (rc != VINF_SUCCESS)
+    {
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPDDst);
+    PdeDst = pPDDst->a[iPDDst];
+#   endif
+    /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
+    if (!SHW_PDE_IS_P(PdeDst))
+    {
+        AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)PdeDst.u));
+        Log(("CPU%d: SyncPage: Pde at %RGv changed behind our back!\n", pVCpu->idCpu, GCPtrPage));
+        return VINF_SUCCESS;    /* force the instruction to be executed again. */
+    }
+
+    /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
+    if (SHW_PDE_IS_BIG(PdeDst))
+    {
+        Assert(pVM->pgm.s.fNestedPaging);
+        Log(("CPU%d: SyncPage: Pde (big:%RX64) at %RGv changed behind our back!\n", pVCpu->idCpu, PdeDst.u, GCPtrPage));
+        return VINF_SUCCESS;
+    }
+
+    /* Mask away the page offset. */
+    GCPtrPage &= ~((RTGCPTR)0xfff);
+
+    PPGMPOOLPAGE  pShwPage = pgmPoolGetPage(pPool, PdeDst.u & SHW_PDE_PG_MASK);
+    PSHWPT        pPTDst   = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+
+    Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
+    if (    cPages > 1
+        &&  !(uErr & X86_TRAP_PF_P)
+        &&  !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+    {
+        /*
+         * This code path is currently only taken when the caller is PGMTrap0eHandler
+         * for non-present pages!
+         *
+         * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
+         * deal with locality.
+         */
+        unsigned        iPTDst    = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        const unsigned  iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
+        if (iPTDst < PGM_SYNC_NR_PAGES / 2)
+            iPTDst = 0;
+        else
+            iPTDst -= PGM_SYNC_NR_PAGES / 2;
+        for (; iPTDst < iPTDstEnd; iPTDst++)
+        {
+            if (!SHW_PTE_IS_P(pPTDst->a[iPTDst]))
+            {
+                RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
+                                                          | (iPTDst << GUEST_PAGE_SHIFT));
+
+                PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
+                Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
+                      GCPtrCurPage,
+                      SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+                      SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+
+                if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                    break;
+            }
+            else
+                Log4(("%RGv iPTDst=%x pPTDst->a[iPTDst] %RX64\n",
+                      (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT)) | (iPTDst << GUEST_PAGE_SHIFT), iPTDst, SHW_PTE_LOG64(pPTDst->a[iPTDst]) ));
+        }
+    }
+    else
+#  endif /* PGM_SYNC_N_PAGES */
+    {
+        const unsigned  iPTDst       = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        RTGCPTR         GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
+                                                          | (iPTDst << GUEST_PAGE_SHIFT));
+
+        PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
+
+        Log2(("SyncPage: 4K  %RGv PteSrc:{P=1 RW=1 U=1}PteDst=%08llx%s\n",
+              GCPtrPage,
+              SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+              SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+    }
+    return VINF_SUCCESS;
+
+# else
+    NOREF(PdeSrc);
+    AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+# endif
+}
+
+#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+
+#if !defined(IN_RING3) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
+
+/**
+ * Sync a shadow page for a nested-guest page.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pPte            The shadow page table entry.
+ * @param   GCPhysPage      The guest-physical address of the page.
+ * @param   pShwPage        The shadow page of the page table.
+ * @param   iPte            The index of the page table entry.
+ * @param   pGstWalkAll     The guest page table walk result.
+ *
+ * @note    Not to be used for 2/4MB pages!
+ */
+static void PGM_BTH_NAME(NestedSyncPageWorker)(PVMCPUCC pVCpu, PSHWPTE pPte, RTGCPHYS GCPhysPage, PPGMPOOLPAGE pShwPage,
+                                               unsigned iPte, PCPGMPTWALKGST pGstWalkAll)
+{
+    /*
+     * Do not make assumptions about anything other than the final PTE entry in the
+     * guest page table walk result. For instance, while mapping 2M PDEs as 4K pages,
+     * the PDE might still be having its leaf bit set.
+     *
+     * In the future, we could consider introducing a generic SLAT macro like PSLATPTE
+     * and using that instead of passing the full SLAT translation result.
+     */
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
+    Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
+    Assert(!pShwPage->fDirty);
+    Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT);
+    AssertMsg((pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK) == GCPhysPage,
+              ("PTE address mismatch. GCPhysPage=%RGp Pte=%RX64\n", GCPhysPage, pGstWalkAll->u.Ept.Pte.u & EPT_PTE_PG_MASK));
+
+    /*
+     * Find the ram range.
+     */
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageEx(pVCpu->CTX_SUFF(pVM), GCPhysPage, &pPage);
+    AssertRCReturnVoid(rc);
+
+    Assert(!PGM_PAGE_IS_BALLOONED(pPage));
+
+# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+    /* Make the page writable if necessary. */
+    /** @todo This needs to be applied to the regular case below, not here. And,
+     *        no we should *NOT* make the page writble, instead we need to write
+     *        protect them if necessary. */
+    if (    PGM_PAGE_GET_TYPE(pPage)  == PGMPAGETYPE_RAM
+        &&  (   PGM_PAGE_IS_ZERO(pPage)
+             || (   (pGstWalkAll->u.Ept.Pte.u & EPT_E_WRITE)
+                 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+#  ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+                 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
+#  endif
+#  ifdef VBOX_WITH_PAGE_SHARING
+                 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
+#  endif
+                 && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_BALLOONED
+             )
+          )
+       )
+    {
+        AssertMsgFailed(("GCPhysPage=%RGp\n", GCPhysPage)); /** @todo Shouldn't happen but if it does deal with it later. */
+    }
+# endif
+
+    /*
+     * Make page table entry.
+     */
+    SHWPTE Pte;
+    uint64_t const fGstShwPteFlags = pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptShadowedPteMask;
+    if (!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) || PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
+    {
+        /** @todo access bit. */
+        Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | fGstShwPteFlags;
+        Log7Func(("regular page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
+    }
+    else if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+    {
+        /** @todo access bit. */
+        Pte.u = PGM_PAGE_GET_HCPHYS(pPage) | (fGstShwPteFlags & ~EPT_E_WRITE);
+        Log7Func(("monitored page (%R[pgmpage]) at %RGp -> %RX64\n", pPage, GCPhysPage, Pte.u));
+    }
+    else
+    {
+        /** @todo Do MMIO optimizations here too? */
+        Log7Func(("mmio/all page (%R[pgmpage]) at %RGp -> 0\n", pPage, GCPhysPage));
+        Pte.u = 0;
+    }
+
+    /* Make sure only allocated pages are mapped writable. */
+    Assert(!SHW_PTE_IS_P_RW(Pte) || PGM_PAGE_IS_ALLOCATED(pPage));
+
+    /*
+     * Keep user track up to date.
+     */
+    if (SHW_PTE_IS_P(Pte))
+    {
+        if (!SHW_PTE_IS_P(*pPte))
+            PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
+        else if (SHW_PTE_GET_HCPHYS(*pPte) != SHW_PTE_GET_HCPHYS(Pte))
+        {
+            Log2(("SyncPageWorker: deref! *pPte=%RX64 Pte=%RX64\n", SHW_PTE_LOG64(*pPte), SHW_PTE_LOG64(Pte)));
+            PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
+            PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPte);
+        }
+    }
+    else if (SHW_PTE_IS_P(*pPte))
+    {
+        Log2(("SyncPageWorker: deref! *pPte=%RX64\n", SHW_PTE_LOG64(*pPte)));
+        PGM_BTH_NAME(SyncPageWorkerTrackDeref)(pVCpu, pShwPage, SHW_PTE_GET_HCPHYS(*pPte), iPte, NIL_RTGCPHYS);
+    }
+
+    /*
+     * Commit the entry.
+     */
+    SHW_PTE_ATOMIC_SET2(*pPte, Pte);
+    return;
+}
+
+
+/**
+ * Syncs a nested-guest page.
+ *
+ * There are no conflicts at this point, neither is there any need for
+ * page table allocations.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   GCPhysNestedPage    The nested-guest physical address of the page being
+ *                              synced.
+ * @param   GCPhysPage          The guest-physical address of the page being synced.
+ * @param   cPages              Number of pages to sync (PGM_SYNC_N_PAGES) (default=1).
+ * @param   uErr                The page fault error (X86_TRAP_PF_XXX).
+ * @param   pGstWalkAll         The guest page table walk result.
+ */
+static int PGM_BTH_NAME(NestedSyncPage)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, unsigned cPages,
+                                        uint32_t uErr, PPGMPTWALKGST pGstWalkAll)
+{
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
+    Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
+    Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
+
+    PVMCC    pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
+    Log7Func(("GCPhysNestedPage=%RGv GCPhysPage=%RGp cPages=%u uErr=%#x\n", GCPhysNestedPage, GCPhysPage, cPages, uErr));
+    RT_NOREF_PV(uErr); RT_NOREF_PV(cPages);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Get the shadow PDE, find the shadow page table in the pool.
+     */
+    unsigned const iPde = ((GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK);
+    PEPTPD pPd;
+    int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, NULL, &pPd, pGstWalkAll);
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+    {
+        Log(("Failed to fetch EPT PD for %RGp (%RGp) rc=%Rrc\n", GCPhysNestedPage, GCPhysPage, rc));
+        return rc;
+    }
+    Assert(pPd);
+    EPTPDE Pde = pPd->a[iPde];
+
+    /* In the guest SMP case we could have blocked while another VCPU reused this page table. */
+    if (!SHW_PDE_IS_P(Pde))
+    {
+        AssertMsg(pVM->cCpus > 1, ("Unexpected missing PDE %RX64\n", (uint64_t)Pde.u));
+        Log7Func(("CPU%d: SyncPage: Pde at %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
+        return VINF_SUCCESS;    /* force the instruction to be executed again. */
+    }
+
+    /* Can happen in the guest SMP case; other VCPU activated this PDE while we were blocking to handle the page fault. */
+    if (SHW_PDE_IS_BIG(Pde))
+    {
+        Log7Func(("CPU%d: SyncPage: %RGp changed behind our back!\n", pVCpu->idCpu, GCPhysNestedPage));
+        return VINF_SUCCESS;
+    }
+
+    PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, Pde.u & EPT_PDE_PG_MASK);
+    PEPTPT       pPt      = (PEPTPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+
+    /*
+     * If we've shadowed a guest EPT PDE that maps a 2M page using a 4K table,
+     * then sync the 4K sub-page in the 2M range.
+     */
+    if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)
+    {
+        Assert(!SHW_PDE_IS_BIG(Pde));
+
+        Assert(pGstWalkAll->u.Ept.Pte.u == 0);
+        Assert((Pde.u & EPT_PRESENT_MASK) == (pGstWalkAll->u.Ept.Pde.u & EPT_PRESENT_MASK));
+        Assert(pShwPage->GCPhys == (pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK));
+
+#if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
+        PPGMPAGE pPage;
+        rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage); AssertRC(rc);
+        Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
+        Assert(pShwPage->enmKind == PGMPOOLKIND_EPT_PT_FOR_EPT_2MB);
+#endif
+        uint64_t const fGstPteFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask & ~EPT_E_LEAF;
+        pGstWalkAll->u.Ept.Pte.u = GCPhysPage | fGstPteFlags;
+
+        unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
+        Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
+
+        /* Restore modifications did to the guest-walk result above in case callers might inspect them later. */
+        pGstWalkAll->u.Ept.Pte.u = 0;
+        return VINF_SUCCESS;
+    }
+
+    Assert(cPages == 1 || !(uErr & X86_TRAP_PF_P));
+# ifdef PGM_SYNC_N_PAGES
+    if (    cPages > 1
+        && !(uErr & X86_TRAP_PF_P)
+        && !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+    {
+        /*
+         * This code path is currently only taken for non-present pages!
+         *
+         * We're setting PGM_SYNC_NR_PAGES pages around the faulting page to sync it and
+         * deal with locality.
+         */
+        unsigned       iPte    = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        unsigned const iPteEnd = RT_MIN(iPte + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPt->a));
+        if (iPte < PGM_SYNC_NR_PAGES / 2)
+            iPte = 0;
+        else
+            iPte -= PGM_SYNC_NR_PAGES / 2;
+        for (; iPte < iPteEnd; iPte++)
+        {
+            if (!SHW_PTE_IS_P(pPt->a[iPte]))
+            {
+                PGMPTWALKGST GstWalkPt;
+                PGMPTWALK    WalkPt;
+                GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
+                GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT);
+                rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt,
+                                    &GstWalkPt);
+                if (RT_SUCCESS(rc))
+                    PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], WalkPt.GCPhys, pShwPage, iPte, &GstWalkPt);
+                else
+                {
+                    /*
+                     * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
+                     * Ensure the shadow tables entry is not-present.
+                     */
+                    /** @todo Potential room for optimization (explained in NestedSyncPT). */
+                    AssertMsg(!pPt->a[iPte].u, ("%RX64\n", pPt->a[iPte].u));
+                }
+                Log7Func(("Many: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
+                if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                    break;
+            }
+            else
+            {
+#  ifdef VBOX_STRICT
+                /* Paranoia - Verify address of the page is what it should be. */
+                PGMPTWALKGST GstWalkPt;
+                PGMPTWALK    WalkPt;
+                GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
+                GCPhysNestedPage |= (iPte << GUEST_PAGE_SHIFT);
+                rc = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/, &WalkPt, &GstWalkPt);
+                AssertRC(rc);
+                PPGMPAGE pPage;
+                rc = pgmPhysGetPageEx(pVM, WalkPt.GCPhys, &pPage);
+                AssertRC(rc);
+                AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]),
+                          ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp HCPhys=%RHp Shw=%RHp\n",
+                           GCPhysNestedPage, WalkPt.GCPhys, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
+#  endif
+                Log7Func(("Many3: %RGp iPte=%u ShwPte=%RX64\n", GCPhysNestedPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
+            }
+        }
+    }
+    else
+# endif /* PGM_SYNC_N_PAGES */
+    {
+        unsigned const iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
+        Log7Func(("4K: GCPhysPage=%RGp iPte=%u ShwPte=%08llx\n", GCPhysPage, iPte, SHW_PTE_LOG64(pPt->a[iPte])));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sync a shadow page table for a nested-guest page table.
+ *
+ * The shadow page table is not present in the shadow PDE.
+ *
+ * Handles mapping conflicts.
+ *
+ * A precondition for this method is that the shadow PDE is not present.  The
+ * caller must take the PGM lock before checking this and continue to hold it
+ * when calling this method.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   GCPhysNestedPage    The nested-guest physical page address of the page
+ *                              being synced.
+ * @param   GCPhysPage          The guest-physical address of the page being synced.
+ * @param   pGstWalkAll         The guest page table walk result.
+ */
+static int PGM_BTH_NAME(NestedSyncPT)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNestedPage, RTGCPHYS GCPhysPage, PPGMPTWALKGST pGstWalkAll)
+{
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysPage);
+    Assert(!(GCPhysNestedPage & GUEST_PAGE_OFFSET_MASK));
+    Assert(!(GCPhysPage & GUEST_PAGE_OFFSET_MASK));
+
+    PVMCC    pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    Log7Func(("GCPhysNestedPage=%RGp GCPhysPage=%RGp\n", GCPhysNestedPage, GCPhysPage));
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+
+    PEPTPD         pPd;
+    PEPTPDPT       pPdpt;
+    unsigned const iPde = (GCPhysNestedPage >> EPT_PD_SHIFT) & EPT_PD_MASK;
+    int rc = pgmShwGetNestedEPTPDPtr(pVCpu, GCPhysNestedPage, &pPdpt, &pPd, pGstWalkAll);
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+    {
+        STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPd);
+    PSHWPDE pPde = &pPd->a[iPde];
+
+    unsigned const iPdpt = (GCPhysNestedPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdpt->a[iPdpt].u & EPT_PDPTE_PG_MASK);
+    Assert(pShwPde->enmKind == PGMPOOLKIND_EPT_PD_FOR_EPT_PD);
+
+    SHWPDE Pde = *pPde;
+    Assert(!SHW_PDE_IS_P(Pde));    /* We're only supposed to call SyncPT on PDE!P and conflicts. */
+
+# ifdef PGM_WITH_LARGE_PAGES
+    if (BTH_IS_NP_ACTIVE(pVM))
+    {
+        /*
+         * Check if the guest is mapping a 2M page here.
+         */
+        PPGMPAGE pPage;
+        rc = pgmPhysGetPageEx(pVM, GCPhysPage & X86_PDE2M_PAE_PG_MASK, &pPage);
+        AssertRCReturn(rc, rc);
+        if (pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF)
+        {
+            /* A20 is always enabled in VMX root and non-root operation. */
+            Assert(PGM_A20_IS_ENABLED(pVCpu));
+
+            RTHCPHYS HCPhys = NIL_RTHCPHYS;
+            if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
+            {
+                STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
+                AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+                HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+            }
+            else if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
+            {
+                /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
+                rc = pgmPhysRecheckLargePage(pVM, GCPhysPage, pPage);
+                if (RT_SUCCESS(rc))
+                {
+                    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+                    Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
+                    HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+                }
+            }
+            else if (PGMIsUsingLargePages(pVM))
+            {
+                rc = pgmPhysAllocLargePage(pVM, GCPhysPage);
+                if (RT_SUCCESS(rc))
+                {
+                    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+                    Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
+                    HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+                }
+            }
+
+            /*
+             * If we have a 2M large page, we can map the guest's 2M large page right away.
+             */
+            uint64_t const fShwBigPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedBigPdeMask;
+            if (HCPhys != NIL_RTHCPHYS)
+            {
+                Pde.u = HCPhys | fShwBigPdeFlags;
+                Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzBigPdeMask));
+                Assert(Pde.u & EPT_E_LEAF);
+                SHW_PDE_ATOMIC_SET2(*pPde, Pde);
+
+                /* Add a reference to the first page only. */
+                PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPde);
+
+                Assert(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED);
+
+                STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+                Log7Func(("GstPde=%RGp ShwPde=%RX64 [2M]\n", pGstWalkAll->u.Ept.Pde.u, Pde.u));
+                return VINF_SUCCESS;
+            }
+
+            /*
+             * We didn't get a perfect 2M fit. Split the 2M page into 4K pages.
+             * The page ought not to be marked as a big (2M) page at this point.
+             */
+            Assert(PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE);
+
+            /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
+            PGMPOOLACCESS enmAccess;
+            {
+                Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_USER_EXECUTE));  /* Mode-based execute control for EPT not supported. */
+                bool const fNoExecute = !(pGstWalkAll->u.Ept.Pde.u & EPT_E_EXECUTE);
+                if (pGstWalkAll->u.Ept.Pde.u & EPT_E_WRITE)
+                    enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
+                else
+                    enmAccess = fNoExecute ? PGMPOOLACCESS_SUPERVISOR_R_NX  : PGMPOOLACCESS_SUPERVISOR_R;
+            }
+
+            /*
+             * Allocate & map a 4K shadow table to cover the 2M guest page.
+             */
+            PPGMPOOLPAGE   pShwPage;
+            RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE2M_PG_MASK;
+            rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_2MB, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
+                              pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage);
+            if (   rc == VINF_SUCCESS
+                || rc == VINF_PGM_CACHED_PAGE)
+            { /* likely */ }
+            else
+            {
+               STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+               AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+            }
+
+            PSHWPT pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+            Assert(pPt);
+            Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
+            if (rc == VINF_SUCCESS)
+            {
+                /* The 4K PTEs shall inherit the flags of the 2M PDE page sans the leaf bit. */
+                uint64_t const fShwPteFlags = fShwBigPdeFlags & ~EPT_E_LEAF;
+
+                /* Sync each 4K pages in the 2M range. */
+                for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++)
+                {
+                    RTGCPHYS const GCPhysSubPage = GCPhysPt | (iPte << GUEST_PAGE_SHIFT);
+                    pGstWalkAll->u.Ept.Pte.u = GCPhysSubPage | fShwPteFlags;
+                    Assert(!(pGstWalkAll->u.Ept.Pte.u & pVCpu->pgm.s.fGstEptMbzPteMask));
+                    PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysSubPage, pShwPage, iPte, pGstWalkAll);
+                    Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [2M->4K]\n", pGstWalkAll->u.Ept.Pte, pPt->a[iPte].u, iPte));
+                    if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                        break;
+                }
+
+                /* Restore modifications did to the guest-walk result above in case callers might inspect them later. */
+                pGstWalkAll->u.Ept.Pte.u = 0;
+            }
+            else
+            {
+                Assert(rc == VINF_PGM_CACHED_PAGE);
+#  if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
+                /* Paranoia - Verify address of each of the subpages are what they should be. */
+                RTGCPHYS GCPhysSubPage = GCPhysPt;
+                for (unsigned iPte = 0; iPte < RT_ELEMENTS(pPt->a); iPte++, GCPhysSubPage += GUEST_PAGE_SIZE)
+                {
+                    PPGMPAGE pSubPage;
+                    rc = pgmPhysGetPageEx(pVM, GCPhysSubPage, &pSubPage);
+                    AssertRC(rc);
+                    AssertMsg(   PGM_PAGE_GET_HCPHYS(pSubPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte])
+                              || !SHW_PTE_IS_P(pPt->a[iPte]),
+                              ("PGM 2M page and shadow PTE conflict. GCPhysSubPage=%RGp Page=%RHp Shw=%RHp\n",
+                               GCPhysSubPage, PGM_PAGE_GET_HCPHYS(pSubPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
+                }
+#  endif
+                rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
+            }
+
+            /* Save the new PDE. */
+            uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
+            Pde.u = pShwPage->Core.Key | fShwPdeFlags;
+            Assert(!(Pde.u & EPT_E_LEAF));
+            Assert(!(Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask));
+            SHW_PDE_ATOMIC_SET2(*pPde, Pde);
+            STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+            Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
+            return rc;
+        }
+    }
+# endif /* PGM_WITH_LARGE_PAGES */
+
+    /*
+     * Allocate & map the shadow page table.
+     */
+    PSHWPT       pPt;
+    PPGMPOOLPAGE pShwPage;
+
+    RTGCPHYS const GCPhysPt = pGstWalkAll->u.Ept.Pde.u & EPT_PDE_PG_MASK;
+    rc = pgmPoolAlloc(pVM, GCPhysPt, PGMPOOLKIND_EPT_PT_FOR_EPT_PT, PGMPOOLACCESS_DONTCARE,
+                      PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPde, false /*fLockPage*/, &pShwPage);
+    if (   rc == VINF_SUCCESS
+        || rc == VINF_PGM_CACHED_PAGE)
+    { /* likely */ }
+    else
+    {
+       STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+       AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+    }
+
+    pPt = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    Assert(pPt);
+    Assert(PGMPOOL_PAGE_IS_NESTED(pShwPage));
+
+    if (rc == VINF_SUCCESS)
+    {
+        /* Sync the page we've already translated through SLAT. */
+        const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPte], GCPhysPage, pShwPage, iPte, pGstWalkAll);
+        Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u, iPte));
+
+        /* Sync the rest of page table (expensive but might be cheaper than nested-guest VM-exits in hardware). */
+        for (unsigned iPteCur = 0; iPteCur < RT_ELEMENTS(pPt->a); iPteCur++)
+        {
+            if (iPteCur != iPte)
+            {
+                PGMPTWALKGST GstWalkPt;
+                PGMPTWALK    WalkPt;
+                GCPhysNestedPage &= ~(SHW_PT_MASK << SHW_PT_SHIFT);
+                GCPhysNestedPage |= (iPteCur << GUEST_PAGE_SHIFT);
+                int const rc2 = pgmGstSlatWalk(pVCpu, GCPhysNestedPage, false /*fIsLinearAddrValid*/, 0 /*GCPtrNested*/,
+                                               &WalkPt, &GstWalkPt);
+                if (RT_SUCCESS(rc2))
+                {
+                    PGM_BTH_NAME(NestedSyncPageWorker)(pVCpu, &pPt->a[iPteCur], WalkPt.GCPhys, pShwPage, iPteCur, &GstWalkPt);
+                    Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u\n", GstWalkPt.u.Ept.Pte.u, pPt->a[iPteCur].u, iPteCur));
+                }
+                else
+                {
+                    /*
+                     * This could be MMIO pages reserved by the nested-hypevisor or genuinely not-present pages.
+                     * Ensure the shadow tables entry is not-present.
+                     */
+                    /** @todo We currently don't configure these to cause EPT misconfigs but rather trap
+                     *        them using EPT violations and walk the guest EPT tables to determine
+                     *        whether they are EPT misconfigs VM-exits for the nested-hypervisor. We
+                     *        could optimize this by using a specific combination of reserved bits
+                     *        which we could immediately identify as EPT misconfigs of the
+                     *        nested-hypervisor without having to walk its EPT tables. However, tracking
+                     *        non-present entries might be tricky...
+                     */
+                    AssertMsg(!pPt->a[iPteCur].u, ("%RX64\n", pPt->a[iPteCur].u));
+                }
+                if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                    break;
+            }
+        }
+    }
+    else
+    {
+        Assert(rc == VINF_PGM_CACHED_PAGE);
+# if defined(VBOX_STRICT) && defined(DEBUG_ramshankar)
+        /* Paranoia - Verify address of the page is what it should be. */
+        PPGMPAGE pPage;
+        rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+        AssertRC(rc);
+        const unsigned iPte = (GCPhysNestedPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == SHW_PTE_GET_HCPHYS(pPt->a[iPte]) || !SHW_PTE_IS_P(pPt->a[iPte]),
+                  ("PGM page and shadow PTE address conflict. GCPhysNestedPage=%RGp GCPhysPage=%RGp Page=%RHp Shw=%RHp\n",
+                   GCPhysNestedPage, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage), SHW_PTE_GET_HCPHYS(pPt->a[iPte])));
+        Log7Func(("GstPte=%RGp ShwPte=%RX64 iPte=%u [cache]\n", pGstWalkAll->u.Ept.Pte.u, pPt->a[iPte].u,  iPte));
+# endif
+        rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
+    }
+
+    /* Save the new PDE. */
+    uint64_t const fShwPdeFlags = pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptShadowedPdeMask;
+    Assert(!(pGstWalkAll->u.Ept.Pde.u & EPT_E_LEAF));
+    Assert(!(pGstWalkAll->u.Ept.Pde.u & pVCpu->pgm.s.fGstEptMbzPdeMask));
+    Pde.u = pShwPage->Core.Key | fShwPdeFlags;
+    SHW_PDE_ATOMIC_SET2(*pPde, Pde);
+    Log7Func(("GstPde=%RGp ShwPde=%RX64 iPde=%u\n", pGstWalkAll->u.Ept.Pde.u, pPde->u, iPde));
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+    return rc;
+}
+
+#endif  /* !IN_RING3 && VBOX_WITH_NESTED_HWVIRT_VMX_EPT && PGM_SHW_TYPE == PGM_TYPE_EPT*/
+#if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
+
+/**
+ * Handle dirty bit tracking faults.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uErr        Page fault error code.
+ * @param   pPdeSrc     Guest page directory entry.
+ * @param   pPdeDst     Shadow page directory entry.
+ * @param   GCPtrPage   Guest context page address.
+ */
+static int PGM_BTH_NAME(CheckDirtyPageFault)(PVMCPUCC pVCpu, uint32_t uErr, PSHWPDE pPdeDst, GSTPDE const *pPdeSrc,
+                                             RTGCPTR GCPtrPage)
+{
+    PVMCC       pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL    pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    NOREF(uErr);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Handle big page.
+     */
+    if ((pPdeSrc->u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
+    {
+        if ((pPdeDst->u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY))
+        {
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
+            Assert(pPdeSrc->u & X86_PDE_RW);
+
+            /* Note: No need to invalidate this entry on other VCPUs as a stale TLB entry will not harm; write access will simply
+             *       fault again and take this path to only invalidate the entry (see below). */
+            SHWPDE PdeDst = *pPdeDst;
+            PdeDst.u &= ~(SHWUINT)PGM_PDFLAGS_TRACK_DIRTY;
+            PdeDst.u |= X86_PDE_RW | X86_PDE_A;
+            SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
+            PGM_INVL_BIG_PG(pVCpu, GCPtrPage);
+            return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;    /* restarts the instruction. */
+        }
+
+# ifdef IN_RING0
+        /* Check for stale TLB entry; only applies to the SMP guest case. */
+        if (   pVM->cCpus > 1
+            && (pPdeDst->u & (X86_PDE_P | X86_PDE_RW | X86_PDE_A)) == (X86_PDE_P | X86_PDE_RW | X86_PDE_A))
+        {
+            PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
+            if (pShwPage)
+            {
+                PSHWPT      pPTDst   = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+                PSHWPTE     pPteDst  = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
+                if (SHW_PTE_IS_P_RW(*pPteDst))
+                {
+                    /* Stale TLB entry. */
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
+                    PGM_INVL_PG(pVCpu, GCPtrPage);
+                    return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;    /* restarts the instruction. */
+                }
+            }
+        }
+# endif /* IN_RING0 */
+        return VINF_PGM_NO_DIRTY_BIT_TRACKING;
+    }
+
+    /*
+     * Map the guest page table.
+     */
+    PGSTPT pPTSrc;
+    int rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, GST_GET_PDE_GCPHYS(*pPdeSrc), &pPTSrc);
+    AssertRCReturn(rc, rc);
+
+    if (SHW_PDE_IS_P(*pPdeDst))
+    {
+        GSTPTE const  *pPteSrc = &pPTSrc->a[(GCPtrPage >> GST_PT_SHIFT) & GST_PT_MASK];
+        const GSTPTE   PteSrc  = *pPteSrc;
+
+        /*
+         * Map shadow page table.
+         */
+        PPGMPOOLPAGE    pShwPage = pgmPoolGetPage(pPool, pPdeDst->u & SHW_PDE_PG_MASK);
+        if (pShwPage)
+        {
+            PSHWPT      pPTDst   = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+            PSHWPTE     pPteDst  = &pPTDst->a[(GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK];
+            if (SHW_PTE_IS_P(*pPteDst))    /** @todo Optimize accessed bit emulation? */
+            {
+                if (SHW_PTE_IS_TRACK_DIRTY(*pPteDst))
+                {
+                    PPGMPAGE pPage  = pgmPhysGetPage(pVM, GST_GET_PTE_GCPHYS(PteSrc));
+                    SHWPTE   PteDst = *pPteDst;
+
+                    LogFlow(("DIRTY page trap addr=%RGv\n", GCPtrPage));
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageTrap));
+
+                    Assert(PteSrc.u & X86_PTE_RW);
+
+                    /* Note: No need to invalidate this entry on other VCPUs as a stale TLB
+                     *       entry will not harm; write access will simply fault again and
+                     *       take this path to only invalidate the entry.
+                     */
+                    if (RT_LIKELY(pPage))
+                    {
+                        if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
+                        {
+                            //AssertMsgFailed(("%R[pgmpage] - we don't set PGM_PTFLAGS_TRACK_DIRTY for these pages\n", pPage));
+                            Assert(!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage));
+                            /* Assuming write handlers here as the PTE is present (otherwise we wouldn't be here). */
+                            SHW_PTE_SET_RO(PteDst);
+                        }
+                        else
+                        {
+                            if (   PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
+                                && PGM_PAGE_GET_TYPE(pPage)  == PGMPAGETYPE_RAM)
+                            {
+                                rc = pgmPhysPageMakeWritable(pVM, pPage, GST_GET_PTE_GCPHYS(PteSrc));
+                                AssertRC(rc);
+                            }
+                            if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
+                                SHW_PTE_SET_RW(PteDst);
+                            else
+                            {
+                                /* Still applies to shared pages. */
+                                Assert(!PGM_PAGE_IS_ZERO(pPage));
+                                SHW_PTE_SET_RO(PteDst);
+                            }
+                        }
+                    }
+                    else
+                        SHW_PTE_SET_RW(PteDst);  /** @todo r=bird: This doesn't make sense to me. */
+
+                    SHW_PTE_SET(PteDst, (SHW_PTE_GET_U(PteDst) | X86_PTE_D | X86_PTE_A) & ~(uint64_t)PGM_PTFLAGS_TRACK_DIRTY);
+                    SHW_PTE_ATOMIC_SET2(*pPteDst, PteDst);
+                    PGM_INVL_PG(pVCpu, GCPtrPage);
+                    return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;    /* restarts the instruction. */
+                }
+
+# ifdef IN_RING0
+                /* Check for stale TLB entry; only applies to the SMP guest case. */
+                if (    pVM->cCpus > 1
+                    &&  SHW_PTE_IS_RW(*pPteDst)
+                    &&  SHW_PTE_IS_A(*pPteDst))
+                {
+                    /* Stale TLB entry. */
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageStale));
+                    PGM_INVL_PG(pVCpu, GCPtrPage);
+                    return VINF_PGM_HANDLED_DIRTY_BIT_FAULT;    /* restarts the instruction. */
+                }
+# endif
+            }
+        }
+        else
+            AssertMsgFailed(("pgmPoolGetPageByHCPhys %RGp failed!\n", pPdeDst->u & SHW_PDE_PG_MASK));
+    }
+
+    return VINF_PGM_NO_DIRTY_BIT_TRACKING;
+}
+
+#endif /* PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
+
+/**
+ * Sync a shadow page table.
+ *
+ * The shadow page table is not present in the shadow PDE.
+ *
+ * Handles mapping conflicts.
+ *
+ * This is called by VerifyAccessSyncPage, PrefetchPage, InvalidatePage (on
+ * conflict), and Trap0eHandler.
+ *
+ * A precondition for this method is that the shadow PDE is not present.  The
+ * caller must take the PGM lock before checking this and continue to hold it
+ * when calling this method.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   iPDSrc      Page directory index.
+ * @param   pPDSrc      Source page directory (i.e. Guest OS page directory).
+ *                      Assume this is a temporary mapping.
+ * @param   GCPtrPage   GC Pointer of the page that caused the fault
+ */
+static int PGM_BTH_NAME(SyncPT)(PVMCPUCC pVCpu, unsigned iPDSrc, PGSTPD pPDSrc, RTGCPTR GCPtrPage)
+{
+    PVMCC       pVM   = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL    pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
+
+#if 0 /* rarely useful; leave for debugging. */
+    STAM_COUNTER_INC(&pVCpu->pgm.s.StatSyncPtPD[iPDSrc]);
+#endif
+    LogFlow(("SyncPT: GCPtrPage=%RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+#if (   PGM_GST_TYPE == PGM_TYPE_32BIT \
+     || PGM_GST_TYPE == PGM_TYPE_PAE \
+     || PGM_GST_TYPE == PGM_TYPE_AMD64) \
+ && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
+ && PGM_SHW_TYPE != PGM_TYPE_NONE
+    int             rc       = VINF_SUCCESS;
+
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+
+    /*
+     * Some input validation first.
+     */
+    AssertMsg(iPDSrc == ((GCPtrPage >> GST_PD_SHIFT) & GST_PD_MASK), ("iPDSrc=%x GCPtrPage=%RGv\n", iPDSrc, GCPtrPage));
+
+    /*
+     * Get the relevant shadow PDE entry.
+     */
+# if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDDst   = GCPtrPage >> SHW_PD_SHIFT;
+    PSHWPDE         pPdeDst  = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde  = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    Assert(pShwPde);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    const unsigned  iPDDst   = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PPGMPOOLPAGE    pShwPde  = NULL;
+    PX86PDPAE       pPDDst;
+    PSHWPDE         pPdeDst;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pShwPde);
+
+    pPDDst  = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+    pPdeDst = &pPDDst->a[iPDDst];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPdpt    = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    const unsigned  iPDDst   = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PX86PDPAE       pPDDst   = NULL;            /* initialized to shut up gcc */
+    PX86PDPT        pPdptDst = NULL;            /* initialized to shut up gcc */
+    rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pPDDst);
+    PSHWPDE         pPdeDst  = &pPDDst->a[iPDDst];
+
+# endif
+    SHWPDE          PdeDst   = *pPdeDst;
+
+# if PGM_GST_TYPE == PGM_TYPE_AMD64
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde  = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
+    Assert(pShwPde);
+# endif
+
+    Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P.*/
+
+    /*
+     * Sync the page directory entry.
+     */
+    GSTPDE      PdeSrc = pPDSrc->a[iPDSrc];
+    const bool  fPageTable = !(PdeSrc.u & X86_PDE_PS) || !GST_IS_PSE_ACTIVE(pVCpu);
+    if (   (PdeSrc.u & X86_PDE_P)
+        && (fPageTable ? GST_IS_PDE_VALID(pVCpu, PdeSrc) : GST_IS_BIG_PDE_VALID(pVCpu, PdeSrc)) )
+    {
+        /*
+         * Allocate & map the page table.
+         */
+        PSHWPT          pPTDst;
+        PPGMPOOLPAGE    pShwPage;
+        RTGCPHYS        GCPhys;
+        if (fPageTable)
+        {
+            GCPhys = GST_GET_PDE_GCPHYS(PdeSrc);
+# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+            /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+            GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
+# endif
+            rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                              pShwPde->idx, iPDDst, false /*fLockPage*/,
+                              &pShwPage);
+        }
+        else
+        {
+            PGMPOOLACCESS enmAccess;
+# if PGM_WITH_NX(PGM_GST_TYPE, PGM_SHW_TYPE)
+            const bool  fNoExecute = (PdeSrc.u & X86_PDE_PAE_NX) && GST_IS_NX_ACTIVE(pVCpu);
+# else
+            const bool  fNoExecute = false;
+# endif
+
+            GCPhys = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
+# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+            /* Select the right PDE as we're emulating a 4MB page directory with two 2 MB shadow PDEs.*/
+            GCPhys = PGM_A20_APPLY(pVCpu, GCPhys | (GCPtrPage & (1 << X86_PD_PAE_SHIFT)));
+# endif
+            /* Determine the right kind of large page to avoid incorrect cached entry reuse. */
+            if (PdeSrc.u & X86_PDE_US)
+            {
+                if (PdeSrc.u & X86_PDE_RW)
+                    enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_RW_NX : PGMPOOLACCESS_USER_RW;
+                else
+                    enmAccess = (fNoExecute) ? PGMPOOLACCESS_USER_R_NX  : PGMPOOLACCESS_USER_R;
+            }
+            else
+            {
+                if (PdeSrc.u & X86_PDE_RW)
+                    enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_RW_NX : PGMPOOLACCESS_SUPERVISOR_RW;
+                else
+                    enmAccess = (fNoExecute) ? PGMPOOLACCESS_SUPERVISOR_R_NX  : PGMPOOLACCESS_SUPERVISOR_R;
+            }
+            rc = pgmPoolAlloc(pVM, GCPhys, BTH_PGMPOOLKIND_PT_FOR_BIG, enmAccess, PGM_A20_IS_ENABLED(pVCpu),
+                              pShwPde->idx, iPDDst, false /*fLockPage*/,
+                              &pShwPage);
+        }
+        if (rc == VINF_SUCCESS)
+            pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+        else if (rc == VINF_PGM_CACHED_PAGE)
+        {
+            /*
+             * The PT was cached, just hook it up.
+             */
+            if (fPageTable)
+                PdeDst.u = pShwPage->Core.Key | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
+            else
+            {
+                PdeDst.u = pShwPage->Core.Key | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
+                /* (see explanation and assumptions further down.) */
+                if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
+                {
+                    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
+                    PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
+                    PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
+                }
+            }
+            SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
+            PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+            return VINF_SUCCESS;
+        }
+        else
+            AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+        /** @todo Why do we bother preserving X86_PDE_AVL_MASK here?
+         * Both PGM_PDFLAGS_MAPPING and PGM_PDFLAGS_TRACK_DIRTY should be
+         * irrelevant at this point. */
+        PdeDst.u &= X86_PDE_AVL_MASK;
+        PdeDst.u |= pShwPage->Core.Key;
+
+        /*
+         * Page directory has been accessed (this is a fault situation, remember).
+         */
+        /** @todo
+         * Well, when the caller is PrefetchPage or InvalidatePage is isn't a
+         * fault situation.  What's more, the Trap0eHandler has already set the
+         * accessed bit.  So, it's actually just VerifyAccessSyncPage which
+         * might need setting the accessed flag.
+         *
+         * The best idea is to leave this change to the caller and add an
+         * assertion that it's set already. */
+        pPDSrc->a[iPDSrc].u |= X86_PDE_A;
+        if (fPageTable)
+        {
+            /*
+             * Page table - 4KB.
+             *
+             * Sync all or just a few entries depending on PGM_SYNC_N_PAGES.
+             */
+            Log2(("SyncPT:   4K  %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx}\n",
+                  GCPtrPage, PdeSrc.u & X86_PTE_P, !!(PdeSrc.u & X86_PTE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u));
+            PGSTPT pPTSrc;
+            rc = PGM_GCPHYS_2_PTR(pVM, GST_GET_PDE_GCPHYS(PdeSrc), &pPTSrc);
+            if (RT_SUCCESS(rc))
+            {
+                /*
+                 * Start by syncing the page directory entry so CSAM's TLB trick works.
+                 */
+                PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | X86_PDE_AVL_MASK))
+                         | GST_GET_PDE_SHW_FLAGS(pVCpu, PdeSrc);
+                SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
+                PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+
+                /*
+                 * Directory/page user or supervisor privilege: (same goes for read/write)
+                 *
+                 * Directory    Page    Combined
+                 * U/S          U/S     U/S
+                 *  0            0       0
+                 *  0            1       0
+                 *  1            0       0
+                 *  1            1       1
+                 *
+                 * Simple AND operation. Table listed for completeness.
+                 *
+                 */
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4K));
+# ifdef PGM_SYNC_N_PAGES
+                unsigned        iPTBase   = (GCPtrPage >> SHW_PT_SHIFT) & SHW_PT_MASK;
+                unsigned        iPTDst    = iPTBase;
+                const unsigned  iPTDstEnd = RT_MIN(iPTDst + PGM_SYNC_NR_PAGES / 2, RT_ELEMENTS(pPTDst->a));
+                if (iPTDst <= PGM_SYNC_NR_PAGES / 2)
+                    iPTDst = 0;
+                else
+                    iPTDst -= PGM_SYNC_NR_PAGES / 2;
+# else /* !PGM_SYNC_N_PAGES */
+                unsigned        iPTDst    = 0;
+                const unsigned  iPTDstEnd = RT_ELEMENTS(pPTDst->a);
+# endif /* !PGM_SYNC_N_PAGES */
+                RTGCPTR         GCPtrCur  = (GCPtrPage & ~(RTGCPTR)((1 << SHW_PD_SHIFT) - 1))
+                                          | ((RTGCPTR)iPTDst << GUEST_PAGE_SHIFT);
+# if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+                /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+                const unsigned  offPTSrc  = ((GCPtrPage >> SHW_PD_SHIFT) & 1) * 512;
+# else
+                const unsigned  offPTSrc  = 0;
+# endif
+                for (; iPTDst < iPTDstEnd; iPTDst++, GCPtrCur += GUEST_PAGE_SIZE)
+                {
+                    const unsigned iPTSrc = iPTDst + offPTSrc;
+                    const GSTPTE   PteSrc = pPTSrc->a[iPTSrc];
+                    if (PteSrc.u & X86_PTE_P)
+                    {
+                        PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], PdeSrc, PteSrc, pShwPage, iPTDst);
+                        Log2(("SyncPT:   4K+ %RGv PteSrc:{P=%d RW=%d U=%d raw=%08llx}%s dst.raw=%08llx iPTSrc=%x PdeSrc.u=%x physpte=%RGp\n",
+                              GCPtrCur,
+                              PteSrc.u & X86_PTE_P,
+                              !!(PteSrc.u & PdeSrc.u & X86_PTE_RW),
+                              !!(PteSrc.u & PdeSrc.u & X86_PTE_US),
+                              (uint64_t)PteSrc.u,
+                              SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : "", SHW_PTE_LOG64(pPTDst->a[iPTDst]), iPTSrc, PdeSrc.au32[0],
+                              (RTGCPHYS)(GST_GET_PDE_GCPHYS(PdeSrc) + iPTSrc*sizeof(PteSrc)) ));
+                    }
+                    /* else: the page table was cleared by the pool */
+                } /* for PTEs */
+            }
+        }
+        else
+        {
+            /*
+             * Big page - 2/4MB.
+             *
+             * We'll walk the ram range list in parallel and optimize lookups.
+             * We will only sync one shadow page table at a time.
+             */
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT4M));
+
+            /**
+             * @todo It might be more efficient to sync only a part of the 4MB
+             *       page (similar to what we do for 4KB PDs).
+             */
+
+            /*
+             * Start by syncing the page directory entry.
+             */
+            PdeDst.u = (PdeDst.u & (SHW_PDE_PG_MASK | (X86_PDE_AVL_MASK & ~PGM_PDFLAGS_TRACK_DIRTY)))
+                     | GST_GET_BIG_PDE_SHW_FLAGS(pVCpu, PdeSrc);
+
+            /*
+             * If the page is not flagged as dirty and is writable, then make it read-only
+             * at PD level, so we can set the dirty bit when the page is modified.
+             *
+             * ASSUMES that page access handlers are implemented on page table entry level.
+             *      Thus we will first catch the dirty access and set PDE.D and restart. If
+             *      there is an access handler, we'll trap again and let it work on the problem.
+             */
+            /** @todo move the above stuff to a section in the PGM documentation. */
+            Assert(!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY));
+            if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
+            {
+                STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,DirtyPageBig));
+                PdeDst.u |= PGM_PDFLAGS_TRACK_DIRTY;
+                PdeDst.u &= ~(SHWUINT)X86_PDE_RW;
+            }
+            SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
+            PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+
+            /*
+             * Fill the shadow page table.
+             */
+            /* Get address and flags from the source PDE. */
+            SHWPTE PteDstBase;
+            SHW_PTE_SET(PteDstBase, GST_GET_BIG_PDE_SHW_FLAGS_4_PTE(pVCpu, PdeSrc));
+
+            /* Loop thru the entries in the shadow PT. */
+            const RTGCPTR   GCPtr  = (GCPtrPage >> SHW_PD_SHIFT) << SHW_PD_SHIFT; NOREF(GCPtr);
+            Log2(("SyncPT:   BIG %RGv PdeSrc:{P=%d RW=%d U=%d raw=%08llx} Shw=%RGv GCPhys=%RGp %s\n",
+                  GCPtrPage, PdeSrc.u & X86_PDE_P, !!(PdeSrc.u & X86_PDE_RW), !!(PdeSrc.u & X86_PDE_US), (uint64_t)PdeSrc.u, GCPtr,
+                  GCPhys, PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY ? " Track-Dirty" : ""));
+            PPGMRAMRANGE    pRam   = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
+            unsigned        iPTDst = 0;
+            while (     iPTDst < RT_ELEMENTS(pPTDst->a)
+                   &&   !VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+            {
+                if (pRam && GCPhys >= pRam->GCPhys)
+                {
+# ifndef PGM_WITH_A20
+                    unsigned iHCPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
+# endif
+                    do
+                    {
+                        /* Make shadow PTE. */
+# ifdef PGM_WITH_A20
+                        PPGMPAGE    pPage = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
+# else
+                        PPGMPAGE    pPage = &pRam->aPages[iHCPage];
+# endif
+                        SHWPTE      PteDst;
+
+# ifndef VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+                        /* Try to make the page writable if necessary. */
+                        if (    PGM_PAGE_GET_TYPE(pPage)  == PGMPAGETYPE_RAM
+                            &&  (   PGM_PAGE_IS_ZERO(pPage)
+                                 || (   SHW_PTE_IS_RW(PteDstBase)
+                                     && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+#  ifdef VBOX_WITH_REAL_WRITE_MONITORED_PAGES
+                                     && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED
+#  endif
+#  ifdef VBOX_WITH_PAGE_SHARING
+                                     && PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_SHARED
+#  endif
+                                     && !PGM_PAGE_IS_BALLOONED(pPage))
+                                 )
+                           )
+                        {
+                            rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+                            AssertRCReturn(rc, rc);
+                            if (VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY))
+                                break;
+                        }
+# endif
+
+                        if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage))
+                            PGM_BTH_NAME(SyncHandlerPte)(pVM, pVCpu, pPage, GCPhys, SHW_PTE_GET_U(PteDstBase), &PteDst);
+                        else if (PGM_PAGE_IS_BALLOONED(pPage))
+                            SHW_PTE_SET(PteDst, 0); /* Handle ballooned pages at #PF time. */
+                        else
+                            SHW_PTE_SET(PteDst, PGM_PAGE_GET_HCPHYS(pPage) | SHW_PTE_GET_U(PteDstBase));
+
+                        /* Only map writable pages writable. */
+                        if (    SHW_PTE_IS_P_RW(PteDst)
+                            &&  PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED)
+                        {
+                            /* Still applies to shared pages. */
+                            Assert(!PGM_PAGE_IS_ZERO(pPage));
+                            SHW_PTE_SET_RO(PteDst);   /** @todo this isn't quite working yet... */
+                            Log3(("SyncPT: write-protecting %RGp pPage=%R[pgmpage] at %RGv\n", GCPhys, pPage, (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT))));
+                        }
+
+                        if (SHW_PTE_IS_P(PteDst))
+                            PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPage, PGM_PAGE_GET_TRACKING(pPage), pPage, iPTDst);
+
+                        /* commit it (not atomic, new table) */
+                        pPTDst->a[iPTDst] = PteDst;
+                        Log4(("SyncPT: BIG %RGv PteDst:{P=%d RW=%d U=%d raw=%08llx}%s\n",
+                              (RTGCPTR)(GCPtr | (iPTDst << SHW_PT_SHIFT)), SHW_PTE_IS_P(PteDst), SHW_PTE_IS_RW(PteDst), SHW_PTE_IS_US(PteDst), SHW_PTE_LOG64(PteDst),
+                              SHW_PTE_IS_TRACK_DIRTY(PteDst) ? " Track-Dirty" : ""));
+
+                        /* advance */
+                        GCPhys += GUEST_PAGE_SIZE;
+                        PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
+# ifndef PGM_WITH_A20
+                        iHCPage++;
+# endif
+                        iPTDst++;
+                    } while (   iPTDst < RT_ELEMENTS(pPTDst->a)
+                             && GCPhys <= pRam->GCPhysLast);
+
+                    /* Advance ram range list. */
+                    while (pRam && GCPhys > pRam->GCPhysLast)
+                        pRam = pRam->CTX_SUFF(pNext);
+                }
+                else if (pRam)
+                {
+                    Log(("Invalid pages at %RGp\n", GCPhys));
+                    do
+                    {
+                        SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
+                        GCPhys += GUEST_PAGE_SIZE;
+                        iPTDst++;
+                    } while (   iPTDst < RT_ELEMENTS(pPTDst->a)
+                             && GCPhys < pRam->GCPhys);
+                    PGM_A20_APPLY_TO_VAR(pVCpu,GCPhys);
+                }
+                else
+                {
+                    Log(("Invalid pages at %RGp (2)\n", GCPhys));
+                    for ( ; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
+                        SHW_PTE_SET(pPTDst->a[iPTDst], 0); /* Invalid page, we must handle them manually. */
+                }
+            } /* while more PTEs */
+        } /* 4KB / 4MB */
+    }
+    else
+        AssertRelease(!SHW_PDE_IS_P(PdeDst));
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+    if (RT_FAILURE(rc))
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
+    return rc;
+
+#elif (PGM_GST_TYPE == PGM_TYPE_REAL || PGM_GST_TYPE == PGM_TYPE_PROT) \
+    && !PGM_TYPE_IS_NESTED(PGM_SHW_TYPE) \
+    && (PGM_SHW_TYPE != PGM_TYPE_EPT || PGM_GST_TYPE == PGM_TYPE_PROT) \
+    && PGM_SHW_TYPE != PGM_TYPE_NONE
+    NOREF(iPDSrc); NOREF(pPDSrc);
+
+    STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+
+    /*
+     * Validate input a little bit.
+     */
+    int             rc = VINF_SUCCESS;
+# if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDDst  = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PSHWPDE         pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    Assert(pShwPde);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    const unsigned  iPDDst  = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PPGMPOOLPAGE    pShwPde = NULL;             /* initialized to shut up gcc */
+    PX86PDPAE       pPDDst;
+    PSHWPDE         pPdeDst;
+
+    /* Fetch the pgm pool shadow descriptor. */
+    rc = pgmShwGetPaePoolPagePD(pVCpu, GCPtrPage, &pShwPde);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pShwPde);
+
+    pPDDst  = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPde);
+    pPdeDst = &pPDDst->a[iPDDst];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPdpt   = (GCPtrPage >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
+    const unsigned  iPDDst  = (GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    PX86PDPAE       pPDDst  = NULL;             /* initialized to shut up gcc */
+    PX86PDPT        pPdptDst= NULL;             /* initialized to shut up gcc */
+    rc = pgmShwGetLongModePDPtr(pVCpu, GCPtrPage, NULL, &pPdptDst, &pPDDst);
+    AssertRCSuccessReturn(rc, rc);
+    Assert(pPDDst);
+    PSHWPDE         pPdeDst = &pPDDst->a[iPDDst];
+
+    /* Fetch the pgm pool shadow descriptor. */
+    PPGMPOOLPAGE    pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & X86_PDPE_PG_MASK);
+    Assert(pShwPde);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_EPT
+    const unsigned  iPdpt   = (GCPtrPage >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
+    const unsigned  iPDDst  = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PEPTPD          pPDDst;
+    PEPTPDPT        pPdptDst;
+
+    rc = pgmShwGetEPTPDPtr(pVCpu, GCPtrPage, &pPdptDst, &pPDDst);
+    if (rc != VINF_SUCCESS)
+    {
+        STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPDDst);
+    PSHWPDE         pPdeDst = &pPDDst->a[iPDDst];
+
+    /* Fetch the pgm pool shadow descriptor. */
+    /** @todo r=bird: didn't pgmShwGetEPTPDPtr just do this lookup already? */
+    PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pPool, pPdptDst->a[iPdpt].u & EPT_PDPTE_PG_MASK);
+    Assert(pShwPde);
+# endif
+    SHWPDE          PdeDst = *pPdeDst;
+
+    Assert(!SHW_PDE_IS_P(PdeDst)); /* We're only supposed to call SyncPT on PDE!P and conflicts.*/
+
+# if defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE
+    if (BTH_IS_NP_ACTIVE(pVM))
+    {
+        Assert(!VM_IS_NEM_ENABLED(pVM));
+
+        /* Check if we allocated a big page before for this 2 MB range. */
+        PPGMPAGE pPage;
+        rc = pgmPhysGetPageEx(pVM, PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PDE2M_PAE_PG_MASK), &pPage);
+        if (RT_SUCCESS(rc))
+        {
+            RTHCPHYS HCPhys = NIL_RTHCPHYS;
+            if (PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE)
+            {
+                if (PGM_A20_IS_ENABLED(pVCpu))
+                {
+                    STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageReused);
+                    AssertRelease(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+                    HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+                }
+                else
+                {
+                    PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
+                    pVM->pgm.s.cLargePagesDisabled++;
+                }
+            }
+            else if (   PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED
+                     && PGM_A20_IS_ENABLED(pVCpu))
+            {
+                /* Recheck the entire 2 MB range to see if we can use it again as a large page. */
+                rc = pgmPhysRecheckLargePage(pVM, GCPtrPage, pPage);
+                if (RT_SUCCESS(rc))
+                {
+                    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+                    Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
+                    HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+                }
+            }
+            else if (   PGMIsUsingLargePages(pVM)
+                     && PGM_A20_IS_ENABLED(pVCpu))
+            {
+                rc = pgmPhysAllocLargePage(pVM, GCPtrPage);
+                if (RT_SUCCESS(rc))
+                {
+                    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+                    Assert(PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE);
+                    HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
+                }
+                else
+                    LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
+            }
+
+            if (HCPhys != NIL_RTHCPHYS)
+            {
+#  if PGM_SHW_TYPE == PGM_TYPE_EPT
+                PdeDst.u = HCPhys | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE | EPT_E_LEAF | EPT_E_IGNORE_PAT | EPT_E_MEMTYPE_WB
+                         | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo do we need this? */;
+#  else
+                PdeDst.u = HCPhys | X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PS
+                         | (PdeDst.u & X86_PDE_AVL_MASK) /** @todo PGM_PD_FLAGS? */;
+#  endif
+                SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
+
+                Log(("SyncPT: Use large page at %RGp PDE=%RX64\n", GCPtrPage, PdeDst.u));
+                /* Add a reference to the first page only. */
+                PGM_BTH_NAME(SyncPageWorkerTrackAddref)(pVCpu, pShwPde, PGM_PAGE_GET_TRACKING(pPage), pPage, iPDDst);
+
+                STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+                return VINF_SUCCESS;
+            }
+        }
+    }
+# endif /* defined(PGM_WITH_LARGE_PAGES) && PGM_SHW_TYPE != PGM_TYPE_32BIT && PGM_SHW_TYPE != PGM_TYPE_PAE */
+
+    /*
+     * Allocate & map the page table.
+     */
+    PSHWPT          pPTDst;
+    PPGMPOOLPAGE    pShwPage;
+    RTGCPHYS        GCPhys;
+
+    /* Virtual address = physical address */
+    GCPhys = PGM_A20_APPLY(pVCpu, GCPtrPage & X86_PAGE_4K_BASE_MASK);
+    rc = pgmPoolAlloc(pVM, GCPhys & ~(RT_BIT_64(SHW_PD_SHIFT) - 1), BTH_PGMPOOLKIND_PT_FOR_PT, PGMPOOLACCESS_DONTCARE,
+                      PGM_A20_IS_ENABLED(pVCpu), pShwPde->idx, iPDDst, false /*fLockPage*/,
+                      &pShwPage);
+    if (    rc == VINF_SUCCESS
+        ||  rc == VINF_PGM_CACHED_PAGE)
+        pPTDst = (PSHWPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
+    else
+    {
+       STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+       AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_IPE_UNEXPECTED_INFO_STATUS);
+    }
+
+    if (rc == VINF_SUCCESS)
+    {
+        /* New page table; fully set it up. */
+        Assert(pPTDst);
+
+        /* Mask away the page offset. */
+        GCPtrPage &= ~(RTGCPTR)GUEST_PAGE_OFFSET_MASK;
+
+        for (unsigned iPTDst = 0; iPTDst < RT_ELEMENTS(pPTDst->a); iPTDst++)
+        {
+            RTGCPTR GCPtrCurPage = PGM_A20_APPLY(pVCpu, (GCPtrPage & ~(RTGCPTR)(SHW_PT_MASK << SHW_PT_SHIFT))
+                                                      | (iPTDst << GUEST_PAGE_SHIFT));
+
+            PGM_BTH_NAME(SyncPageWorker)(pVCpu, &pPTDst->a[iPTDst], GCPtrCurPage, pShwPage, iPTDst);
+            Log2(("SyncPage: 4K+ %RGv PteSrc:{P=1 RW=1 U=1} PteDst=%08llx%s\n",
+                  GCPtrCurPage,
+                  SHW_PTE_LOG64(pPTDst->a[iPTDst]),
+                  SHW_PTE_IS_TRACK_DIRTY(pPTDst->a[iPTDst]) ? " Track-Dirty" : ""));
+
+            if (RT_UNLIKELY(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)))
+                break;
+        }
+    }
+    else
+        rc = VINF_SUCCESS; /* Cached entry; assume it's still fully valid. */
+
+    /* Save the new PDE. */
+# if PGM_SHW_TYPE == PGM_TYPE_EPT
+    PdeDst.u = pShwPage->Core.Key | EPT_E_READ | EPT_E_WRITE | EPT_E_EXECUTE
+             | (PdeDst.u & X86_PDE_AVL_MASK /** @todo do we really need this? */);
+# else
+    PdeDst.u = pShwPage->Core.Key | X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A
+             | (PdeDst.u & X86_PDE_AVL_MASK /** @todo use a PGM_PD_FLAGS define */);
+# endif
+    SHW_PDE_ATOMIC_SET2(*pPdeDst, PdeDst);
+
+    STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPT), a);
+    if (RT_FAILURE(rc))
+        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncPTFailed));
+    return rc;
+
+#else
+    NOREF(iPDSrc); NOREF(pPDSrc);
+    AssertReleaseMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_SHW_TYPE, PGM_GST_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+#endif
+}
+
+
+
+/**
+ * Prefetch a page/set of pages.
+ *
+ * Typically used to sync commonly used pages before entering raw mode
+ * after a CR3 reload.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   Page to invalidate.
+ */
+PGM_BTH_DECL(int, PrefetchPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
+{
+#if (   PGM_GST_TYPE == PGM_TYPE_32BIT \
+     || PGM_GST_TYPE == PGM_TYPE_REAL \
+     || PGM_GST_TYPE == PGM_TYPE_PROT \
+     || PGM_GST_TYPE == PGM_TYPE_PAE \
+     || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
+ && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
+ && PGM_SHW_TYPE != PGM_TYPE_NONE
+    /*
+     * Check that all Guest levels thru the PDE are present, getting the
+     * PD and PDE in the processes.
+     */
+    int             rc     = VINF_SUCCESS;
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+#  if PGM_GST_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
+    PGSTPD          pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
+#  elif PGM_GST_TYPE == PGM_TYPE_PAE
+    unsigned        iPDSrc;
+    X86PDPE         PdpeSrc;
+    PGSTPD          pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
+    if (!pPDSrc)
+        return VINF_SUCCESS; /* not present */
+#  elif PGM_GST_TYPE == PGM_TYPE_AMD64
+    unsigned        iPDSrc;
+    PX86PML4E       pPml4eSrc;
+    X86PDPE         PdpeSrc;
+    PGSTPD          pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
+    if (!pPDSrc)
+        return VINF_SUCCESS; /* not present */
+#  endif
+    const GSTPDE    PdeSrc = pPDSrc->a[iPDSrc];
+# else
+    PGSTPD          pPDSrc = NULL;
+    const unsigned  iPDSrc = 0;
+    GSTPDE const    PdeSrc = { X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A }; /* faked so we don't have to #ifdef everything */
+# endif
+
+    if ((PdeSrc.u & (X86_PDE_P | X86_PDE_A)) == (X86_PDE_P | X86_PDE_A))
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        PGM_LOCK_VOID(pVM);
+
+# if PGM_SHW_TYPE == PGM_TYPE_32BIT
+        const X86PDE    PdeDst = pgmShwGet32BitPDE(pVCpu, GCPtrPage);
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE
+        const unsigned  iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+        PX86PDPAE       pPDDst;
+        X86PDEPAE       PdeDst;
+#   if PGM_GST_TYPE != PGM_TYPE_PAE
+        X86PDPE         PdpeSrc;
+
+        /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
+        PdpeSrc.u  = X86_PDPE_P;   /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
+#   endif
+        rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
+        if (rc != VINF_SUCCESS)
+        {
+            PGM_UNLOCK(pVM);
+            AssertRC(rc);
+            return rc;
+        }
+        Assert(pPDDst);
+        PdeDst = pPDDst->a[iPDDst];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+        const unsigned  iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+        PX86PDPAE       pPDDst;
+        X86PDEPAE       PdeDst;
+
+#  if PGM_GST_TYPE == PGM_TYPE_PROT
+        /* AMD-V nested paging */
+        X86PML4E        Pml4eSrc;
+        X86PDPE         PdpeSrc;
+        PX86PML4E       pPml4eSrc = &Pml4eSrc;
+
+        /* Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
+        Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
+        PdpeSrc.u  = X86_PDPE_P | X86_PDPE_RW | X86_PDPE_US | X86_PDPE_A;
+#  endif
+
+        rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
+        if (rc != VINF_SUCCESS)
+        {
+            PGM_UNLOCK(pVM);
+            AssertRC(rc);
+            return rc;
+        }
+        Assert(pPDDst);
+        PdeDst = pPDDst->a[iPDDst];
+# endif
+        if (!(PdeDst.u & X86_PDE_P))
+        {
+            /** @todo r=bird: This guy will set the A bit on the PDE,
+             *    probably harmless. */
+            rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
+        }
+        else
+        {
+            /* Note! We used to sync PGM_SYNC_NR_PAGES pages, which triggered assertions in CSAM, because
+             *       R/W attributes of nearby pages were reset. Not sure how that could happen. Anyway, it
+             *       makes no sense to prefetch more than one page.
+             */
+            rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
+            if (RT_SUCCESS(rc))
+                rc = VINF_SUCCESS;
+        }
+        PGM_UNLOCK(pVM);
+    }
+    return rc;
+
+#elif PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
+    NOREF(pVCpu); NOREF(GCPtrPage);
+    return VINF_SUCCESS; /* ignore */
+#else
+    AssertCompile(0);
+#endif
+}
+
+
+
+
+/**
+ * Syncs a page during a PGMVerifyAccess() call.
+ *
+ * @returns VBox status code (informational included).
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtrPage   The address of the page to sync.
+ * @param   fPage       The effective guest page flags.
+ * @param   uErr        The trap error code.
+ * @remarks This will normally never be called on invalid guest page
+ *          translation entries.
+ */
+PGM_BTH_DECL(int, VerifyAccessSyncPage)(PVMCPUCC pVCpu, RTGCPTR GCPtrPage, unsigned fPage, unsigned uErr)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
+
+    LogFlow(("VerifyAccessSyncPage: GCPtrPage=%RGv fPage=%#x uErr=%#x\n", GCPtrPage, fPage, uErr));
+    RT_NOREF_PV(GCPtrPage); RT_NOREF_PV(fPage); RT_NOREF_PV(uErr);
+
+    Assert(!pVM->pgm.s.fNestedPaging);
+#if   (   PGM_GST_TYPE == PGM_TYPE_32BIT \
+       || PGM_GST_TYPE == PGM_TYPE_REAL \
+       || PGM_GST_TYPE == PGM_TYPE_PROT \
+       || PGM_GST_TYPE == PGM_TYPE_PAE \
+       || PGM_GST_TYPE == PGM_TYPE_AMD64 ) \
+    && !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) \
+    && PGM_SHW_TYPE != PGM_TYPE_NONE
+
+    /*
+     * Get guest PD and index.
+     */
+    /** @todo Performance: We've done all this a jiffy ago in the
+     *        PGMGstGetPage call. */
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+#  if PGM_GST_TYPE == PGM_TYPE_32BIT
+    const unsigned  iPDSrc = (uint32_t)GCPtrPage >> GST_PD_SHIFT;
+    PGSTPD          pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
+
+#  elif PGM_GST_TYPE == PGM_TYPE_PAE
+    unsigned        iPDSrc = 0;
+    X86PDPE         PdpeSrc;
+    PGSTPD          pPDSrc = pgmGstGetPaePDPtr(pVCpu, GCPtrPage, &iPDSrc, &PdpeSrc);
+    if (RT_UNLIKELY(!pPDSrc))
+    {
+        Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
+        return VINF_EM_RAW_GUEST_TRAP;
+    }
+
+#  elif PGM_GST_TYPE == PGM_TYPE_AMD64
+    unsigned        iPDSrc = 0;         /* shut up gcc */
+    PX86PML4E       pPml4eSrc = NULL;   /* ditto */
+    X86PDPE         PdpeSrc;
+    PGSTPD          pPDSrc = pgmGstGetLongModePDPtr(pVCpu, GCPtrPage, &pPml4eSrc, &PdpeSrc, &iPDSrc);
+    if (RT_UNLIKELY(!pPDSrc))
+    {
+        Log(("PGMVerifyAccess: access violation for %RGv due to non-present PDPTR\n", GCPtrPage));
+        return VINF_EM_RAW_GUEST_TRAP;
+    }
+#  endif
+
+# else  /* !PGM_WITH_PAGING */
+    PGSTPD          pPDSrc = NULL;
+    const unsigned  iPDSrc = 0;
+# endif /* !PGM_WITH_PAGING */
+    int             rc = VINF_SUCCESS;
+
+    PGM_LOCK_VOID(pVM);
+
+    /*
+     * First check if the shadow pd is present.
+     */
+# if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    PX86PDE         pPdeDst = pgmShwGet32BitPDEPtr(pVCpu, GCPtrPage);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE
+    PX86PDEPAE      pPdeDst;
+    const unsigned  iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PX86PDPAE       pPDDst;
+#   if PGM_GST_TYPE != PGM_TYPE_PAE
+    /* Fake PDPT entry; access control handled on the page table level, so allow everything. */
+    X86PDPE         PdpeSrc;
+    PdpeSrc.u  = X86_PDPE_P;   /* rw/us are reserved for PAE pdpte's; accessed bit causes invalid VT-x guest state errors */
+#   endif
+    rc = pgmShwSyncPaePDPtr(pVCpu, GCPtrPage, PdpeSrc.u, &pPDDst);
+    if (rc != VINF_SUCCESS)
+    {
+        PGM_UNLOCK(pVM);
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPDDst);
+    pPdeDst = &pPDDst->a[iPDDst];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    const unsigned  iPDDst = ((GCPtrPage >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    PX86PDPAE       pPDDst;
+    PX86PDEPAE      pPdeDst;
+
+#  if PGM_GST_TYPE == PGM_TYPE_PROT
+    /* AMD-V nested paging: Fake PML4 & PDPT entry; access control handled on the page table level, so allow everything. */
+    X86PML4E        Pml4eSrc;
+    X86PDPE         PdpeSrc;
+    PX86PML4E       pPml4eSrc = &Pml4eSrc;
+    Pml4eSrc.u = X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_A;
+    PdpeSrc.u  = X86_PDPE_P  | X86_PDPE_RW  | X86_PDPE_US  | X86_PDPE_A;
+#  endif
+
+    rc = pgmShwSyncLongModePDPtr(pVCpu, GCPtrPage, pPml4eSrc->u, PdpeSrc.u, &pPDDst);
+    if (rc != VINF_SUCCESS)
+    {
+        PGM_UNLOCK(pVM);
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPDDst);
+    pPdeDst = &pPDDst->a[iPDDst];
+# endif
+
+    if (!(pPdeDst->u & X86_PDE_P))
+    {
+        rc = PGM_BTH_NAME(SyncPT)(pVCpu, iPDSrc, pPDSrc, GCPtrPage);
+        if (rc != VINF_SUCCESS)
+        {
+            PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+            PGM_UNLOCK(pVM);
+            AssertRC(rc);
+            return rc;
+        }
+    }
+
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+    /* Check for dirty bit fault */
+    rc = PGM_BTH_NAME(CheckDirtyPageFault)(pVCpu, uErr, pPdeDst, &pPDSrc->a[iPDSrc], GCPtrPage);
+    if (rc == VINF_PGM_HANDLED_DIRTY_BIT_FAULT)
+        Log(("PGMVerifyAccess: success (dirty)\n"));
+    else
+# endif
+    {
+# if PGM_WITH_PAGING(PGM_GST_TYPE, PGM_SHW_TYPE)
+        GSTPDE PdeSrc       = pPDSrc->a[iPDSrc];
+# else
+        GSTPDE const PdeSrc = { X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_A }; /* faked so we don't have to #ifdef everything */
+# endif
+
+        Assert(rc != VINF_EM_RAW_GUEST_TRAP);
+        if (uErr & X86_TRAP_PF_US)
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncUser));
+        else /* supervisor */
+            STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,PageOutOfSyncSupervisor));
+
+        rc = PGM_BTH_NAME(SyncPage)(pVCpu, PdeSrc, GCPtrPage, 1, 0);
+        if (RT_SUCCESS(rc))
+        {
+            /* Page was successfully synced */
+            Log2(("PGMVerifyAccess: success (sync)\n"));
+            rc = VINF_SUCCESS;
+        }
+        else
+        {
+            Log(("PGMVerifyAccess: access violation for %RGv rc=%Rrc\n", GCPtrPage, rc));
+            rc = VINF_EM_RAW_GUEST_TRAP;
+        }
+    }
+    PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdeDst);
+    PGM_UNLOCK(pVM);
+    return rc;
+
+#else  /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
+
+    AssertLogRelMsgFailed(("Shw=%d Gst=%d is not implemented!\n", PGM_GST_TYPE, PGM_SHW_TYPE));
+    return VERR_PGM_NOT_USED_IN_MODE;
+#endif /* PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) */
+}
+
+
+/**
+ * Syncs the paging hierarchy starting at CR3.
+ *
+ * @returns VBox status code, R0/RC may return VINF_PGM_SYNC_CR3, no other
+ *          informational status codes.
+ * @retval  VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
+ *          the VMM into guest context.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cr0         Guest context CR0 register.
+ * @param   cr3         Guest context CR3 register. Not subjected to the A20
+ *                      mask.
+ * @param   cr4         Guest context CR4 register.
+ * @param   fGlobal     Including global page directories or not
+ */
+PGM_BTH_DECL(int, SyncCR3)(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
+    NOREF(cr0); NOREF(cr3); NOREF(cr4); NOREF(fGlobal);
+
+    LogFlow(("SyncCR3 FF=%d fGlobal=%d\n", !!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), fGlobal));
+
+#if !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    PGM_LOCK_VOID(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    if (pPool->cDirtyPages)
+        pgmPoolResetDirtyPages(pVM);
+    PGM_UNLOCK(pVM);
+# endif
+#endif /* !NESTED && !EPT */
+
+#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
+    /*
+     * Nested / EPT / None - No work.
+     */
+    return VINF_SUCCESS;
+
+#elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    /*
+     * AMD64 (Shw & Gst) - No need to check all paging levels; we zero
+     * out the shadow parts when the guest modifies its tables.
+     */
+    return VINF_SUCCESS;
+
+#else /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
+
+    return VINF_SUCCESS;
+#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_AMD64 */
+}
+
+
+
+
+#ifdef VBOX_STRICT
+
+/**
+ * Checks that the shadow page table is in sync with the guest one.
+ *
+ * @returns The number of errors.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cr3         Guest context CR3 register.
+ * @param   cr4         Guest context CR4 register.
+ * @param   GCPtr       Where to start. Defaults to 0.
+ * @param   cb          How much to check. Defaults to everything.
+ */
+PGM_BTH_DECL(unsigned, AssertCR3)(PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4, RTGCPTR GCPtr, RTGCPTR cb)
+{
+    NOREF(pVCpu); NOREF(cr3); NOREF(cr4); NOREF(GCPtr); NOREF(cb);
+#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NONE
+    return 0;
+#else
+    unsigned cErrors = 0;
+    PVMCC    pVM     = pVCpu->CTX_SUFF(pVM);
+    PPGMPOOL pPool   = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
+
+# if PGM_GST_TYPE == PGM_TYPE_PAE
+    /** @todo currently broken; crashes below somewhere */
+    AssertFailed();
+# endif
+
+# if   PGM_GST_TYPE == PGM_TYPE_32BIT \
+    || PGM_GST_TYPE == PGM_TYPE_PAE \
+    || PGM_GST_TYPE == PGM_TYPE_AMD64
+
+    bool            fBigPagesSupported = GST_IS_PSE_ACTIVE(pVCpu);
+    PPGMCPU         pPGM = &pVCpu->pgm.s;
+    RTGCPHYS        GCPhysGst;              /* page address derived from the guest page tables. */
+    RTHCPHYS        HCPhysShw;              /* page address derived from the shadow page tables. */
+#  ifndef IN_RING0
+    RTHCPHYS        HCPhys;                 /* general usage. */
+#  endif
+    int             rc;
+
+    /*
+     * Check that the Guest CR3 and all its mappings are correct.
+     */
+    AssertMsgReturn(pPGM->GCPhysCR3 == PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK),
+                    ("Invalid GCPhysCR3=%RGp cr3=%RGp\n", pPGM->GCPhysCR3, (RTGCPHYS)cr3),
+                    false);
+#  if !defined(IN_RING0) && PGM_GST_TYPE != PGM_TYPE_AMD64
+#   if 0
+#    if PGM_GST_TYPE == PGM_TYPE_32BIT
+    rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGst32BitPdRC, NULL, &HCPhysShw);
+#    else
+    rc = PGMShwGetPage(pVCpu, (RTRCUINTPTR)pPGM->pGstPaePdptRC, NULL, &HCPhysShw);
+#    endif
+    AssertRCReturn(rc, 1);
+    HCPhys = NIL_RTHCPHYS;
+    rc = pgmRamGCPhys2HCPhys(pVM, PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK), &HCPhys);
+    AssertMsgReturn(HCPhys == HCPhysShw, ("HCPhys=%RHp HCPhyswShw=%RHp (cr3)\n", HCPhys, HCPhysShw), false);
+#   endif
+#   if PGM_GST_TYPE == PGM_TYPE_32BIT && defined(IN_RING3)
+    pgmGstGet32bitPDPtr(pVCpu);
+    RTGCPHYS GCPhys;
+    rc = PGMR3DbgR3Ptr2GCPhys(pVM->pUVM, pPGM->pGst32BitPdR3, &GCPhys);
+    AssertRCReturn(rc, 1);
+    AssertMsgReturn(PGM_A20_APPLY(pVCpu, cr3 & GST_CR3_PAGE_MASK) == GCPhys, ("GCPhys=%RGp cr3=%RGp\n", GCPhys, (RTGCPHYS)cr3), false);
+#   endif
+#  endif /* !IN_RING0 */
+
+    /*
+     * Get and check the Shadow CR3.
+     */
+#  if PGM_SHW_TYPE == PGM_TYPE_32BIT
+    unsigned        cPDEs       = X86_PG_ENTRIES;
+    unsigned        cIncrement  = X86_PG_ENTRIES * GUEST_PAGE_SIZE;
+#  elif PGM_SHW_TYPE == PGM_TYPE_PAE
+#   if PGM_GST_TYPE == PGM_TYPE_32BIT
+    unsigned        cPDEs       = X86_PG_PAE_ENTRIES * 4;   /* treat it as a 2048 entry table. */
+#   else
+    unsigned        cPDEs       = X86_PG_PAE_ENTRIES;
+#   endif
+    unsigned        cIncrement  = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
+#  elif PGM_SHW_TYPE == PGM_TYPE_AMD64
+    unsigned        cPDEs       = X86_PG_PAE_ENTRIES;
+    unsigned        cIncrement  = X86_PG_PAE_ENTRIES * GUEST_PAGE_SIZE;
+#  endif
+    if (cb != ~(RTGCPTR)0)
+        cPDEs = RT_MIN(cb >> SHW_PD_SHIFT, 1);
+
+/** @todo call the other two PGMAssert*() functions. */
+
+#  if PGM_GST_TYPE == PGM_TYPE_AMD64
+    unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
+
+    for (; iPml4 < X86_PG_PAE_ENTRIES; iPml4++)
+    {
+        PPGMPOOLPAGE    pShwPdpt = NULL;
+        PX86PML4E       pPml4eSrc;
+        PX86PML4E       pPml4eDst;
+        RTGCPHYS        GCPhysPdptSrc;
+
+        pPml4eSrc     = pgmGstGetLongModePML4EPtr(pVCpu, iPml4);
+        pPml4eDst     = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
+
+        /* Fetch the pgm pool shadow descriptor if the shadow pml4e is present. */
+        if (!(pPml4eDst->u & X86_PML4E_P))
+        {
+            GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
+            continue;
+        }
+
+        pShwPdpt = pgmPoolGetPage(pPool, pPml4eDst->u & X86_PML4E_PG_MASK);
+        GCPhysPdptSrc = PGM_A20_APPLY(pVCpu, pPml4eSrc->u & X86_PML4E_PG_MASK);
+
+        if ((pPml4eSrc->u & X86_PML4E_P) != (pPml4eDst->u & X86_PML4E_P))
+        {
+            AssertMsgFailed(("Present bit doesn't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
+            GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
+            cErrors++;
+            continue;
+        }
+
+        if (GCPhysPdptSrc != pShwPdpt->GCPhys)
+        {
+            AssertMsgFailed(("Physical address doesn't match! iPml4 %d pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, pPml4eDst->u, pPml4eSrc->u, pShwPdpt->GCPhys, GCPhysPdptSrc));
+            GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
+            cErrors++;
+            continue;
+        }
+
+        if (   (pPml4eDst->u & (X86_PML4E_US | X86_PML4E_RW | X86_PML4E_NX))
+            != (pPml4eSrc->u & (X86_PML4E_US | X86_PML4E_RW | X86_PML4E_NX)))
+        {
+            AssertMsgFailed(("User/Write/NoExec bits don't match! pPml4eDst.u=%#RX64 pPml4eSrc.u=%RX64\n", pPml4eDst->u, pPml4eSrc->u));
+            GCPtr += _2M * UINT64_C(512) * UINT64_C(512);
+            cErrors++;
+            continue;
+        }
+#  else  /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
+    {
+#  endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 */
+
+#  if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
+        /*
+         * Check the PDPTEs too.
+         */
+        unsigned iPdpt = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
+
+        for (;iPdpt <= SHW_PDPT_MASK; iPdpt++)
+        {
+            unsigned        iPDSrc  = 0;        /* initialized to shut up gcc */
+            PPGMPOOLPAGE    pShwPde = NULL;
+            PX86PDPE        pPdpeDst;
+            RTGCPHYS        GCPhysPdeSrc;
+            X86PDPE         PdpeSrc;
+            PdpeSrc.u = 0;                      /* initialized to shut up gcc 4.5 */
+#   if PGM_GST_TYPE == PGM_TYPE_PAE
+            PGSTPD          pPDSrc    = pgmGstGetPaePDPtr(pVCpu, GCPtr, &iPDSrc, &PdpeSrc);
+            PX86PDPT        pPdptDst  = pgmShwGetPaePDPTPtr(pVCpu);
+#   else
+            PX86PML4E       pPml4eSrcIgn;
+            PX86PDPT        pPdptDst;
+            PX86PDPAE       pPDDst;
+            PGSTPD          pPDSrc    = pgmGstGetLongModePDPtr(pVCpu, GCPtr, &pPml4eSrcIgn, &PdpeSrc, &iPDSrc);
+
+            rc = pgmShwGetLongModePDPtr(pVCpu, GCPtr, NULL, &pPdptDst, &pPDDst);
+            if (rc != VINF_SUCCESS)
+            {
+                AssertMsg(rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT, ("Unexpected rc=%Rrc\n", rc));
+                GCPtr += 512 * _2M;
+                continue;   /* next PDPTE */
+            }
+            Assert(pPDDst);
+#   endif
+            Assert(iPDSrc == 0);
+
+            pPdpeDst = &pPdptDst->a[iPdpt];
+
+            if (!(pPdpeDst->u & X86_PDPE_P))
+            {
+                GCPtr += 512 * _2M;
+                continue;   /* next PDPTE */
+            }
+
+            pShwPde      = pgmPoolGetPage(pPool, pPdpeDst->u & X86_PDPE_PG_MASK);
+            GCPhysPdeSrc = PGM_A20_APPLY(pVCpu, PdpeSrc.u & X86_PDPE_PG_MASK);
+
+            if ((pPdpeDst->u & X86_PDPE_P) != (PdpeSrc.u & X86_PDPE_P))
+            {
+                AssertMsgFailed(("Present bit doesn't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
+                GCPtr += 512 * _2M;
+                cErrors++;
+                continue;
+            }
+
+            if (GCPhysPdeSrc != pShwPde->GCPhys)
+            {
+#   if PGM_GST_TYPE == PGM_TYPE_AMD64
+                AssertMsgFailed(("Physical address doesn't match! iPml4 %d iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPml4, iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
+#   else
+                AssertMsgFailed(("Physical address doesn't match! iPdpt %d pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64 Phys %RX64 vs %RX64\n", iPdpt, pPdpeDst->u, PdpeSrc.u, pShwPde->GCPhys, GCPhysPdeSrc));
+#   endif
+                GCPtr += 512 * _2M;
+                cErrors++;
+                continue;
+            }
+
+#   if PGM_GST_TYPE == PGM_TYPE_AMD64
+            if (    (pPdpeDst->u & (X86_PDPE_US | X86_PDPE_RW | X86_PDPE_LM_NX))
+                !=  (PdpeSrc.u   & (X86_PDPE_US | X86_PDPE_RW | X86_PDPE_LM_NX)))
+            {
+                AssertMsgFailed(("User/Write/NoExec bits don't match! pPdpeDst.u=%#RX64 pPdpeSrc.u=%RX64\n", pPdpeDst->u, PdpeSrc.u));
+                GCPtr += 512 * _2M;
+                cErrors++;
+                continue;
+            }
+#   endif
+
+#  else  /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
+        {
+#  endif /* PGM_GST_TYPE != PGM_TYPE_AMD64 && PGM_GST_TYPE != PGM_TYPE_PAE */
+#  if PGM_GST_TYPE == PGM_TYPE_32BIT
+            GSTPD const    *pPDSrc = pgmGstGet32bitPDPtr(pVCpu);
+#   if PGM_SHW_TYPE == PGM_TYPE_32BIT
+            PCX86PD         pPDDst = pgmShwGet32BitPDPtr(pVCpu);
+#   endif
+#  endif /* PGM_GST_TYPE == PGM_TYPE_32BIT */
+            /*
+            * Iterate the shadow page directory.
+            */
+            GCPtr = (GCPtr >> SHW_PD_SHIFT) << SHW_PD_SHIFT;
+            unsigned iPDDst = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
+
+            for (;
+                iPDDst < cPDEs;
+                iPDDst++, GCPtr += cIncrement)
+            {
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE
+                const SHWPDE PdeDst = *pgmShwGetPaePDEPtr(pVCpu, GCPtr);
+#  else
+                const SHWPDE PdeDst = pPDDst->a[iPDDst];
+#  endif
+                if (   (PdeDst.u & X86_PDE_P)
+                    || ((PdeDst.u & (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) == (X86_PDE_P | PGM_PDFLAGS_TRACK_DIRTY)) )
+                {
+                    HCPhysShw = PdeDst.u & SHW_PDE_PG_MASK;
+                    PPGMPOOLPAGE pPoolPage = pgmPoolGetPage(pPool, HCPhysShw);
+                    if (!pPoolPage)
+                    {
+                        AssertMsgFailed(("Invalid page table address %RHp at %RGv! PdeDst=%#RX64\n",
+                                        HCPhysShw, GCPtr, (uint64_t)PdeDst.u));
+                        cErrors++;
+                        continue;
+                    }
+                    const SHWPT *pPTDst = (const SHWPT *)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pPoolPage);
+
+                    if (PdeDst.u & (X86_PDE4M_PWT | X86_PDE4M_PCD))
+                    {
+                        AssertMsgFailed(("PDE flags PWT and/or PCD is set at %RGv! These flags are not virtualized! PdeDst=%#RX64\n",
+                                        GCPtr, (uint64_t)PdeDst.u));
+                        cErrors++;
+                    }
+
+                    if (PdeDst.u & (X86_PDE4M_G | X86_PDE4M_D))
+                    {
+                        AssertMsgFailed(("4K PDE reserved flags at %RGv! PdeDst=%#RX64\n",
+                                        GCPtr, (uint64_t)PdeDst.u));
+                        cErrors++;
+                    }
+
+                    const GSTPDE PdeSrc = pPDSrc->a[(iPDDst >> (GST_PD_SHIFT - SHW_PD_SHIFT)) & GST_PD_MASK];
+                    if (!(PdeSrc.u & X86_PDE_P))
+                    {
+                        AssertMsgFailed(("Guest PDE at %RGv is not present! PdeDst=%#RX64 PdeSrc=%#RX64\n",
+                                        GCPtr, (uint64_t)PdeDst.u, (uint64_t)PdeSrc.u));
+                        cErrors++;
+                        continue;
+                    }
+
+                    if (   !(PdeSrc.u & X86_PDE_PS)
+                        || !fBigPagesSupported)
+                    {
+                        GCPhysGst = GST_GET_PDE_GCPHYS(PdeSrc);
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+                        GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | ((iPDDst & 1) * (GUEST_PAGE_SIZE / 2)));
+#  endif
+                    }
+                    else
+                    {
+#  if PGM_GST_TYPE == PGM_TYPE_32BIT
+                        if (PdeSrc.u & X86_PDE4M_PG_HIGH_MASK)
+                        {
+                            AssertMsgFailed(("Guest PDE at %RGv is using PSE36 or similar! PdeSrc=%#RX64\n",
+                                            GCPtr, (uint64_t)PdeSrc.u));
+                            cErrors++;
+                            continue;
+                        }
+#  endif
+                        GCPhysGst = GST_GET_BIG_PDE_GCPHYS(pVM, PdeSrc);
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+                        GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst | (GCPtr & RT_BIT(X86_PAGE_2M_SHIFT)));
+#  endif
+                    }
+
+                    if (   pPoolPage->enmKind
+                        != (!(PdeSrc.u & X86_PDE_PS) || !fBigPagesSupported ? BTH_PGMPOOLKIND_PT_FOR_PT : BTH_PGMPOOLKIND_PT_FOR_BIG))
+                    {
+                        AssertMsgFailed(("Invalid shadow page table kind %d at %RGv! PdeSrc=%#RX64\n",
+                                        pPoolPage->enmKind, GCPtr, (uint64_t)PdeSrc.u));
+                        cErrors++;
+                    }
+
+                    PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
+                    if (!pPhysPage)
+                    {
+                        AssertMsgFailed(("Cannot find guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
+                                        GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
+                        cErrors++;
+                        continue;
+                    }
+
+                    if (GCPhysGst != pPoolPage->GCPhys)
+                    {
+                        AssertMsgFailed(("GCPhysGst=%RGp != pPage->GCPhys=%RGp at %RGv\n",
+                                        GCPhysGst, pPoolPage->GCPhys, GCPtr));
+                        cErrors++;
+                        continue;
+                    }
+
+                    if (    !(PdeSrc.u & X86_PDE_PS)
+                        ||  !fBigPagesSupported)
+                    {
+                        /*
+                         * Page Table.
+                         */
+                        const GSTPT *pPTSrc;
+                        rc = PGM_GCPHYS_2_PTR_V2(pVM, pVCpu, PGM_A20_APPLY(pVCpu, GCPhysGst & ~(RTGCPHYS)(GUEST_PAGE_SIZE - 1)),
+                                                 &pPTSrc);
+                        if (RT_FAILURE(rc))
+                        {
+                            AssertMsgFailed(("Cannot map/convert guest physical address %RGp in the PDE at %RGv! PdeSrc=%#RX64\n",
+                                            GCPhysGst, GCPtr, (uint64_t)PdeSrc.u));
+                            cErrors++;
+                            continue;
+                        }
+                        if (    (PdeSrc.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/))
+                            !=  (PdeDst.u & (X86_PDE_P | X86_PDE_US | X86_PDE_RW/* | X86_PDE_A*/)))
+                        {
+                            /// @todo We get here a lot on out-of-sync CR3 entries. The access handler should zap them to avoid false alarms here!
+                            // (This problem will go away when/if we shadow multiple CR3s.)
+                            AssertMsgFailed(("4K PDE flags mismatch at %RGv! PdeSrc=%#RX64 PdeDst=%#RX64\n",
+                                            GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                            cErrors++;
+                            continue;
+                        }
+                        if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
+                        {
+                            AssertMsgFailed(("4K PDEs cannot have PGM_PDFLAGS_TRACK_DIRTY set! GCPtr=%RGv PdeDst=%#RX64\n",
+                                            GCPtr, (uint64_t)PdeDst.u));
+                            cErrors++;
+                            continue;
+                        }
+
+                        /* iterate the page table. */
+#  if PGM_SHW_TYPE == PGM_TYPE_PAE && PGM_GST_TYPE == PGM_TYPE_32BIT
+                        /* Select the right PDE as we're emulating a 4kb page table with 2 shadow page tables. */
+                        const unsigned offPTSrc  = ((GCPtr >> SHW_PD_SHIFT) & 1) * 512;
+#  else
+                        const unsigned offPTSrc  = 0;
+#  endif
+                        for (unsigned iPT = 0, off = 0;
+                            iPT < RT_ELEMENTS(pPTDst->a);
+                            iPT++, off += GUEST_PAGE_SIZE)
+                        {
+                            const SHWPTE PteDst = pPTDst->a[iPT];
+
+                            /* skip not-present and dirty tracked entries. */
+                            if (!(SHW_PTE_GET_U(PteDst) & (X86_PTE_P | PGM_PTFLAGS_TRACK_DIRTY))) /** @todo deal with ALL handlers and CSAM !P pages! */
+                                continue;
+                            Assert(SHW_PTE_IS_P(PteDst));
+
+                            const GSTPTE PteSrc = pPTSrc->a[iPT + offPTSrc];
+                            if (!(PteSrc.u & X86_PTE_P))
+                            {
+#  ifdef IN_RING3
+                                PGMAssertHandlerAndFlagsInSync(pVM);
+                                DBGFR3PagingDumpEx(pVM->pUVM, pVCpu->idCpu, DBGFPGDMP_FLAGS_CURRENT_CR3 | DBGFPGDMP_FLAGS_CURRENT_MODE
+                                                   | DBGFPGDMP_FLAGS_GUEST | DBGFPGDMP_FLAGS_HEADER | DBGFPGDMP_FLAGS_PRINT_CR3,
+                                                   0, 0, UINT64_MAX, 99, NULL);
+#  endif
+                                AssertMsgFailed(("Out of sync (!P) PTE at %RGv! PteSrc=%#RX64 PteDst=%#RX64 pPTSrc=%RGv iPTSrc=%x PdeSrc=%x physpte=%RGp\n",
+                                                GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst), pPTSrc, iPT + offPTSrc, PdeSrc.au32[0],
+                                                (uint64_t)GST_GET_PDE_GCPHYS(PdeSrc) + (iPT + offPTSrc) * sizeof(PteSrc)));
+                                cErrors++;
+                                continue;
+                            }
+
+                            uint64_t fIgnoreFlags = GST_PTE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_G | X86_PTE_D | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT;
+#  if 1 /** @todo sync accessed bit properly... */
+                            fIgnoreFlags |= X86_PTE_A;
+#  endif
+
+                            /* match the physical addresses */
+                            HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
+                            GCPhysGst = GST_GET_PTE_GCPHYS(PteSrc);
+
+#  ifdef IN_RING3
+                            rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
+                            if (RT_FAILURE(rc))
+                            {
+#   if 0
+                                if (HCPhysShw != MMR3PageDummyHCPhys(pVM)) /** @todo this is wrong. */
+                                {
+                                    AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                    continue;
+                                }
+#   endif
+                            }
+                            else if (HCPhysShw != (HCPhys & SHW_PTE_PG_MASK))
+                            {
+                                AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+#  endif
+
+                            pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
+                            if (!pPhysPage)
+                            {
+#  if 0
+                                if (HCPhysShw != MMR3PageDummyHCPhys(pVM))  /** @todo this is wrong. */
+                                {
+                                    AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPhysGst, GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                    continue;
+                                }
+#  endif
+                                if (SHW_PTE_IS_RW(PteDst))
+                                {
+                                    AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPtr + off, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                }
+                                fIgnoreFlags |= X86_PTE_RW;
+                            }
+                            else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
+                            {
+                                AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage:%R[pgmpage] GCPhysGst=%RGp PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+
+                            /* flags */
+                            if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage) && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage))
+                            {
+                                if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
+                                {
+                                    if (SHW_PTE_IS_RW(PteDst))
+                                    {
+                                        AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    fIgnoreFlags |= X86_PTE_RW;
+                                }
+                                else
+                                {
+                                    if (   SHW_PTE_IS_P(PteDst)
+#  if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
+                                        && !PGM_PAGE_IS_MMIO(pPhysPage)
+#  endif
+                                       )
+                                    {
+                                        AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, pPhysPage, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    fIgnoreFlags |= X86_PTE_P;
+                                }
+                            }
+                            else
+                            {
+                                if ((PteSrc.u & (X86_PTE_RW | X86_PTE_D)) == X86_PTE_RW)
+                                {
+                                    if (SHW_PTE_IS_RW(PteDst))
+                                    {
+                                        AssertMsgFailed(("!DIRTY page at %RGv is writable! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    if (!SHW_PTE_IS_TRACK_DIRTY(PteDst))
+                                    {
+                                        AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    if (SHW_PTE_IS_D(PteDst))
+                                    {
+                                        AssertMsgFailed(("!DIRTY page at %RGv is marked DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                    }
+#  if 0 /** @todo sync access bit properly... */
+                                    if (PteDst.n.u1Accessed != PteSrc.n.u1Accessed)
+                                    {
+                                        AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                        GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                    }
+                                    fIgnoreFlags |= X86_PTE_RW;
+#  else
+                                    fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
+#  endif
+                                }
+                                else if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
+                                {
+                                    /* access bit emulation (not implemented). */
+                                    if ((PteSrc.u & X86_PTE_A) || SHW_PTE_IS_P(PteDst))
+                                    {
+                                        AssertMsgFailed(("PGM_PTFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    if (!SHW_PTE_IS_A(PteDst))
+                                    {
+                                        AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                    }
+                                    fIgnoreFlags |= X86_PTE_P;
+                                }
+#  ifdef DEBUG_sandervl
+                                fIgnoreFlags |= X86_PTE_D | X86_PTE_A;
+#  endif
+                            }
+
+                            if (    (PteSrc.u & ~fIgnoreFlags)                != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
+                                &&  (PteSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
+                            )
+                            {
+                                AssertMsgFailed(("Flags mismatch at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, (uint64_t)PteSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
+                                                 fIgnoreFlags, (uint64_t)PteSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+                        } /* foreach PTE */
+                    }
+                    else
+                    {
+                        /*
+                        * Big Page.
+                        */
+                        uint64_t fIgnoreFlags = X86_PDE_AVL_MASK | GST_PDE_PG_MASK | X86_PDE4M_G | X86_PDE4M_D | X86_PDE4M_PS | X86_PDE4M_PWT | X86_PDE4M_PCD;
+                        if ((PdeSrc.u & (X86_PDE_RW | X86_PDE4M_D)) == X86_PDE_RW)
+                        {
+                            if (PdeDst.u & X86_PDE_RW)
+                            {
+                                AssertMsgFailed(("!DIRTY page at %RGv is writable! PdeSrc=%#RX64 PdeDst=%#RX64\n",
+                                                GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                                cErrors++;
+                                continue;
+                            }
+                            if (!(PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY))
+                            {
+                                AssertMsgFailed(("!DIRTY page at %RGv is not marked TRACK_DIRTY! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                                cErrors++;
+                                continue;
+                            }
+#  if 0 /** @todo sync access bit properly... */
+                            if (PdeDst.n.u1Accessed != PdeSrc.b.u1Accessed)
+                            {
+                                AssertMsgFailed(("!DIRTY page at %RGv is has mismatching accessed bit! PteSrc=%#RX64 PteDst=%#RX64\n",
+                                                GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                                cErrors++;
+                            }
+                            fIgnoreFlags |= X86_PTE_RW;
+#  else
+                            fIgnoreFlags |= X86_PTE_RW | X86_PTE_A;
+#  endif
+                        }
+                        else if (PdeDst.u & PGM_PDFLAGS_TRACK_DIRTY)
+                        {
+                            /* access bit emulation (not implemented). */
+                            if ((PdeSrc.u & X86_PDE_A) || SHW_PDE_IS_P(PdeDst))
+                            {
+                                AssertMsgFailed(("PGM_PDFLAGS_TRACK_DIRTY set at %RGv but no accessed bit emulation! PdeSrc=%#RX64 PdeDst=%#RX64\n",
+                                                GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                                cErrors++;
+                                continue;
+                            }
+                            if (!SHW_PDE_IS_A(PdeDst))
+                            {
+                                AssertMsgFailed(("!ACCESSED page at %RGv is has the accessed bit set! PdeSrc=%#RX64 PdeDst=%#RX64\n",
+                                                GCPtr, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                                cErrors++;
+                            }
+                            fIgnoreFlags |= X86_PTE_P;
+                        }
+
+                        if ((PdeSrc.u & ~fIgnoreFlags) != (PdeDst.u & ~fIgnoreFlags))
+                        {
+                            AssertMsgFailed(("Flags mismatch (B) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PdeDst=%#RX64\n",
+                                            GCPtr, (uint64_t)PdeSrc.u & ~fIgnoreFlags, (uint64_t)PdeDst.u & ~fIgnoreFlags,
+                                            fIgnoreFlags, (uint64_t)PdeSrc.u, (uint64_t)PdeDst.u));
+                            cErrors++;
+                        }
+
+                        /* iterate the page table. */
+                        for (unsigned iPT = 0, off = 0;
+                            iPT < RT_ELEMENTS(pPTDst->a);
+                            iPT++, off += GUEST_PAGE_SIZE, GCPhysGst = PGM_A20_APPLY(pVCpu, GCPhysGst + GUEST_PAGE_SIZE))
+                        {
+                            const SHWPTE PteDst = pPTDst->a[iPT];
+
+                            if (SHW_PTE_IS_TRACK_DIRTY(PteDst))
+                            {
+                                AssertMsgFailed(("The PTE at %RGv emulating a 2/4M page is marked TRACK_DIRTY! PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                            }
+
+                            /* skip not-present entries. */
+                            if (!SHW_PTE_IS_P(PteDst)) /** @todo deal with ALL handlers and CSAM !P pages! */
+                                continue;
+
+                            fIgnoreFlags = X86_PTE_PAE_PG_MASK | X86_PTE_AVL_MASK | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_PAT | X86_PTE_D | X86_PTE_A | X86_PTE_G | X86_PTE_PAE_NX;
+
+                            /* match the physical addresses */
+                            HCPhysShw = SHW_PTE_GET_HCPHYS(PteDst);
+
+#  ifdef IN_RING3
+                            rc = PGMPhysGCPhys2HCPhys(pVM, GCPhysGst, &HCPhys);
+                            if (RT_FAILURE(rc))
+                            {
+#   if 0
+                                if (HCPhysShw != MMR3PageDummyHCPhys(pVM))  /** @todo this is wrong. */
+                                {
+                                    AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                }
+#   endif
+                            }
+                            else if (HCPhysShw != (HCPhys & X86_PTE_PAE_PG_MASK))
+                            {
+                                AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp HCPhys=%RHp GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, HCPhysShw, HCPhys, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+#  endif
+                            pPhysPage = pgmPhysGetPage(pVM, GCPhysGst);
+                            if (!pPhysPage)
+                            {
+#  if 0 /** @todo make MMR3PageDummyHCPhys an 'All' function! */
+                                if (HCPhysShw != MMR3PageDummyHCPhys(pVM))  /** @todo this is wrong. */
+                                {
+                                    AssertMsgFailed(("Cannot find guest physical address %RGp at %RGv! PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPhysGst, GCPtr + off, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                    continue;
+                                }
+#  endif
+                                if (SHW_PTE_IS_RW(PteDst))
+                                {
+                                    AssertMsgFailed(("Invalid guest page at %RGv is writable! GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                     GCPtr + off, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                    cErrors++;
+                                }
+                                fIgnoreFlags |= X86_PTE_RW;
+                            }
+                            else if (HCPhysShw != PGM_PAGE_GET_HCPHYS(pPhysPage))
+                            {
+                                AssertMsgFailed(("Out of sync (phys) at %RGv! HCPhysShw=%RHp pPhysPage=%R[pgmpage] GCPhysGst=%RGp PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, HCPhysShw, pPhysPage, GCPhysGst, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+
+                            /* flags */
+                            if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPhysPage))
+                            {
+                                if (!PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPhysPage))
+                                {
+                                    if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
+                                    {
+                                        if (   SHW_PTE_IS_RW(PteDst)
+                                            && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage))
+                                        {
+                                            AssertMsgFailed(("WRITE access flagged at %RGv but the page is writable! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                             GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                            cErrors++;
+                                            continue;
+                                        }
+                                        fIgnoreFlags |= X86_PTE_RW;
+                                    }
+                                }
+                                else
+                                {
+                                    if (   SHW_PTE_IS_P(PteDst)
+                                        && !PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPhysPage)
+#  if PGM_SHW_TYPE == PGM_TYPE_EPT || PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_AMD64
+                                        && !PGM_PAGE_IS_MMIO(pPhysPage)
+#  endif
+                                        )
+                                    {
+                                        AssertMsgFailed(("ALL access flagged at %RGv but the page is present! pPhysPage=%R[pgmpage] PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                         GCPtr + off, pPhysPage, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                        cErrors++;
+                                        continue;
+                                    }
+                                    fIgnoreFlags |= X86_PTE_P;
+                                }
+                            }
+
+                            if (    (PdeSrc.u & ~fIgnoreFlags)                != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags)
+                                &&  (PdeSrc.u & ~(fIgnoreFlags | X86_PTE_RW)) != (SHW_PTE_GET_U(PteDst) & ~fIgnoreFlags) /* lazy phys handler dereg. */
+                            )
+                            {
+                                AssertMsgFailed(("Flags mismatch (BT) at %RGv! %#RX64 != %#RX64 fIgnoreFlags=%#RX64 PdeSrc=%#RX64 PteDst=%#RX64\n",
+                                                 GCPtr + off, (uint64_t)PdeSrc.u & ~fIgnoreFlags, SHW_PTE_LOG64(PteDst) & ~fIgnoreFlags,
+                                                 fIgnoreFlags, (uint64_t)PdeSrc.u, SHW_PTE_LOG64(PteDst)));
+                                cErrors++;
+                                continue;
+                            }
+                        } /* for each PTE */
+                    }
+                }
+                /* not present */
+
+            } /* for each PDE */
+
+        } /* for each PDPTE */
+
+    } /* for each PML4E */
+
+#  ifdef DEBUG
+    if (cErrors)
+        LogFlow(("AssertCR3: cErrors=%d\n", cErrors));
+#  endif
+# endif /* GST is in {32BIT, PAE, AMD64} */
+    return cErrors;
+#endif /* !PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE) && PGM_SHW_TYPE != PGM_TYPE_NONE */
+}
+#endif /* VBOX_STRICT */
+
+
+/**
+ * Sets up the CR3 for shadow paging
+ *
+ * @returns Strict VBox status code.
+ * @retval  VINF_SUCCESS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   GCPhysCR3       The physical address in the CR3 register. (A20 mask
+ *                          already applied.)
+ */
+PGM_BTH_DECL(int, MapCR3)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
+    int rc = VINF_SUCCESS;
+
+    /* Update guest paging info. */
+#if PGM_GST_TYPE == PGM_TYPE_32BIT \
+ || PGM_GST_TYPE == PGM_TYPE_PAE \
+ || PGM_GST_TYPE == PGM_TYPE_AMD64
+
+    LogFlow(("MapCR3: %RGp\n", GCPhysCR3));
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
+
+# if PGM_GST_TYPE == PGM_TYPE_PAE
+    if (   !pVCpu->pgm.s.CTX_SUFF(fPaePdpesAndCr3Mapped)
+        ||  pVCpu->pgm.s.GCPhysPaeCR3 != GCPhysCR3)
+# endif
+    {
+        /*
+         * Map the page CR3 points at.
+         */
+        RTHCPTR HCPtrGuestCR3;
+        rc = pgmGstMapCr3(pVCpu, GCPhysCR3, &HCPtrGuestCR3);
+        if (RT_SUCCESS(rc))
+        {
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+#  ifdef IN_RING3
+            pVCpu->pgm.s.pGst32BitPdR3 = (PX86PD)HCPtrGuestCR3;
+            pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
+#  else
+            pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR3PTR;
+            pVCpu->pgm.s.pGst32BitPdR0 = (PX86PD)HCPtrGuestCR3;
+#  endif
+
+# elif PGM_GST_TYPE == PGM_TYPE_PAE
+#  ifdef IN_RING3
+            pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCR3;
+            pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
+#  else
+            pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
+            pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCR3;
+#  endif
+
+            X86PDPE aGstPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            /*
+             * When EPT is enabled by the nested-hypervisor and the nested-guest is in PAE mode,
+             * the guest-CPU context would've already been updated with the 4 PAE PDPEs specified
+             * in the virtual VMCS. The PDPEs can differ from those in guest memory referenced by
+             * the translated nested-guest CR3. We -MUST- use the PDPEs provided in the virtual VMCS
+             * rather than those in guest memory.
+             *
+             * See Intel spec. 26.3.2.4 "Loading Page-Directory-Pointer-Table Entries".
+             */
+            if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
+                CPUMGetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
+            else
+#endif
+            {
+                /* Update CPUM with the PAE PDPEs referenced by CR3. */
+                memcpy(&aGstPaePdpes, HCPtrGuestCR3, sizeof(aGstPaePdpes));
+                CPUMSetGuestPaePdpes(pVCpu, &aGstPaePdpes[0]);
+            }
+
+            /*
+             * Map the 4 PAE PDPEs.
+             */
+            rc = PGMGstMapPaePdpes(pVCpu, &aGstPaePdpes[0]);
+            if (RT_SUCCESS(rc))
+            {
+#  ifdef IN_RING3
+                pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = true;
+                pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
+#  else
+                pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
+                pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = true;
+#  endif
+                pVCpu->pgm.s.GCPhysPaeCR3 = GCPhysCR3;
+            }
+
+# elif PGM_GST_TYPE == PGM_TYPE_AMD64
+#  ifdef IN_RING3
+            pVCpu->pgm.s.pGstAmd64Pml4R3 = (PX86PML4)HCPtrGuestCR3;
+            pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
+#  else
+            pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
+            pVCpu->pgm.s.pGstAmd64Pml4R0 = (PX86PML4)HCPtrGuestCR3;
+#  endif
+# endif
+        }
+        else
+            AssertMsgFailed(("rc=%Rrc GCPhysGuestPD=%RGp\n", rc, GCPhysCR3));
+    }
+#endif
+
+    /*
+     * Update shadow paging info for guest modes with paging (32-bit, PAE, AMD64).
+     */
+# if  (   (   PGM_SHW_TYPE == PGM_TYPE_32BIT \
+           || PGM_SHW_TYPE == PGM_TYPE_PAE    \
+           || PGM_SHW_TYPE == PGM_TYPE_AMD64) \
+       && (   PGM_GST_TYPE != PGM_TYPE_REAL   \
+           && PGM_GST_TYPE != PGM_TYPE_PROT))
+
+    Assert(!pVM->pgm.s.fNestedPaging);
+    PGM_A20_ASSERT_MASKED(pVCpu, GCPhysCR3);
+
+    /*
+     * Update the shadow root page as well since that's not fixed.
+     */
+    PPGMPOOL     pPool             = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPAGE pOldShwPageCR3    = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
+    PPGMPOOLPAGE pNewShwPageCR3;
+
+    PGM_LOCK_VOID(pVM);
+
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    if (pPool->cDirtyPages)
+        pgmPoolResetDirtyPages(pVM);
+# endif
+
+    Assert(!(GCPhysCR3 >> (GUEST_PAGE_SHIFT + 32))); /** @todo what is this for? */
+    int const rc2 = pgmPoolAlloc(pVM, GCPhysCR3 & GST_CR3_PAGE_MASK, BTH_PGMPOOLKIND_ROOT, PGMPOOLACCESS_DONTCARE,
+                                 PGM_A20_IS_ENABLED(pVCpu), NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/, &pNewShwPageCR3);
+    AssertFatalRC(rc2);
+
+    pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pPool, pNewShwPageCR3);
+    pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pPool, pNewShwPageCR3);
+
+    /* Set the current hypervisor CR3. */
+    CPUMSetHyperCR3(pVCpu, PGMGetHyperCR3(pVCpu));
+
+    /* Clean up the old CR3 root. */
+    if (    pOldShwPageCR3
+        &&  pOldShwPageCR3 != pNewShwPageCR3    /* @todo can happen due to incorrect syncing between REM & PGM; find the real cause */)
+    {
+        Assert(pOldShwPageCR3->enmKind != PGMPOOLKIND_FREE);
+
+        /* Mark the page as unlocked; allow flushing again. */
+        pgmPoolUnlockPage(pPool, pOldShwPageCR3);
+
+        pgmPoolFreeByPage(pPool, pOldShwPageCR3, NIL_PGMPOOL_IDX, UINT32_MAX);
+    }
+    PGM_UNLOCK(pVM);
+# else
+    NOREF(GCPhysCR3);
+# endif
+
+    return rc;
+}
+
+/**
+ * Unmaps the shadow CR3.
+ *
+ * @returns VBox status, no specials.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+PGM_BTH_DECL(int, UnmapCR3)(PVMCPUCC pVCpu)
+{
+    LogFlow(("UnmapCR3\n"));
+
+    int   rc  = VINF_SUCCESS;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM); NOREF(pVM);
+
+    /*
+     * Update guest paging info.
+     */
+#if PGM_GST_TYPE == PGM_TYPE_32BIT
+    pVCpu->pgm.s.pGst32BitPdR3 = 0;
+    pVCpu->pgm.s.pGst32BitPdR0 = 0;
+
+#elif PGM_GST_TYPE == PGM_TYPE_PAE
+    pVCpu->pgm.s.pGstPaePdptR3 = 0;
+    pVCpu->pgm.s.pGstPaePdptR0 = 0;
+    for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
+    {
+        pVCpu->pgm.s.apGstPaePDsR3[i]    = 0;
+        pVCpu->pgm.s.apGstPaePDsR0[i]    = 0;
+        pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
+    }
+
+#elif PGM_GST_TYPE == PGM_TYPE_AMD64
+    pVCpu->pgm.s.pGstAmd64Pml4R3 = 0;
+    pVCpu->pgm.s.pGstAmd64Pml4R0 = 0;
+
+#else /* prot/real mode stub */
+    /* nothing to do */
+#endif
+
+    /*
+     * PAE PDPEs (and CR3) might have been mapped via PGMGstMapPaePdpesAtCr3()
+     * prior to switching to PAE in pfnMapCr3(), so we need to clear them here.
+     */
+    pVCpu->pgm.s.fPaePdpesAndCr3MappedR3 = false;
+    pVCpu->pgm.s.fPaePdpesAndCr3MappedR0 = false;
+    pVCpu->pgm.s.GCPhysPaeCR3            = NIL_RTGCPHYS;
+
+    /*
+     * Update shadow paging info.
+     */
+#if  (   (   PGM_SHW_TYPE == PGM_TYPE_32BIT \
+          || PGM_SHW_TYPE == PGM_TYPE_PAE \
+          || PGM_SHW_TYPE == PGM_TYPE_AMD64))
+# if PGM_GST_TYPE != PGM_TYPE_REAL
+    Assert(!pVM->pgm.s.fNestedPaging);
+# endif
+    PGM_LOCK_VOID(pVM);
+
+    if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
+    {
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+        if (pPool->cDirtyPages)
+            pgmPoolResetDirtyPages(pVM);
+# endif
+
+        /* Mark the page as unlocked; allow flushing again. */
+        pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+
+        pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
+        pVCpu->pgm.s.pShwPageCR3R3 = 0;
+        pVCpu->pgm.s.pShwPageCR3R0 = 0;
+    }
+
+    PGM_UNLOCK(pVM);
+#endif
+
+    return rc;
+}
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllGst.h b/src/VBox/VMM/VMMAll/PGMAllGst.h
new file mode 100644
index 00000000..9bf00255
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllGst.h
@@ -0,0 +1,545 @@
+/* $Id: PGMAllGst.h $ */
+/** @file
+ * VBox - Page Manager, Guest Paging Template - All context code.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+/** @todo Do we really need any of these forward declarations? */
+#if PGM_GST_TYPE == PGM_TYPE_32BIT \
+ || PGM_GST_TYPE == PGM_TYPE_PAE \
+ || PGM_GST_TYPE == PGM_TYPE_AMD64
+DECLINLINE(int) PGM_GST_NAME(Walk)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PGSTPTWALK pGstWalk);
+#endif
+PGM_GST_DECL(int,  Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3);
+PGM_GST_DECL(int,  GetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk);
+PGM_GST_DECL(int,  ModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask);
+PGM_GST_DECL(int,  Exit)(PVMCPUCC pVCpu);
+
+#ifdef IN_RING3 /* r3 only for now.  */
+PGM_GST_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
+#endif
+RT_C_DECLS_END
+
+
+/**
+ * Enters the guest mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPhysCR3   The physical address from the CR3 register.
+ */
+PGM_GST_DECL(int, Enter)(PVMCPUCC pVCpu, RTGCPHYS GCPhysCR3)
+{
+    /*
+     * Map and monitor CR3
+     */
+    uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
+    return g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3);
+}
+
+
+/**
+ * Exits the guest mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+PGM_GST_DECL(int, Exit)(PVMCPUCC pVCpu)
+{
+    uintptr_t idxBth = pVCpu->pgm.s.idxBothModeData;
+    AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
+    AssertReturn(g_aPgmBothModeData[idxBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
+    return g_aPgmBothModeData[idxBth].pfnUnmapCR3(pVCpu);
+}
+
+
+#if PGM_GST_TYPE == PGM_TYPE_32BIT \
+ || PGM_GST_TYPE == PGM_TYPE_PAE \
+ || PGM_GST_TYPE == PGM_TYPE_AMD64
+
+
+DECLINLINE(int) PGM_GST_NAME(WalkReturnNotPresent)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, int iLevel)
+{
+    NOREF(iLevel); NOREF(pVCpu);
+    pWalk->fNotPresent     = true;
+    pWalk->uLevel          = (uint8_t)iLevel;
+    return VERR_PAGE_TABLE_NOT_PRESENT;
+}
+
+DECLINLINE(int) PGM_GST_NAME(WalkReturnBadPhysAddr)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, int iLevel, int rc)
+{
+    AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc); NOREF(pVCpu);
+    pWalk->fBadPhysAddr    = true;
+    pWalk->uLevel          = (uint8_t)iLevel;
+    return VERR_PAGE_TABLE_NOT_PRESENT;
+}
+
+DECLINLINE(int) PGM_GST_NAME(WalkReturnRsvdError)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, int iLevel)
+{
+    NOREF(pVCpu);
+    pWalk->fRsvdError      = true;
+    pWalk->uLevel          = (uint8_t)iLevel;
+    return VERR_PAGE_TABLE_NOT_PRESENT;
+}
+
+
+/**
+ * Performs a guest page table walk.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT on failure.  Check pWalk for details.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtr       The guest virtual address to walk by.
+ * @param   pWalk       The page walk info.
+ * @param   pGstWalk    The guest mode specific page walk info.
+ */
+DECLINLINE(int) PGM_GST_NAME(Walk)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk, PGSTPTWALK pGstWalk)
+{
+    int rc;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/** @def PGM_GST_SLAT_WALK
+ * Macro to perform guest second-level address translation (EPT or Nested).
+ *
+ * @param   a_pVCpu         The cross context virtual CPU structure of the calling
+ *                          EMT.
+ * @param   a_GCPtrNested   The nested-guest linear address that caused the
+ *                          second-level translation.
+ * @param   a_GCPhysNested  The nested-guest physical address to translate.
+ * @param   a_GCPhysOut     Where to store the guest-physical address (result).
+ */
+# define PGM_GST_SLAT_WALK(a_pVCpu, a_GCPtrNested, a_GCPhysNested, a_GCPhysOut, a_pWalk) \
+    do { \
+        if ((a_pVCpu)->pgm.s.enmGuestSlatMode == PGMSLAT_EPT) \
+        { \
+            PGMPTWALK    WalkSlat; \
+            PGMPTWALKGST WalkGstSlat; \
+            int const rcX = pgmGstSlatWalk(a_pVCpu, a_GCPhysNested, true /* fIsLinearAddrValid */, a_GCPtrNested, &WalkSlat, \
+                                           &WalkGstSlat); \
+            if (RT_SUCCESS(rcX)) \
+                (a_GCPhysOut) = WalkSlat.GCPhys; \
+            else \
+            { \
+                *(a_pWalk) = WalkSlat; \
+                return rcX; \
+            } \
+        } \
+    } while (0)
+#endif
+
+    /*
+     * Init the walking structures.
+     */
+    RT_ZERO(*pWalk);
+    RT_ZERO(*pGstWalk);
+    pWalk->GCPtr = GCPtr;
+
+# if PGM_GST_TYPE == PGM_TYPE_32BIT \
+  || PGM_GST_TYPE == PGM_TYPE_PAE
+    /*
+     * Boundary check for PAE and 32-bit (prevents trouble further down).
+     */
+    if (RT_UNLIKELY(GCPtr >= _4G))
+        return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 8);
+# endif
+
+    uint64_t fEffective;
+    {
+# if PGM_GST_TYPE == PGM_TYPE_AMD64
+        /*
+         * The PML4 table.
+         */
+        rc = pgmGstGetLongModePML4PtrEx(pVCpu, &pGstWalk->pPml4);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 4, rc);
+
+        PX86PML4E pPml4e;
+        pGstWalk->pPml4e  = pPml4e  = &pGstWalk->pPml4->a[(GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK];
+        X86PML4E  Pml4e;
+        pGstWalk->Pml4e.u = Pml4e.u = pPml4e->u;
+
+        if (GST_IS_PGENTRY_PRESENT(pVCpu, Pml4e)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 4);
+
+        if (RT_LIKELY(GST_IS_PML4E_VALID(pVCpu, Pml4e))) { /* likely */ }
+        else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 4);
+
+        fEffective = Pml4e.u & (  X86_PML4E_P | X86_PML4E_RW | X86_PML4E_US | X86_PML4E_PWT | X86_PML4E_PCD | X86_PML4E_A
+                                | X86_PML4E_NX);
+        pWalk->fEffective = fEffective;
+
+        /*
+         * The PDPT.
+         */
+        RTGCPHYS GCPhysPdpt = Pml4e.u & X86_PML4E_PG_MASK;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPdpt, GCPhysPdpt, pWalk);
+#endif
+        rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPdpt, &pGstWalk->pPdpt);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc);
+
+# elif PGM_GST_TYPE == PGM_TYPE_PAE
+        rc = pgmGstGetPaePDPTPtrEx(pVCpu, &pGstWalk->pPdpt);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
+#endif
+    }
+    {
+# if PGM_GST_TYPE == PGM_TYPE_AMD64 || PGM_GST_TYPE == PGM_TYPE_PAE
+        PX86PDPE pPdpe;
+        pGstWalk->pPdpe  = pPdpe  = &pGstWalk->pPdpt->a[(GCPtr >> GST_PDPT_SHIFT) & GST_PDPT_MASK];
+        X86PDPE  Pdpe;
+        pGstWalk->Pdpe.u = Pdpe.u = pPdpe->u;
+
+        if (GST_IS_PGENTRY_PRESENT(pVCpu, Pdpe)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 3);
+
+        if (RT_LIKELY(GST_IS_PDPE_VALID(pVCpu, Pdpe))) { /* likely */ }
+        else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 3);
+
+# if PGM_GST_TYPE == PGM_TYPE_AMD64
+        fEffective &= (Pdpe.u & (  X86_PDPE_P   | X86_PDPE_RW  | X86_PDPE_US
+                                 | X86_PDPE_PWT | X86_PDPE_PCD | X86_PDPE_A));
+        fEffective |= Pdpe.u & X86_PDPE_LM_NX;
+# else
+        /*
+         * NX in the legacy-mode PAE PDPE is reserved. The valid check above ensures the NX bit is not set.
+         * The RW, US, A bits MBZ in PAE PDPTE entries but must be 1 the way we compute cumulative (effective) access rights.
+         */
+        Assert(!(Pdpe.u & X86_PDPE_LM_NX));
+        fEffective = X86_PDPE_P | X86_PDPE_RW  | X86_PDPE_US | X86_PDPE_A
+                   | (Pdpe.u & (X86_PDPE_PWT | X86_PDPE_PCD));
+# endif
+        pWalk->fEffective = fEffective;
+
+        /*
+         * The PD.
+         */
+        RTGCPHYS GCPhysPd = Pdpe.u & X86_PDPE_PG_MASK;
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPd, GCPhysPd, pWalk);
+# endif
+        rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPd, &pGstWalk->pPd);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 2, rc);
+
+# elif PGM_GST_TYPE == PGM_TYPE_32BIT
+        rc = pgmGstGet32bitPDPtrEx(pVCpu, &pGstWalk->pPd);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 8, rc);
+# endif
+    }
+    {
+        PGSTPDE pPde;
+        pGstWalk->pPde  = pPde  = &pGstWalk->pPd->a[(GCPtr >> GST_PD_SHIFT) & GST_PD_MASK];
+        GSTPDE  Pde;
+        pGstWalk->Pde.u = Pde.u = pPde->u;
+        if (GST_IS_PGENTRY_PRESENT(pVCpu, Pde)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 2);
+        if ((Pde.u & X86_PDE_PS) && GST_IS_PSE_ACTIVE(pVCpu))
+        {
+            if (RT_LIKELY(GST_IS_BIG_PDE_VALID(pVCpu, Pde))) { /* likely */ }
+            else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
+
+            /*
+             * We're done.
+             */
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+            fEffective  = Pde.u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PWT | X86_PDE4M_PCD | X86_PDE4M_A);
+# else
+            fEffective &= Pde.u & (X86_PDE4M_P | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PWT | X86_PDE4M_PCD | X86_PDE4M_A);
+            fEffective |= Pde.u & X86_PDE2M_PAE_NX;
+# endif
+            fEffective |= Pde.u & (X86_PDE4M_D | X86_PDE4M_G);
+            fEffective |= (Pde.u & X86_PDE4M_PAT) >> X86_PDE4M_PAT_SHIFT;
+            pWalk->fEffective = fEffective;
+            Assert(GST_IS_NX_ACTIVE(pVCpu) || !(fEffective & PGM_PTATTRS_NX_MASK));
+            Assert(fEffective & PGM_PTATTRS_R_MASK);
+
+            pWalk->fBigPage   = true;
+            pWalk->fSucceeded = true;
+            RTGCPHYS GCPhysPde = GST_GET_BIG_PDE_GCPHYS(pVCpu->CTX_SUFF(pVM), Pde)
+                               | (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPde, GCPhysPde, pWalk);
+# endif
+            pWalk->GCPhys     = GCPhysPde;
+            PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->GCPhys);
+            return VINF_SUCCESS;
+        }
+
+        if (RT_UNLIKELY(!GST_IS_PDE_VALID(pVCpu, Pde)))
+            return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 2);
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+        fEffective  = Pde.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD | X86_PDE_A);
+# else
+        fEffective &= Pde.u & (X86_PDE_P | X86_PDE_RW | X86_PDE_US | X86_PDE_PWT | X86_PDE_PCD | X86_PDE_A);
+        fEffective |= Pde.u & X86_PDE_PAE_NX;
+# endif
+        pWalk->fEffective = fEffective;
+
+        /*
+         * The PT.
+         */
+        RTGCPHYS GCPhysPt = GST_GET_PDE_GCPHYS(Pde);
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPt, GCPhysPt, pWalk);
+# endif
+        rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, GCPhysPt, &pGstWalk->pPt);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnBadPhysAddr)(pVCpu, pWalk, 1, rc);
+    }
+    {
+        PGSTPTE pPte;
+        pGstWalk->pPte  = pPte  = &pGstWalk->pPt->a[(GCPtr >> GST_PT_SHIFT) & GST_PT_MASK];
+        GSTPTE  Pte;
+        pGstWalk->Pte.u = Pte.u = pPte->u;
+
+        if (GST_IS_PGENTRY_PRESENT(pVCpu, Pte)) { /* probable */ }
+        else return PGM_GST_NAME(WalkReturnNotPresent)(pVCpu, pWalk, 1);
+
+        if (RT_LIKELY(GST_IS_PTE_VALID(pVCpu, Pte))) { /* likely */ }
+        else return PGM_GST_NAME(WalkReturnRsvdError)(pVCpu, pWalk, 1);
+
+        /*
+         * We're done.
+         */
+        fEffective &= Pte.u & (X86_PTE_P | X86_PTE_RW | X86_PTE_US | X86_PTE_PWT | X86_PTE_PCD | X86_PTE_A);
+        fEffective |= Pte.u & (X86_PTE_D | X86_PTE_PAT | X86_PTE_G);
+# if PGM_GST_TYPE != PGM_TYPE_32BIT
+        fEffective |= Pte.u & X86_PTE_PAE_NX;
+# endif
+        pWalk->fEffective = fEffective;
+        Assert(GST_IS_NX_ACTIVE(pVCpu) || !(fEffective & PGM_PTATTRS_NX_MASK));
+        Assert(fEffective & PGM_PTATTRS_R_MASK);
+
+        pWalk->fSucceeded = true;
+        RTGCPHYS GCPhysPte = GST_GET_PTE_GCPHYS(Pte)
+                           | (GCPtr & GUEST_PAGE_OFFSET_MASK);
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        PGM_GST_SLAT_WALK(pVCpu, GCPtr, GCPhysPte, GCPhysPte, pWalk);
+# endif
+        pWalk->GCPhys     = GCPhysPte;
+        return VINF_SUCCESS;
+    }
+}
+
+#endif /* 32BIT, PAE, AMD64 */
+
+/**
+ * Gets effective Guest OS page information.
+ *
+ * When GCPtr is in a big page, the function will return as if it was a normal
+ * 4KB page. If the need for distinguishing between big and normal page becomes
+ * necessary at a later point, a PGMGstGetPage Ex() will be created for that
+ * purpose.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Guest Context virtual address of the page.
+ * @param   pWalk       Where to store the page walk info.
+ */
+PGM_GST_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALK pWalk)
+{
+#if PGM_GST_TYPE == PGM_TYPE_REAL \
+ || PGM_GST_TYPE == PGM_TYPE_PROT
+
+    RT_ZERO(*pWalk);
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
+    {
+        PGMPTWALK    WalkSlat;
+        PGMPTWALKGST WalkGstSlat;
+        int const rc = pgmGstSlatWalk(pVCpu, GCPtr, true /* fIsLinearAddrValid */, GCPtr, &WalkSlat, &WalkGstSlat);
+        if (RT_SUCCESS(rc))
+        {
+            pWalk->fSucceeded = true;
+            pWalk->GCPtr      = GCPtr;
+            pWalk->GCPhys     = WalkSlat.GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+            pWalk->fEffective = X86_PTE_P | X86_PTE_RW | X86_PTE_US;
+        }
+        else
+            *pWalk = WalkSlat;
+        return rc;
+    }
+#  endif
+
+    /*
+     * Fake it.
+     */
+    pWalk->fSucceeded = true;
+    pWalk->GCPtr      = GCPtr;
+    pWalk->GCPhys     = GCPtr & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    pWalk->fEffective = X86_PTE_P | X86_PTE_RW | X86_PTE_US;
+    NOREF(pVCpu);
+    return VINF_SUCCESS;
+
+#elif PGM_GST_TYPE == PGM_TYPE_32BIT \
+   || PGM_GST_TYPE == PGM_TYPE_PAE \
+   || PGM_GST_TYPE == PGM_TYPE_AMD64
+
+    GSTPTWALK GstWalk;
+    int rc = PGM_GST_NAME(Walk)(pVCpu, GCPtr, pWalk, &GstWalk);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    Assert(pWalk->fSucceeded);
+    Assert(pWalk->GCPtr == GCPtr);
+
+    PGMPTATTRS fFlags;
+    if (!pWalk->fBigPage)
+        fFlags = (GstWalk.Pte.u & ~(GST_PTE_PG_MASK | X86_PTE_RW | X86_PTE_US))                      /* NX not needed */
+               | (pWalk->fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK))
+# if PGM_WITH_NX(PGM_GST_TYPE, PGM_GST_TYPE)
+               | (pWalk->fEffective & PGM_PTATTRS_NX_MASK)
+# endif
+                 ;
+    else
+    {
+        fFlags = (GstWalk.Pde.u & ~(GST_PTE_PG_MASK | X86_PDE4M_RW | X86_PDE4M_US | X86_PDE4M_PS))   /* NX not needed */
+               | (pWalk->fEffective & (PGM_PTATTRS_W_MASK | PGM_PTATTRS_US_MASK | PGM_PTATTRS_PAT_MASK))
+# if PGM_WITH_NX(PGM_GST_TYPE, PGM_GST_TYPE)
+               | (pWalk->fEffective & PGM_PTATTRS_NX_MASK)
+# endif
+               ;
+    }
+
+    pWalk->GCPhys    &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    pWalk->fEffective = fFlags;
+    return VINF_SUCCESS;
+
+#else
+# error "shouldn't be here!"
+    /* something else... */
+    return VERR_NOT_SUPPORTED;
+#endif
+}
+
+
+/**
+ * Modify page flags for a range of pages in the guest's tables
+ *
+ * The existing flags are ANDed with the fMask and ORed with the fFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range. Page aligned!
+ * @param   cb          Size (in bytes) of the page range to apply the modification to. Page aligned!
+ * @param   fFlags      The OR  mask - page flags X86_PTE_*, excluding the page mask of course.
+ * @param   fMask       The AND mask - page flags X86_PTE_*.
+ */
+PGM_GST_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
+{
+    Assert((cb & GUEST_PAGE_OFFSET_MASK) == 0); RT_NOREF_PV(cb);
+
+#if PGM_GST_TYPE == PGM_TYPE_32BIT \
+ || PGM_GST_TYPE == PGM_TYPE_PAE \
+ || PGM_GST_TYPE == PGM_TYPE_AMD64
+    for (;;)
+    {
+        PGMPTWALK Walk;
+        GSTPTWALK GstWalk;
+        int rc = PGM_GST_NAME(Walk)(pVCpu, GCPtr, &Walk, &GstWalk);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        if (!Walk.fBigPage)
+        {
+            /*
+             * 4KB Page table, process
+             *
+             * Walk pages till we're done.
+             */
+            unsigned iPTE = (GCPtr >> GST_PT_SHIFT) & GST_PT_MASK;
+            while (iPTE < RT_ELEMENTS(GstWalk.pPt->a))
+            {
+                GSTPTE Pte = GstWalk.pPt->a[iPTE];
+                Pte.u = (Pte.u & (fMask | X86_PTE_PAE_PG_MASK))
+                      | (fFlags & ~GST_PTE_PG_MASK);
+                GstWalk.pPt->a[iPTE] = Pte;
+
+                /* next page */
+                cb -= GUEST_PAGE_SIZE;
+                if (!cb)
+                    return VINF_SUCCESS;
+                GCPtr += GUEST_PAGE_SIZE;
+                iPTE++;
+            }
+        }
+        else
+        {
+            /*
+             * 2/4MB Page table
+             */
+            GSTPDE PdeNew;
+# if PGM_GST_TYPE == PGM_TYPE_32BIT
+            PdeNew.u = (GstWalk.Pde.u & (fMask | ((fMask & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT) | GST_PDE_BIG_PG_MASK | X86_PDE4M_PG_HIGH_MASK | X86_PDE4M_PS))
+# else
+            PdeNew.u = (GstWalk.Pde.u & (fMask | ((fMask & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT) | GST_PDE_BIG_PG_MASK | X86_PDE4M_PS))
+# endif
+                     | (fFlags & ~GST_PTE_PG_MASK)
+                     | ((fFlags & X86_PTE_PAT) << X86_PDE4M_PAT_SHIFT);
+            *GstWalk.pPde = PdeNew;
+
+            /* advance */
+            const unsigned cbDone = GST_BIG_PAGE_SIZE - (GCPtr & GST_BIG_PAGE_OFFSET_MASK);
+            if (cbDone >= cb)
+                return VINF_SUCCESS;
+            cb    -= cbDone;
+            GCPtr += cbDone;
+        }
+    }
+
+#else
+    /* real / protected mode: ignore. */
+    NOREF(pVCpu); NOREF(GCPtr); NOREF(fFlags); NOREF(fMask);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+#ifdef IN_RING3
+/**
+ * Relocate any GC pointers related to guest mode paging.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offDelta    The relocation offset.
+ */
+PGM_GST_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta)
+{
+    RT_NOREF(pVCpu, offDelta);
+    return VINF_SUCCESS;
+}
+#endif
diff --git a/src/VBox/VMM/VMMAll/PGMAllGstSlatEpt.cpp.h b/src/VBox/VMM/VMMAll/PGMAllGstSlatEpt.cpp.h
new file mode 100644
index 00000000..410abae1
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllGstSlatEpt.cpp.h
@@ -0,0 +1,408 @@
+/* $Id: PGMAllGstSlatEpt.cpp.h $ */
+/** @file
+ * VBox - Page Manager, Guest EPT SLAT - All context code.
+ */
+
+/*
+ * Copyright (C) 2021-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+#if PGM_SLAT_TYPE != PGM_SLAT_TYPE_EPT
+# error "Unsupported SLAT type."
+#endif
+
+/**
+ * Checks if the EPT PTE permissions are valid.
+ *
+ * @returns @c true if valid, @c false otherwise.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   uEntry  The EPT page table entry to check.
+ */
+DECLINLINE(bool) PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(PCVMCPUCC pVCpu, uint64_t uEntry)
+{
+    if (!(uEntry & EPT_E_READ))
+    {
+        Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt);
+        Assert(!RT_BF_GET(pVCpu->pgm.s.uEptVpidCapMsr, VMX_BF_EPT_VPID_CAP_EXEC_ONLY));
+        NOREF(pVCpu);
+        if (uEntry & (EPT_E_WRITE | EPT_E_EXECUTE))
+            return false;
+    }
+    return true;
+}
+
+
+/**
+ * Checks if the EPT memory type is valid.
+ *
+ * @returns @c true if valid, @c false otherwise.
+ * @param   uEntry      The EPT page table entry to check.
+ * @param   uLevel      The page table walk level.
+ */
+DECLINLINE(bool) PGM_GST_SLAT_NAME_EPT(WalkIsMemTypeValid)(uint64_t uEntry, uint8_t uLevel)
+{
+    Assert(uLevel <= 3 && uLevel >= 1); NOREF(uLevel);
+    uint8_t const fEptMemTypeMask = uEntry & VMX_BF_EPT_PT_MEMTYPE_MASK;
+    switch (fEptMemTypeMask)
+    {
+        case EPT_E_MEMTYPE_WB:
+        case EPT_E_MEMTYPE_UC:
+        case EPT_E_MEMTYPE_WP:
+        case EPT_E_MEMTYPE_WT:
+        case EPT_E_MEMTYPE_WC:
+            return true;
+    }
+    return false;
+}
+
+
+/**
+ * Updates page walk result info when a not-present page is encountered.
+ *
+ * @returns VERR_PAGE_TABLE_NOT_PRESENT.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pWalk   The page walk info to update.
+ * @param   uEntry  The EPT PTE that is not present.
+ * @param   uLevel  The page table walk level.
+ */
+DECLINLINE(int) PGM_GST_SLAT_NAME_EPT(WalkReturnNotPresent)(PCVMCPUCC pVCpu, PPGMPTWALK pWalk, uint64_t uEntry, uint8_t uLevel)
+{
+    static PGMWALKFAIL const s_afEptViolations[] = { PGM_WALKFAIL_EPT_VIOLATION, PGM_WALKFAIL_EPT_VIOLATION_CONVERTIBLE };
+    uint8_t const fEptVeSupported  = pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxEptXcptVe;
+    uint8_t const fConvertible     = RT_BOOL(uLevel == 1 || (uEntry & EPT_E_BIT_LEAF));
+    uint8_t const idxViolationType = fEptVeSupported & fConvertible & !RT_BF_GET(uEntry, VMX_BF_EPT_PT_SUPPRESS_VE);
+
+    pWalk->fNotPresent = true;
+    pWalk->uLevel      = uLevel;
+    pWalk->fFailed     = s_afEptViolations[idxViolationType];
+    return VERR_PAGE_TABLE_NOT_PRESENT;
+}
+
+
+/**
+ * Updates page walk result info when a bad physical address is encountered.
+ *
+ * @returns VERR_PAGE_TABLE_NOT_PRESENT .
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pWalk   The page walk info to update.
+ * @param   uLevel  The page table walk level.
+ * @param   rc      The error code that caused this bad physical address situation.
+ */
+DECLINLINE(int) PGM_GST_SLAT_NAME_EPT(WalkReturnBadPhysAddr)(PCVMCPUCC pVCpu, PPGMPTWALK pWalk, uint8_t uLevel, int rc)
+{
+    AssertMsg(rc == VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS, ("%Rrc\n", rc)); NOREF(rc); NOREF(pVCpu);
+    pWalk->fBadPhysAddr = true;
+    pWalk->uLevel       = uLevel;
+    pWalk->fFailed      = PGM_WALKFAIL_EPT_VIOLATION;
+    return VERR_PAGE_TABLE_NOT_PRESENT;
+}
+
+
+/**
+ * Updates page walk result info when reserved bits are encountered.
+ *
+ * @returns VERR_PAGE_TABLE_NOT_PRESENT.
+ * @param   pVCpu   The cross context virtual CPU structure of the calling EMT.
+ * @param   pWalk   The page walk info to update.
+ * @param   uLevel  The page table walk level.
+ */
+DECLINLINE(int) PGM_GST_SLAT_NAME_EPT(WalkReturnRsvdError)(PVMCPUCC pVCpu, PPGMPTWALK pWalk, uint8_t uLevel)
+{
+    NOREF(pVCpu);
+    pWalk->fRsvdError = true;
+    pWalk->uLevel     = uLevel;
+    pWalk->fFailed    = PGM_WALKFAIL_EPT_MISCONFIG;
+    return VERR_PAGE_TABLE_NOT_PRESENT;
+}
+
+
+/**
+ * Walks the guest's EPT page table (second-level address translation).
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPhysNested        The nested-guest physical address to walk.
+ * @param   fIsLinearAddrValid  Whether the linear-address in @c GCPtrNested caused
+ *                              this page walk.
+ * @param   GCPtrNested         The nested-guest linear address that caused this
+ *                              translation. If @c fIsLinearAddrValid is false, pass
+ *                              0.
+ * @param   pWalk               The page walk info.
+ * @param   pSlatWalk           The SLAT mode specific page walk info.
+ */
+DECLINLINE(int) PGM_GST_SLAT_NAME_EPT(Walk)(PVMCPUCC pVCpu, RTGCPHYS GCPhysNested, bool fIsLinearAddrValid, RTGCPTR GCPtrNested,
+                                            PPGMPTWALK pWalk, PSLATPTWALK pSlatWalk)
+{
+    Assert(fIsLinearAddrValid || GCPtrNested == 0);
+
+    /*
+     * Init walk structures.
+     */
+    RT_ZERO(*pWalk);
+    RT_ZERO(*pSlatWalk);
+
+    pWalk->GCPtr              = GCPtrNested;
+    pWalk->GCPhysNested       = GCPhysNested;
+    pWalk->fIsLinearAddrValid = fIsLinearAddrValid;
+    pWalk->fIsSlat            = true;
+
+    /*
+     * Figure out EPT attributes that are cumulative (logical-AND) across page walks.
+     *   - R, W, X_SUPER are unconditionally cumulative.
+     *     See Intel spec. Table 26-7 "Exit Qualification for EPT Violations".
+     *
+     *   - X_USER is cumulative but relevant only when mode-based execute control for EPT
+     *     which we currently don't support it (asserted below).
+     *
+     *   - MEMTYPE is not cumulative and only applicable to the final paging entry.
+     *
+     *   - A, D EPT bits map to the regular page-table bit positions. Thus, they're not
+     *     included in the mask below and handled separately. Accessed bits are
+     *     cumulative but dirty bits are not cumulative as they're only applicable to
+     *     the final paging entry.
+     */
+    Assert(!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fVmxModeBasedExecuteEpt);
+    uint64_t const fEptAndMask = (  PGM_PTATTRS_EPT_R_MASK
+                                  | PGM_PTATTRS_EPT_W_MASK
+                                  | PGM_PTATTRS_EPT_X_SUPER_MASK) & PGM_PTATTRS_EPT_MASK;
+
+    /*
+     * Do the walk.
+     */
+    uint64_t fEffective;
+    {
+        /*
+         * EPTP.
+         *
+         * We currently only support 4-level EPT paging.
+         * EPT 5-level paging was documented at some point (bit 7 of MSR_IA32_VMX_EPT_VPID_CAP)
+         * but for some reason seems to have been removed from subsequent specs.
+         */
+        int const rc = pgmGstGetEptPML4PtrEx(pVCpu, &pSlatWalk->pPml4);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnBadPhysAddr)(pVCpu, pWalk, 4, rc);
+    }
+    {
+        /*
+         * PML4E.
+         */
+        PEPTPML4E pPml4e;
+        pSlatWalk->pPml4e = pPml4e = &pSlatWalk->pPml4->a[(GCPhysNested >> SLAT_PML4_SHIFT) & SLAT_PML4_MASK];
+        EPTPML4E  Pml4e;
+        pSlatWalk->Pml4e.u = Pml4e.u = pPml4e->u;
+
+        if (SLAT_IS_PGENTRY_PRESENT(pVCpu, Pml4e)) { /* probable */ }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnNotPresent)(pVCpu, pWalk, Pml4e.u, 4);
+
+        if (RT_LIKELY(   SLAT_IS_PML4E_VALID(pVCpu, Pml4e)
+                      && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pml4e.u)))
+        { /* likely */ }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnRsvdError)(pVCpu, pWalk, 4);
+
+        uint64_t const fEptAttrs   = Pml4e.u & EPT_PML4E_ATTR_MASK;
+        uint8_t const  fRead       = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ);
+        uint8_t const  fWrite      = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE);
+        uint8_t const  fExecute    = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE);
+        uint8_t const  fAccessed   = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED);
+        uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask;
+        fEffective = RT_BF_MAKE(PGM_PTATTRS_R,   fRead)
+                   | RT_BF_MAKE(PGM_PTATTRS_W,   fWrite)
+                   | RT_BF_MAKE(PGM_PTATTRS_NX, !fExecute)
+                   | RT_BF_MAKE(PGM_PTATTRS_A,   fAccessed)
+                   | fEptAndBits;
+        pWalk->fEffective = fEffective;
+
+        int const rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, Pml4e.u & EPT_PML4E_PG_MASK, &pSlatWalk->pPdpt);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc);
+    }
+    {
+        /*
+         * PDPTE.
+         */
+        PEPTPDPTE pPdpte;
+        pSlatWalk->pPdpte = pPdpte = &pSlatWalk->pPdpt->a[(GCPhysNested >> SLAT_PDPT_SHIFT) & SLAT_PDPT_MASK];
+        EPTPDPTE  Pdpte;
+        pSlatWalk->Pdpte.u = Pdpte.u = pPdpte->u;
+
+        if (SLAT_IS_PGENTRY_PRESENT(pVCpu, Pdpte)) { /* probable */ }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnNotPresent)(pVCpu, pWalk, Pdpte.u, 3);
+
+        /* The order of the following "if" and "else if" statements matter. */
+        if (   SLAT_IS_PDPE_VALID(pVCpu, Pdpte)
+            && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pdpte.u))
+        {
+            uint64_t const fEptAttrs   = Pdpte.u & EPT_PDPTE_ATTR_MASK;
+            uint8_t const  fRead       = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ);
+            uint8_t const  fWrite      = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE);
+            uint8_t const  fExecute    = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE);
+            uint8_t const  fAccessed   = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED);
+            uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask;
+            fEffective &= RT_BF_MAKE(PGM_PTATTRS_R,   fRead)
+                        | RT_BF_MAKE(PGM_PTATTRS_W,   fWrite)
+                        | RT_BF_MAKE(PGM_PTATTRS_A,   fAccessed)
+                        | fEptAndBits;
+            fEffective |= RT_BF_MAKE(PGM_PTATTRS_NX, !fExecute);
+            pWalk->fEffective = fEffective;
+        }
+        else if (   SLAT_IS_BIG_PDPE_VALID(pVCpu, Pdpte)
+                 && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pdpte.u)
+                 && PGM_GST_SLAT_NAME_EPT(WalkIsMemTypeValid)(Pdpte.u, 3))
+        {
+            uint64_t const fEptAttrs   = Pdpte.u & EPT_PDPTE1G_ATTR_MASK;
+            uint8_t const  fRead       = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ);
+            uint8_t const  fWrite      = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE);
+            uint8_t const  fExecute    = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE);
+            uint8_t const  fAccessed   = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED);
+            uint8_t const  fDirty      = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_DIRTY);
+            uint8_t const  fMemType    = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_MEMTYPE);
+            uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask;
+            fEffective &= RT_BF_MAKE(PGM_PTATTRS_R,            fRead)
+                        | RT_BF_MAKE(PGM_PTATTRS_W,            fWrite)
+                        | RT_BF_MAKE(PGM_PTATTRS_A,            fAccessed)
+                        | fEptAndBits;
+            fEffective |= RT_BF_MAKE(PGM_PTATTRS_D,            fDirty)
+                        | RT_BF_MAKE(PGM_PTATTRS_EPT_MEMTYPE,  fMemType)
+                        | RT_BF_MAKE(PGM_PTATTRS_NX,          !fExecute);
+            pWalk->fEffective = fEffective;
+
+            pWalk->fGigantPage = true;
+            pWalk->fSucceeded  = true;
+            pWalk->GCPhys      = SLAT_GET_PDPE1G_GCPHYS(pVCpu, Pdpte)
+                               | (GCPhysNested & SLAT_PAGE_1G_OFFSET_MASK);
+            PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->GCPhys);
+            return VINF_SUCCESS;
+        }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnRsvdError)(pVCpu, pWalk, 3);
+
+        int const rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, Pdpte.u & EPT_PDPTE_PG_MASK, &pSlatWalk->pPd);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnBadPhysAddr)(pVCpu, pWalk, 3, rc);
+    }
+    {
+        /*
+         * PDE.
+         */
+        PSLATPDE pPde;
+        pSlatWalk->pPde  = pPde  = &pSlatWalk->pPd->a[(GCPhysNested >> SLAT_PD_SHIFT) & SLAT_PD_MASK];
+        SLATPDE  Pde;
+        pSlatWalk->Pde.u = Pde.u = pPde->u;
+
+        if (SLAT_IS_PGENTRY_PRESENT(pVCpu, Pde)) { /* probable */ }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnNotPresent)(pVCpu, pWalk, Pde.u, 2);
+
+        /* The order of the following "if" and "else if" statements matter. */
+        if (   SLAT_IS_PDE_VALID(pVCpu, Pde)
+            && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pde.u))
+        {
+            uint64_t const fEptAttrs   = Pde.u & EPT_PDE_ATTR_MASK;
+            uint8_t const  fRead       = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ);
+            uint8_t const  fWrite      = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE);
+            uint8_t const  fExecute    = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE);
+            uint8_t const  fAccessed   = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED);
+            uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask;
+            fEffective &= RT_BF_MAKE(PGM_PTATTRS_R,   fRead)
+                        | RT_BF_MAKE(PGM_PTATTRS_W,   fWrite)
+                        | RT_BF_MAKE(PGM_PTATTRS_A,   fAccessed)
+                        | fEptAndBits;
+            fEffective |= RT_BF_MAKE(PGM_PTATTRS_NX, !fExecute);
+            pWalk->fEffective = fEffective;
+        }
+        else if (   SLAT_IS_BIG_PDE_VALID(pVCpu, Pde)
+                 && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pde.u)
+                 && PGM_GST_SLAT_NAME_EPT(WalkIsMemTypeValid)(Pde.u, 2))
+        {
+            uint64_t const fEptAttrs   = Pde.u & EPT_PDE2M_ATTR_MASK;
+            uint8_t const  fRead       = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ);
+            uint8_t const  fWrite      = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE);
+            uint8_t const  fExecute    = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE);
+            uint8_t const  fAccessed   = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED);
+            uint8_t const  fDirty      = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_DIRTY);
+            uint8_t const  fMemType    = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_MEMTYPE);
+            uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask;
+            fEffective &= RT_BF_MAKE(PGM_PTATTRS_R,            fRead)
+                        | RT_BF_MAKE(PGM_PTATTRS_W,            fWrite)
+                        | RT_BF_MAKE(PGM_PTATTRS_A,            fAccessed)
+                        | fEptAndBits;
+            fEffective |= RT_BF_MAKE(PGM_PTATTRS_D,            fDirty)
+                        | RT_BF_MAKE(PGM_PTATTRS_EPT_MEMTYPE,  fMemType)
+                        | RT_BF_MAKE(PGM_PTATTRS_NX,          !fExecute);
+            pWalk->fEffective = fEffective;
+
+            pWalk->fBigPage    = true;
+            pWalk->fSucceeded  = true;
+            pWalk->GCPhys      = SLAT_GET_PDE2M_GCPHYS(pVCpu, Pde)
+                               | (GCPhysNested & SLAT_PAGE_2M_OFFSET_MASK);
+            PGM_A20_APPLY_TO_VAR(pVCpu, pWalk->GCPhys);
+            return VINF_SUCCESS;
+        }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnRsvdError)(pVCpu, pWalk, 2);
+
+        int const rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, Pde.u & EPT_PDE_PG_MASK, &pSlatWalk->pPt);
+        if (RT_SUCCESS(rc)) { /* probable */ }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnBadPhysAddr)(pVCpu, pWalk, 1, rc);
+    }
+    {
+        /*
+         * PTE.
+         */
+        PSLATPTE pPte;
+        pSlatWalk->pPte  = pPte  = &pSlatWalk->pPt->a[(GCPhysNested >> SLAT_PT_SHIFT) & SLAT_PT_MASK];
+        SLATPTE  Pte;
+        pSlatWalk->Pte.u = Pte.u = pPte->u;
+
+        if (SLAT_IS_PGENTRY_PRESENT(pVCpu, Pte)) { /* probable */ }
+        else return PGM_GST_SLAT_NAME_EPT(WalkReturnNotPresent)(pVCpu, pWalk, Pte.u, 1);
+
+        if (   SLAT_IS_PTE_VALID(pVCpu, Pte)
+            && PGM_GST_SLAT_NAME_EPT(WalkIsPermValid)(pVCpu, Pte.u)
+            && PGM_GST_SLAT_NAME_EPT(WalkIsMemTypeValid)(Pte.u, 1))
+        { /* likely*/ }
+        else
+            return PGM_GST_SLAT_NAME_EPT(WalkReturnRsvdError)(pVCpu, pWalk, 1);
+
+        uint64_t const fEptAttrs   = Pte.u & EPT_PTE_ATTR_MASK;
+        uint8_t const  fRead       = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_READ);
+        uint8_t const  fWrite      = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_WRITE);
+        uint8_t const  fExecute    = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_EXECUTE);
+        uint8_t const  fAccessed   = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_ACCESSED);
+        uint8_t const  fDirty      = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_DIRTY);
+        uint8_t const  fMemType    = RT_BF_GET(fEptAttrs, VMX_BF_EPT_PT_MEMTYPE);
+        uint64_t const fEptAndBits = (fEptAttrs << PGM_PTATTRS_EPT_SHIFT) & fEptAndMask;
+        fEffective &= RT_BF_MAKE(PGM_PTATTRS_R,            fRead)
+                    | RT_BF_MAKE(PGM_PTATTRS_W,            fWrite)
+                    | RT_BF_MAKE(PGM_PTATTRS_A,            fAccessed)
+                    | fEptAndBits;
+        fEffective |= RT_BF_MAKE(PGM_PTATTRS_D,            fDirty)
+                    | RT_BF_MAKE(PGM_PTATTRS_EPT_MEMTYPE,  fMemType)
+                    | RT_BF_MAKE(PGM_PTATTRS_NX,          !fExecute);
+        pWalk->fEffective = fEffective;
+
+        pWalk->fSucceeded   = true;
+        pWalk->GCPhys       = SLAT_GET_PTE_GCPHYS(pVCpu, Pte) | (GCPhysNested & GUEST_PAGE_OFFSET_MASK);
+        return VINF_SUCCESS;
+    }
+}
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllHandler.cpp b/src/VBox/VMM/VMMAll/PGMAllHandler.cpp
new file mode 100644
index 00000000..c7d8a433
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllHandler.cpp
@@ -0,0 +1,2001 @@
+/* $Id: PGMAllHandler.cpp $ */
+/** @file
+ * PGM - Page Manager / Monitor, Access Handlers.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM
+#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/nem.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/dbgf.h>
+#ifdef IN_RING0
+# include <VBox/vmm/pdmdev.h>
+#endif
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include "PGMInline.h"
+
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+# include <iprt/asm-amd64-x86.h>
+#endif
+#include <iprt/string.h>
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/vmm/selm.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** Dummy physical access handler type record. */
+CTX_SUFF(PGMPHYSHANDLERTYPEINT) const g_pgmHandlerPhysicalDummyType =
+{
+    /* .hType = */              UINT64_C(0x93b7557e1937aaff),
+    /* .enmKind = */            PGMPHYSHANDLERKIND_INVALID,
+    /* .uState = */             PGM_PAGE_HNDL_PHYS_STATE_ALL,
+    /* .fKeepPgmLock = */       true,
+    /* .fRing0DevInsIdx = */    false,
+#ifdef IN_RING0
+    /* .fNotInHm = */           false,
+    /* .pfnHandler = */         pgmR0HandlerPhysicalHandlerToRing3,
+    /* .pfnPfHandler = */       pgmR0HandlerPhysicalPfHandlerToRing3,
+#elif defined(IN_RING3)
+    /* .fRing0Enabled = */      false,
+    /* .fNotInHm = */           false,
+    /* .pfnHandler = */         pgmR3HandlerPhysicalHandlerInvalid,
+#else
+# error "unsupported context"
+#endif
+    /* .pszDesc = */            "dummy"
+};
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+static int  pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(PVMCC pVM, PPGMPHYSHANDLER pCur, PPGMRAMRANGE pRam,
+                                                           void *pvBitmap, uint32_t offBitmap);
+static void pgmHandlerPhysicalDeregisterNotifyNEM(PVMCC pVM, PPGMPHYSHANDLER pCur);
+static void pgmHandlerPhysicalResetRamFlags(PVMCC pVM, PPGMPHYSHANDLER pCur);
+
+
+#ifndef IN_RING3
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER,
+ *      Dummy for forcing ring-3 handling of the access.}
+ */
+DECLCALLBACK(VBOXSTRICTRC)
+pgmR0HandlerPhysicalHandlerToRing3(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
+                                   PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
+{
+    RT_NOREF(pVM, pVCpu, GCPhys, pvPhys, pvBuf, cbBuf, enmAccessType, enmOrigin, uUser);
+    return VINF_EM_RAW_EMULATE_INSTR;
+}
+
+
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      Dummy for forcing ring-3 handling of the access.}
+ */
+DECLCALLBACK(VBOXSTRICTRC)
+pgmR0HandlerPhysicalPfHandlerToRing3(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
+                                     RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
+{
+    RT_NOREF(pVM, pVCpu, uErrorCode, pCtx, pvFault, GCPhysFault, uUser);
+    return VINF_EM_RAW_EMULATE_INSTR;
+}
+
+#endif /* !IN_RING3 */
+
+
+/**
+ * Creates a physical access handler, allocation part.
+ *
+ * @returns VBox status code.
+ * @retval  VERR_OUT_OF_RESOURCES if no more handlers available.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   hType           The handler type registration handle.
+ * @param   uUser           User argument to the handlers (not pointer).
+ * @param   pszDesc         Description of this handler.  If NULL, the type
+ *                          description will be used instead.
+ * @param   ppPhysHandler   Where to return the access handler structure on
+ *                          success.
+ */
+int pgmHandlerPhysicalExCreate(PVMCC pVM, PGMPHYSHANDLERTYPE hType, uint64_t uUser,
+                               R3PTRTYPE(const char *) pszDesc, PPGMPHYSHANDLER *ppPhysHandler)
+{
+    /*
+     * Validate input.
+     */
+    PCPGMPHYSHANDLERTYPEINT const pType = pgmHandlerPhysicalTypeHandleToPtr(pVM, hType);
+    AssertReturn(pType, VERR_INVALID_HANDLE);
+    AssertReturn(pType->enmKind > PGMPHYSHANDLERKIND_INVALID && pType->enmKind < PGMPHYSHANDLERKIND_END, VERR_INVALID_HANDLE);
+    AssertPtr(ppPhysHandler);
+
+    Log(("pgmHandlerPhysicalExCreate: uUser=%#RX64 hType=%#x (%d, %s) pszDesc=%RHv:%s\n",
+         uUser, hType, pType->enmKind, pType->pszDesc, pszDesc, R3STRING(pszDesc)));
+
+    /*
+     * Allocate and initialize the new entry.
+     */
+    int rc = PGM_LOCK(pVM);
+    AssertRCReturn(rc, rc);
+
+    PPGMPHYSHANDLER pNew = pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator.allocateNode();
+    if (pNew)
+    {
+        pNew->Key           = NIL_RTGCPHYS;
+        pNew->KeyLast       = NIL_RTGCPHYS;
+        pNew->cPages        = 0;
+        pNew->cAliasedPages = 0;
+        pNew->cTmpOffPages  = 0;
+        pNew->uUser         = uUser;
+        pNew->hType         = hType;
+        pNew->pszDesc       = pszDesc != NIL_RTR3PTR ? pszDesc
+#ifdef IN_RING3
+                            : pType->pszDesc;
+#else
+                            : pVM->pgm.s.aPhysHandlerTypes[hType & PGMPHYSHANDLERTYPE_IDX_MASK].pszDesc;
+#endif
+
+        PGM_UNLOCK(pVM);
+        *ppPhysHandler = pNew;
+        return VINF_SUCCESS;
+    }
+
+    PGM_UNLOCK(pVM);
+    return VERR_OUT_OF_RESOURCES;
+}
+
+
+/**
+ * Duplicates a physical access handler.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when successfully installed.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pPhysHandlerSrc The source handler to duplicate
+ * @param   ppPhysHandler   Where to return the access handler structure on
+ *                          success.
+ */
+int pgmHandlerPhysicalExDup(PVMCC pVM, PPGMPHYSHANDLER pPhysHandlerSrc, PPGMPHYSHANDLER *ppPhysHandler)
+{
+    return pgmHandlerPhysicalExCreate(pVM, pPhysHandlerSrc->hType, pPhysHandlerSrc->uUser,
+                                      pPhysHandlerSrc->pszDesc, ppPhysHandler);
+}
+
+
+/**
+ * Register a access handler for a physical range.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when successfully installed.
+ * @retval  VINF_PGM_GCPHYS_ALIASED could be returned.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pPhysHandler    The physical handler.
+ * @param   GCPhys          Start physical address.
+ * @param   GCPhysLast      Last physical address. (inclusive)
+ */
+int pgmHandlerPhysicalExRegister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast)
+{
+    /*
+     * Validate input.
+     */
+    AssertReturn(pPhysHandler, VERR_INVALID_POINTER);
+    PGMPHYSHANDLERTYPE const      hType = pPhysHandler->hType;
+    PCPGMPHYSHANDLERTYPEINT const pType = pgmHandlerPhysicalTypeHandleToPtr(pVM, hType);
+    AssertReturn(pType, VERR_INVALID_HANDLE);
+    AssertReturn(pType->enmKind > PGMPHYSHANDLERKIND_INVALID && pType->enmKind < PGMPHYSHANDLERKIND_END, VERR_INVALID_HANDLE);
+
+    AssertPtr(pPhysHandler);
+
+    Log(("pgmHandlerPhysicalExRegister: GCPhys=%RGp GCPhysLast=%RGp hType=%#x (%d, %s) pszDesc=%RHv:%s\n", GCPhys, GCPhysLast,
+         hType, pType->enmKind, pType->pszDesc, pPhysHandler->pszDesc, R3STRING(pPhysHandler->pszDesc)));
+    AssertReturn(pPhysHandler->Key == NIL_RTGCPHYS, VERR_WRONG_ORDER);
+
+    AssertMsgReturn(GCPhys < GCPhysLast, ("GCPhys >= GCPhysLast (%#x >= %#x)\n", GCPhys, GCPhysLast), VERR_INVALID_PARAMETER);
+    Assert(GCPhysLast - GCPhys < _4G); /* ASSUMPTION in PGMAllPhys.cpp */
+
+    switch (pType->enmKind)
+    {
+        case PGMPHYSHANDLERKIND_WRITE:
+            if (!pType->fNotInHm)
+                break;
+            RT_FALL_THRU(); /* Simplification: fNotInHm can only be used with full pages */
+        case PGMPHYSHANDLERKIND_MMIO:
+        case PGMPHYSHANDLERKIND_ALL:
+            /* Simplification for PGMPhysRead, PGMR0Trap0eHandlerNPMisconfig and others: Full pages. */
+            AssertMsgReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), ("%RGp\n", GCPhys), VERR_INVALID_PARAMETER);
+            AssertMsgReturn((GCPhysLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK, ("%RGp\n", GCPhysLast), VERR_INVALID_PARAMETER);
+            break;
+        default:
+            AssertMsgFailed(("Invalid input enmKind=%d!\n", pType->enmKind));
+            return VERR_INVALID_PARAMETER;
+    }
+
+    /*
+     * We require the range to be within registered ram.
+     * There is no apparent need to support ranges which cover more than one ram range.
+     */
+    PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
+    if (   !pRam
+        || GCPhysLast > pRam->GCPhysLast)
+    {
+#ifdef IN_RING3
+        DBGFR3Info(pVM->pUVM, "phys", NULL, NULL);
+#endif
+        AssertMsgFailed(("No RAM range for %RGp-%RGp\n", GCPhys, GCPhysLast));
+        return VERR_PGM_HANDLER_PHYSICAL_NO_RAM_RANGE;
+    }
+    Assert(GCPhys >= pRam->GCPhys && GCPhys < pRam->GCPhysLast);
+    Assert(GCPhysLast <= pRam->GCPhysLast && GCPhysLast >= pRam->GCPhys);
+
+    /*
+     * Try insert into list.
+     */
+    pPhysHandler->Key     = GCPhys;
+    pPhysHandler->KeyLast = GCPhysLast;
+    pPhysHandler->cPages  = (GCPhysLast - (GCPhys & X86_PTE_PAE_PG_MASK) + GUEST_PAGE_SIZE) >> GUEST_PAGE_SHIFT;
+
+    int rc = PGM_LOCK(pVM);
+    if (RT_SUCCESS(rc))
+    {
+        rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->insert(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, pPhysHandler);
+        if (RT_SUCCESS(rc))
+        {
+            rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pPhysHandler, pRam, NULL /*pvBitmap*/, 0 /*offBitmap*/);
+            if (rc == VINF_PGM_SYNC_CR3)
+                rc = VINF_PGM_GCPHYS_ALIASED;
+
+#if defined(IN_RING3) || defined(IN_RING0)
+            NEMHCNotifyHandlerPhysicalRegister(pVM, pType->enmKind, GCPhys, GCPhysLast - GCPhys + 1);
+#endif
+            PGM_UNLOCK(pVM);
+
+            if (rc != VINF_SUCCESS)
+                Log(("PGMHandlerPhysicalRegisterEx: returns %Rrc (%RGp-%RGp)\n", rc, GCPhys, GCPhysLast));
+            return rc;
+        }
+        PGM_UNLOCK(pVM);
+    }
+
+    pPhysHandler->Key     = NIL_RTGCPHYS;
+    pPhysHandler->KeyLast = NIL_RTGCPHYS;
+
+    AssertMsgReturn(rc == VERR_ALREADY_EXISTS, ("%Rrc GCPhys=%RGp GCPhysLast=%RGp\n", rc, GCPhys, GCPhysLast), rc);
+
+#if defined(IN_RING3) && defined(VBOX_STRICT)
+    DBGFR3Info(pVM->pUVM, "handlers", "phys nostats", NULL);
+#endif
+    AssertMsgFailed(("Conflict! GCPhys=%RGp GCPhysLast=%RGp pszDesc=%s/%s\n",
+                     GCPhys, GCPhysLast, R3STRING(pPhysHandler->pszDesc), R3STRING(pType->pszDesc)));
+    return VERR_PGM_HANDLER_PHYSICAL_CONFLICT;
+}
+
+
+/**
+ * Register a access handler for a physical range.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when successfully installed.
+ * @retval  VINF_PGM_GCPHYS_ALIASED when the shadow PTs could be updated because
+ *          the guest page aliased or/and mapped by multiple PTs. A CR3 sync has been
+ *          flagged together with a pool clearing.
+ * @retval  VERR_PGM_HANDLER_PHYSICAL_CONFLICT if the range conflicts with an existing
+ *          one. A debug assertion is raised.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Start physical address.
+ * @param   GCPhysLast      Last physical address. (inclusive)
+ * @param   hType           The handler type registration handle.
+ * @param   uUser           User argument to the handler.
+ * @param   pszDesc         Description of this handler.  If NULL, the type
+ *                          description will be used instead.
+ */
+VMMDECL(int) PGMHandlerPhysicalRegister(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, PGMPHYSHANDLERTYPE hType,
+                                        uint64_t uUser, R3PTRTYPE(const char *) pszDesc)
+{
+#ifdef LOG_ENABLED
+    PCPGMPHYSHANDLERTYPEINT pType = pgmHandlerPhysicalTypeHandleToPtr(pVM, hType);
+    Log(("PGMHandlerPhysicalRegister: GCPhys=%RGp GCPhysLast=%RGp uUser=%#RX64 hType=%#x (%d, %s) pszDesc=%RHv:%s\n",
+         GCPhys, GCPhysLast, uUser, hType, pType->enmKind, R3STRING(pType->pszDesc), pszDesc, R3STRING(pszDesc)));
+#endif
+
+    PPGMPHYSHANDLER pNew;
+    int rc = pgmHandlerPhysicalExCreate(pVM, hType, uUser, pszDesc, &pNew);
+    if (RT_SUCCESS(rc))
+    {
+        rc = pgmHandlerPhysicalExRegister(pVM, pNew, GCPhys, GCPhysLast);
+        if (RT_SUCCESS(rc))
+            return rc;
+        pgmHandlerPhysicalExDestroy(pVM, pNew);
+    }
+    return rc;
+}
+
+
+/**
+ * Sets ram range flags and attempts updating shadow PTs.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS when shadow PTs was successfully updated.
+ * @retval  VINF_PGM_SYNC_CR3 when the shadow PTs could be updated because
+ *          the guest page aliased or/and mapped by multiple PTs. FFs set.
+ * @param   pVM         The cross context VM structure.
+ * @param   pCur        The physical handler.
+ * @param   pRam        The RAM range.
+ * @param   pvBitmap    Dirty bitmap. Optional.
+ * @param   offBitmap   Dirty bitmap offset.
+ */
+static int pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(PVMCC pVM, PPGMPHYSHANDLER pCur, PPGMRAMRANGE pRam,
+                                                          void *pvBitmap, uint32_t offBitmap)
+{
+    /*
+     * Iterate the guest ram pages updating the flags and flushing PT entries
+     * mapping the page.
+     */
+    bool                    fFlushTLBs = false;
+    int                     rc         = VINF_SUCCESS;
+    PCPGMPHYSHANDLERTYPEINT pCurType   = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+    const unsigned          uState     = pCurType->uState;
+    uint32_t                cPages     = pCur->cPages;
+    uint32_t                i          = (pCur->Key - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
+    for (;;)
+    {
+        PPGMPAGE pPage = &pRam->aPages[i];
+        AssertMsg(pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO || PGM_PAGE_IS_MMIO(pPage),
+                  ("%RGp %R[pgmpage]\n", pRam->GCPhys + (i << GUEST_PAGE_SHIFT), pPage));
+
+        /* Only do upgrades. */
+        if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) < uState)
+        {
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, uState, pCurType->fNotInHm);
+
+            const RTGCPHYS GCPhysPage = pRam->GCPhys + (i << GUEST_PAGE_SHIFT);
+            int rc2 = pgmPoolTrackUpdateGCPhys(pVM, GCPhysPage, pPage,
+                                               false /* allow updates of PTEs (instead of flushing) */, &fFlushTLBs);
+            if (rc2 != VINF_SUCCESS && rc == VINF_SUCCESS)
+                rc = rc2;
+
+#ifdef VBOX_WITH_NATIVE_NEM
+            /* Tell NEM about the protection update. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                NEMHCNotifyPhysPageProtChanged(pVM, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage),
+                                               PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage),
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+#endif
+            if (pvBitmap)
+                ASMBitSet(pvBitmap, offBitmap);
+        }
+
+        /* next */
+        if (--cPages == 0)
+            break;
+        i++;
+        offBitmap++;
+    }
+
+    if (fFlushTLBs)
+    {
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+        Log(("pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs: flushing guest TLBs; rc=%d\n", rc));
+    }
+    else
+        Log(("pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs: doesn't flush guest TLBs. rc=%Rrc; sync flags=%x VMCPU_FF_PGM_SYNC_CR3=%d\n", rc, VMMGetCpu(pVM)->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(VMMGetCpu(pVM), VMCPU_FF_PGM_SYNC_CR3)));
+
+    return rc;
+}
+
+
+/**
+ * Deregister a physical page access handler.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pPhysHandler    The handler to deregister (but not free).
+ */
+int pgmHandlerPhysicalExDeregister(PVMCC pVM, PPGMPHYSHANDLER pPhysHandler)
+{
+    LogFlow(("pgmHandlerPhysicalExDeregister: Removing Range %RGp-%RGp %s\n",
+             pPhysHandler->Key, pPhysHandler->KeyLast, R3STRING(pPhysHandler->pszDesc)));
+
+    int rc = PGM_LOCK(pVM);
+    AssertRCReturn(rc, rc);
+
+    RTGCPHYS const GCPhys = pPhysHandler->Key;
+    AssertReturnStmt(GCPhys != NIL_RTGCPHYS, PGM_UNLOCK(pVM), VERR_PGM_HANDLER_NOT_FOUND);
+
+    /*
+     * Remove the handler from the tree.
+     */
+
+    PPGMPHYSHANDLER pRemoved;
+    rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->remove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pRemoved);
+    if (RT_SUCCESS(rc))
+    {
+        if (pRemoved == pPhysHandler)
+        {
+            /*
+             * Clear the page bits, notify the REM about this change and clear
+             * the cache.
+             */
+            pgmHandlerPhysicalResetRamFlags(pVM, pPhysHandler);
+            if (VM_IS_NEM_ENABLED(pVM))
+                pgmHandlerPhysicalDeregisterNotifyNEM(pVM, pPhysHandler);
+            pVM->pgm.s.idxLastPhysHandler = 0;
+
+            pPhysHandler->Key     = NIL_RTGCPHYS;
+            pPhysHandler->KeyLast = NIL_RTGCPHYS;
+
+            PGM_UNLOCK(pVM);
+
+            return VINF_SUCCESS;
+        }
+
+        /*
+         * Both of the failure conditions here are considered internal processing
+         * errors because they can only be caused by race conditions or corruption.
+         * If we ever need to handle concurrent deregistration, we have to move
+         * the NIL_RTGCPHYS check inside the PGM lock.
+         */
+        pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->insert(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, pRemoved);
+    }
+
+    PGM_UNLOCK(pVM);
+
+    if (RT_FAILURE(rc))
+        AssertMsgFailed(("Didn't find range starting at %RGp in the tree! %Rrc=rc\n", GCPhys, rc));
+    else
+        AssertMsgFailed(("Found different handle at %RGp in the tree: got %p insteaded of %p\n",
+                         GCPhys, pRemoved, pPhysHandler));
+    return VERR_PGM_HANDLER_IPE_1;
+}
+
+
+/**
+ * Destroys (frees) a physical handler.
+ *
+ * The caller must deregister it before destroying it!
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pHandler    The handler to free.  NULL if ignored.
+ */
+int pgmHandlerPhysicalExDestroy(PVMCC pVM, PPGMPHYSHANDLER pHandler)
+{
+    if (pHandler)
+    {
+        AssertPtr(pHandler);
+        AssertReturn(pHandler->Key == NIL_RTGCPHYS, VERR_WRONG_ORDER);
+
+        int rc = PGM_LOCK(pVM);
+        if (RT_SUCCESS(rc))
+        {
+            rc = pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator.freeNode(pHandler);
+            PGM_UNLOCK(pVM);
+        }
+        return rc;
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Deregister a physical page access handler.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      Start physical address.
+ */
+VMMDECL(int)  PGMHandlerPhysicalDeregister(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    AssertReturn(pVM->VMCC_CTX(pgm).s.pPhysHandlerTree, VERR_PGM_HANDLER_IPE_1);
+
+    /*
+     * Find the handler.
+     */
+    int rc = PGM_LOCK(pVM);
+    AssertRCReturn(rc, rc);
+
+    PPGMPHYSHANDLER pRemoved;
+    rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->remove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pRemoved);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pRemoved->Key == GCPhys);
+        LogFlow(("PGMHandlerPhysicalDeregister: Removing Range %RGp-%RGp %s\n",
+                 pRemoved->Key, pRemoved->KeyLast, R3STRING(pRemoved->pszDesc)));
+
+        /*
+         * Clear the page bits, notify the REM about this change and clear
+         * the cache.
+         */
+        pgmHandlerPhysicalResetRamFlags(pVM, pRemoved);
+        if (VM_IS_NEM_ENABLED(pVM))
+            pgmHandlerPhysicalDeregisterNotifyNEM(pVM, pRemoved);
+        pVM->pgm.s.idxLastPhysHandler = 0;
+
+        pRemoved->Key = NIL_RTGCPHYS;
+        rc = pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator.freeNode(pRemoved);
+
+        PGM_UNLOCK(pVM);
+        return rc;
+    }
+
+    PGM_UNLOCK(pVM);
+
+    if (rc == VERR_NOT_FOUND)
+    {
+        AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+    return rc;
+}
+
+
+/**
+ * Shared code with modify.
+ */
+static void pgmHandlerPhysicalDeregisterNotifyNEM(PVMCC pVM, PPGMPHYSHANDLER pCur)
+{
+#ifdef VBOX_WITH_NATIVE_NEM
+    PCPGMPHYSHANDLERTYPEINT pCurType    = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+    RTGCPHYS                GCPhysStart = pCur->Key;
+    RTGCPHYS                GCPhysLast  = pCur->KeyLast;
+
+    /*
+     * Page align the range.
+     *
+     * Since we've reset (recalculated) the physical handler state of all pages
+     * we can make use of the page states to figure out whether a page should be
+     * included in the REM notification or not.
+     */
+    if (   (pCur->Key           & GUEST_PAGE_OFFSET_MASK)
+        || ((pCur->KeyLast + 1) & GUEST_PAGE_OFFSET_MASK))
+    {
+        Assert(pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO);
+
+        if (GCPhysStart & GUEST_PAGE_OFFSET_MASK)
+        {
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhysStart);
+            if (    pPage
+                &&  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
+            {
+                RTGCPHYS GCPhys = (GCPhysStart + (GUEST_PAGE_SIZE - 1)) & X86_PTE_PAE_PG_MASK;
+                if (    GCPhys > GCPhysLast
+                    ||  GCPhys < GCPhysStart)
+                    return;
+                GCPhysStart = GCPhys;
+            }
+            else
+                GCPhysStart &= X86_PTE_PAE_PG_MASK;
+            Assert(!pPage || PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO); /* these are page aligned atm! */
+        }
+
+        if (GCPhysLast & GUEST_PAGE_OFFSET_MASK)
+        {
+            PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhysLast);
+            if (    pPage
+                &&  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
+            {
+                RTGCPHYS GCPhys = (GCPhysLast & X86_PTE_PAE_PG_MASK) - 1;
+                if (    GCPhys < GCPhysStart
+                    ||  GCPhys > GCPhysLast)
+                    return;
+                GCPhysLast = GCPhys;
+            }
+            else
+                GCPhysLast |= GUEST_PAGE_OFFSET_MASK;
+            Assert(!pPage || PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO); /* these are page aligned atm! */
+        }
+    }
+
+    /*
+     * Tell NEM.
+     */
+    PPGMRAMRANGE const pRam    = pgmPhysGetRange(pVM, GCPhysStart);
+    RTGCPHYS const     cb      = GCPhysLast - GCPhysStart + 1;
+    uint8_t            u2State = UINT8_MAX;
+    NEMHCNotifyHandlerPhysicalDeregister(pVM, pCurType->enmKind, GCPhysStart, cb,
+                                         pRam ? PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysStart) : NULL, &u2State);
+    if (u2State != UINT8_MAX && pRam)
+        pgmPhysSetNemStateForPages(&pRam->aPages[(GCPhysStart - pRam->GCPhys) >> GUEST_PAGE_SHIFT],
+                                   cb >> GUEST_PAGE_SHIFT, u2State);
+#else
+    RT_NOREF(pVM, pCur);
+#endif
+}
+
+
+/**
+ * pgmHandlerPhysicalResetRamFlags helper that checks for other handlers on
+ * edge pages.
+ */
+DECLINLINE(void) pgmHandlerPhysicalRecalcPageState(PVMCC pVM, RTGCPHYS GCPhys, bool fAbove, PPGMRAMRANGE *ppRamHint)
+{
+    /*
+     * Look for other handlers.
+     */
+    unsigned uState = PGM_PAGE_HNDL_PHYS_STATE_NONE;
+    for (;;)
+    {
+        PPGMPHYSHANDLER pCur;
+        int rc;
+        if (fAbove)
+            rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator,
+                                                                              GCPhys, &pCur);
+        else
+            rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookupMatchingOrBelow(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator,
+                                                                              GCPhys, &pCur);
+        if (rc == VERR_NOT_FOUND)
+            break;
+        AssertRCBreak(rc);
+        if (((fAbove ? pCur->Key : pCur->KeyLast) >> GUEST_PAGE_SHIFT) != (GCPhys >> GUEST_PAGE_SHIFT))
+            break;
+        PCPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+        uState = RT_MAX(uState, pCurType->uState);
+
+        /* next? */
+        RTGCPHYS GCPhysNext = fAbove
+                            ? pCur->KeyLast + 1
+                            : pCur->Key     - 1;
+        if ((GCPhysNext >> GUEST_PAGE_SHIFT) != (GCPhys >> GUEST_PAGE_SHIFT))
+            break;
+        GCPhys = GCPhysNext;
+    }
+
+    /*
+     * Update if we found something that is a higher priority state than the current.
+     * Note! The PGMPHYSHANDLER_F_NOT_IN_HM can be ignored here as it requires whole pages.
+     */
+    if (uState != PGM_PAGE_HNDL_PHYS_STATE_NONE)
+    {
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, ppRamHint);
+        if (   RT_SUCCESS(rc)
+            && PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) < uState)
+        {
+            /* This should normally not be necessary. */
+            PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(pPage, uState);
+            bool fFlushTLBs;
+            rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, false /*fFlushPTEs*/, &fFlushTLBs);
+            if (RT_SUCCESS(rc) && fFlushTLBs)
+                PGM_INVL_ALL_VCPU_TLBS(pVM);
+            else
+                AssertRC(rc);
+
+#ifdef VBOX_WITH_NATIVE_NEM
+            /* Tell NEM about the protection update. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
+                                               PGM_RAMRANGE_CALC_PAGE_R3PTR(*ppRamHint, GCPhys),
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+#endif
+        }
+        else
+            AssertRC(rc);
+    }
+}
+
+
+/**
+ * Resets an aliased page.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pPage           The page.
+ * @param   GCPhysPage      The page address in case it comes in handy.
+ * @param   pRam            The RAM range the page is associated with (for NEM
+ *                          notifications).
+ * @param   fDoAccounting   Whether to perform accounting.  (Only set during
+ *                          reset where pgmR3PhysRamReset doesn't have the
+ *                          handler structure handy.)
+ * @param   fFlushIemTlbs   Whether to perform IEM TLB flushing or not.  This
+ *                          can be cleared only if the caller does the flushing
+ *                          after calling this function.
+ */
+void pgmHandlerPhysicalResetAliasedPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhysPage, PPGMRAMRANGE pRam,
+                                        bool fDoAccounting, bool fFlushIemTlbs)
+{
+    Assert(   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
+           || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO);
+    Assert(PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_DISABLED);
+#ifdef VBOX_WITH_NATIVE_NEM
+    RTHCPHYS const HCPhysPrev = PGM_PAGE_GET_HCPHYS(pPage);
+#endif
+
+    /*
+     * Flush any shadow page table references *first*.
+     */
+    bool fFlushTLBs = false;
+    int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysPage, pPage, true /*fFlushPTEs*/, &fFlushTLBs);
+    AssertLogRelRCReturnVoid(rc);
+    HMFlushTlbOnAllVCpus(pVM);
+
+    /*
+     * Make it an MMIO/Zero page.
+     */
+    PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
+    PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_MMIO);
+    PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
+    PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
+    PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(pPage, PGM_PAGE_HNDL_PHYS_STATE_ALL);
+
+    /*
+     * Flush its TLB entry.
+     */
+    pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhysPage);
+    if (fFlushIemTlbs)
+        IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID);
+
+    /*
+     * Do accounting for pgmR3PhysRamReset.
+     */
+    if (fDoAccounting)
+    {
+        PPGMPHYSHANDLER pHandler;
+        rc = pgmHandlerPhysicalLookup(pVM, GCPhysPage, &pHandler);
+        if (RT_SUCCESS(rc))
+        {
+            Assert(pHandler->cAliasedPages > 0);
+            pHandler->cAliasedPages--;
+        }
+        else
+            AssertMsgFailed(("rc=%Rrc GCPhysPage=%RGp\n", rc, GCPhysPage));
+    }
+
+#ifdef VBOX_WITH_NATIVE_NEM
+    /*
+     * Tell NEM about the protection change.
+     */
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+        NEMHCNotifyPhysPageChanged(pVM, GCPhysPage, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg,
+                                   PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage),
+                                   NEM_PAGE_PROT_NONE, PGMPAGETYPE_MMIO, &u2State);
+        PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+    }
+#else
+    RT_NOREF(pRam);
+#endif
+}
+
+
+/**
+ * Resets ram range flags.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   pCur    The physical handler.
+ *
+ * @remark  We don't start messing with the shadow page tables, as we've
+ *          already got code in Trap0e which deals with out of sync handler
+ *          flags (originally conceived for global pages).
+ */
+static void pgmHandlerPhysicalResetRamFlags(PVMCC pVM, PPGMPHYSHANDLER pCur)
+{
+    /*
+     * Iterate the guest ram pages updating the state.
+     */
+    RTUINT          cPages   = pCur->cPages;
+    RTGCPHYS        GCPhys   = pCur->Key;
+    PPGMRAMRANGE    pRamHint = NULL;
+    for (;;)
+    {
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageWithHintEx(pVM, GCPhys, &pPage, &pRamHint);
+        if (RT_SUCCESS(rc))
+        {
+            /* Reset aliased MMIO pages to MMIO, since this aliasing is our business.
+               (We don't flip MMIO to RAM though, that's PGMPhys.cpp's job.)  */
+            bool fNemNotifiedAlready = false;
+            if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
+                || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO)
+            {
+                Assert(pCur->cAliasedPages > 0);
+                pgmHandlerPhysicalResetAliasedPage(pVM, pPage, GCPhys, pRamHint, false /*fDoAccounting*/, true /*fFlushIemTlbs*/);
+                pCur->cAliasedPages--;
+                fNemNotifiedAlready = true;
+            }
+#ifdef VBOX_STRICT
+            PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+            AssertMsg(pCurType && (pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO || PGM_PAGE_IS_MMIO(pPage)),
+                      ("%RGp %R[pgmpage]\n", GCPhys, pPage));
+#endif
+            PGM_PAGE_SET_HNDL_PHYS_STATE(pPage, PGM_PAGE_HNDL_PHYS_STATE_NONE, false);
+
+#ifdef VBOX_WITH_NATIVE_NEM
+            /* Tell NEM about the protection change. */
+            if (VM_IS_NEM_ENABLED(pVM) && !fNemNotifiedAlready)
+            {
+                uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
+                                               PGM_RAMRANGE_CALC_PAGE_R3PTR(pRamHint, GCPhys),
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+#endif
+            RT_NOREF(fNemNotifiedAlready);
+        }
+        else
+            AssertRC(rc);
+
+        /* next */
+        if (--cPages == 0)
+            break;
+        GCPhys += GUEST_PAGE_SIZE;
+    }
+
+    pCur->cAliasedPages = 0;
+    pCur->cTmpOffPages  = 0;
+
+    /*
+     * Check for partial start and end pages.
+     */
+    if (pCur->Key & GUEST_PAGE_OFFSET_MASK)
+        pgmHandlerPhysicalRecalcPageState(pVM, pCur->Key - 1, false /* fAbove */, &pRamHint);
+    if ((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) != GUEST_PAGE_OFFSET_MASK)
+        pgmHandlerPhysicalRecalcPageState(pVM, pCur->KeyLast + 1, true /* fAbove */, &pRamHint);
+}
+
+
+#if 0 /* unused */
+/**
+ * Modify a physical page access handler.
+ *
+ * Modification can only be done to the range it self, not the type or anything else.
+ *
+ * @returns VBox status code.
+ *          For all return codes other than VERR_PGM_HANDLER_NOT_FOUND and VINF_SUCCESS the range is deregistered
+ *          and a new registration must be performed!
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhysCurrent   Current location.
+ * @param   GCPhys          New location.
+ * @param   GCPhysLast      New last location.
+ */
+VMMDECL(int) PGMHandlerPhysicalModify(PVMCC pVM, RTGCPHYS GCPhysCurrent, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast)
+{
+    /*
+     * Remove it.
+     */
+    int rc;
+    PGM_LOCK_VOID(pVM);
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhysCurrent);
+    if (pCur)
+    {
+        /*
+         * Clear the ram flags. (We're gonna move or free it!)
+         */
+        pgmHandlerPhysicalResetRamFlags(pVM, pCur);
+        PCPGMPHYSHANDLERTYPEINT const pCurType     = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+        @todo pCurType validation
+        bool const                   fRestoreAsRAM = pCurType->pfnHandlerR3 /** @todo this isn't entirely correct. */
+                                                  && pCurType->enmKind != PGMPHYSHANDLERKIND_MMIO;
+
+        /*
+         * Validate the new range, modify and reinsert.
+         */
+        if (GCPhysLast >= GCPhys)
+        {
+            /*
+             * We require the range to be within registered ram.
+             * There is no apparent need to support ranges which cover more than one ram range.
+             */
+            PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
+            if (   pRam
+                && GCPhys <= pRam->GCPhysLast
+                && GCPhysLast >= pRam->GCPhys)
+            {
+                pCur->Core.Key      = GCPhys;
+                pCur->Core.KeyLast  = GCPhysLast;
+                pCur->cPages        = (GCPhysLast - (GCPhys & X86_PTE_PAE_PG_MASK) + 1) >> GUEST_PAGE_SHIFT;
+
+                if (RTAvlroGCPhysInsert(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, &pCur->Core))
+                {
+                    RTGCPHYS            const cb            = GCPhysLast - GCPhys + 1;
+                    PGMPHYSHANDLERKIND  const enmKind       = pCurType->enmKind;
+
+                    /*
+                     * Set ram flags, flush shadow PT entries and finally tell REM about this.
+                     */
+                    rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pCur, pRam, NULL, 0);
+
+                    /** @todo NEM: not sure we need this notification... */
+                    NEMHCNotifyHandlerPhysicalModify(pVM, enmKind, GCPhysCurrent, GCPhys, cb, fRestoreAsRAM);
+
+                    PGM_UNLOCK(pVM);
+
+                    PGM_INVL_ALL_VCPU_TLBS(pVM);
+                    Log(("PGMHandlerPhysicalModify: GCPhysCurrent=%RGp -> GCPhys=%RGp GCPhysLast=%RGp\n",
+                         GCPhysCurrent, GCPhys, GCPhysLast));
+                    return VINF_SUCCESS;
+                }
+
+                AssertMsgFailed(("Conflict! GCPhys=%RGp GCPhysLast=%RGp\n", GCPhys, GCPhysLast));
+                rc = VERR_PGM_HANDLER_PHYSICAL_CONFLICT;
+            }
+            else
+            {
+                AssertMsgFailed(("No RAM range for %RGp-%RGp\n", GCPhys, GCPhysLast));
+                rc = VERR_PGM_HANDLER_PHYSICAL_NO_RAM_RANGE;
+            }
+        }
+        else
+        {
+            AssertMsgFailed(("Invalid range %RGp-%RGp\n", GCPhys, GCPhysLast));
+            rc = VERR_INVALID_PARAMETER;
+        }
+
+        /*
+         * Invalid new location, flush the cache and free it.
+         * We've only gotta notify REM and free the memory.
+         */
+        if (VM_IS_NEM_ENABLED(pVM))
+            pgmHandlerPhysicalDeregisterNotifyNEM(pVM, pCur);
+        pVM->pgm.s.pLastPhysHandlerR0 = 0;
+        pVM->pgm.s.pLastPhysHandlerR3 = 0;
+        PGMHandlerPhysicalTypeRelease(pVM, pCur->hType);
+        MMHyperFree(pVM, pCur);
+    }
+    else
+    {
+        AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhysCurrent));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+#endif /* unused */
+
+
+/**
+ * Changes the user callback arguments associated with a physical access handler.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Start physical address of the handler.
+ * @param   uUser           User argument to the handlers.
+ */
+VMMDECL(int) PGMHandlerPhysicalChangeUserArg(PVMCC pVM, RTGCPHYS GCPhys, uint64_t uUser)
+{
+    /*
+     * Find the handler and make the change.
+     */
+    int rc = PGM_LOCK(pVM);
+    AssertRCReturn(rc, rc);
+
+    PPGMPHYSHANDLER pCur;
+    rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pCur->Key == GCPhys);
+        pCur->uUser = uUser;
+    }
+    else if (rc == VERR_NOT_FOUND)
+    {
+        AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+#if 0 /* unused */
+
+/**
+ * Splits a physical access handler in two.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Start physical address of the handler.
+ * @param   GCPhysSplit     The split address.
+ */
+VMMDECL(int) PGMHandlerPhysicalSplit(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysSplit)
+{
+    AssertReturn(GCPhys < GCPhysSplit, VERR_INVALID_PARAMETER);
+
+    /*
+     * Do the allocation without owning the lock.
+     */
+    PPGMPHYSHANDLER pNew;
+    int rc = MMHyperAlloc(pVM, sizeof(*pNew), 0, MM_TAG_PGM_HANDLERS, (void **)&pNew);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /*
+     * Get the handler.
+     */
+    PGM_LOCK_VOID(pVM);
+    PPGMPHYSHANDLER pCur = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys);
+    if (RT_LIKELY(pCur))
+    {
+        if (RT_LIKELY(GCPhysSplit <= pCur->Core.KeyLast))
+        {
+            /*
+             * Create new handler node for the 2nd half.
+             */
+            *pNew = *pCur;
+            pNew->Core.Key      = GCPhysSplit;
+            pNew->cPages        = (pNew->Core.KeyLast - (pNew->Core.Key & X86_PTE_PAE_PG_MASK) + GUEST_PAGE_SIZE) >> GUEST_PAGE_SHIFT;
+
+            pCur->Core.KeyLast  = GCPhysSplit - 1;
+            pCur->cPages        = (pCur->Core.KeyLast - (pCur->Core.Key & X86_PTE_PAE_PG_MASK) + GUEST_PAGE_SIZE) >> GUEST_PAGE_SHIFT;
+
+            if (RT_LIKELY(RTAvlroGCPhysInsert(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, &pNew->Core)))
+            {
+                LogFlow(("PGMHandlerPhysicalSplit: %RGp-%RGp and %RGp-%RGp\n",
+                         pCur->Core.Key, pCur->Core.KeyLast, pNew->Core.Key, pNew->Core.KeyLast));
+                PGM_UNLOCK(pVM);
+                return VINF_SUCCESS;
+            }
+            AssertMsgFailed(("whu?\n"));
+            rc = VERR_PGM_PHYS_HANDLER_IPE;
+        }
+        else
+        {
+            AssertMsgFailed(("outside range: %RGp-%RGp split %RGp\n", pCur->Core.Key, pCur->Core.KeyLast, GCPhysSplit));
+            rc = VERR_INVALID_PARAMETER;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+    PGM_UNLOCK(pVM);
+    MMHyperFree(pVM, pNew);
+    return rc;
+}
+
+
+/**
+ * Joins up two adjacent physical access handlers which has the same callbacks.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys1         Start physical address of the first handler.
+ * @param   GCPhys2         Start physical address of the second handler.
+ */
+VMMDECL(int) PGMHandlerPhysicalJoin(PVMCC pVM, RTGCPHYS GCPhys1, RTGCPHYS GCPhys2)
+{
+    /*
+     * Get the handlers.
+     */
+    int rc;
+    PGM_LOCK_VOID(pVM);
+    PPGMPHYSHANDLER pCur1 = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys1);
+    if (RT_LIKELY(pCur1))
+    {
+        PPGMPHYSHANDLER pCur2 = (PPGMPHYSHANDLER)RTAvlroGCPhysGet(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys2);
+        if (RT_LIKELY(pCur2))
+        {
+            /*
+             * Make sure that they are adjacent, and that they've got the same callbacks.
+             */
+            if (RT_LIKELY(pCur1->Core.KeyLast + 1 == pCur2->Core.Key))
+            {
+                if (RT_LIKELY(pCur1->hType == pCur2->hType))
+                {
+                    PPGMPHYSHANDLER pCur3 = (PPGMPHYSHANDLER)RTAvlroGCPhysRemove(&pVM->pgm.s.CTX_SUFF(pTrees)->PhysHandlers, GCPhys2);
+                    if (RT_LIKELY(pCur3 == pCur2))
+                    {
+                        pCur1->Core.KeyLast  = pCur2->Core.KeyLast;
+                        pCur1->cPages        = (pCur1->Core.KeyLast - (pCur1->Core.Key & X86_PTE_PAE_PG_MASK) + GUEST_PAGE_SIZE) >> GUEST_PAGE_SHIFT;
+                        LogFlow(("PGMHandlerPhysicalJoin: %RGp-%RGp %RGp-%RGp\n",
+                                 pCur1->Core.Key, pCur1->Core.KeyLast, pCur2->Core.Key, pCur2->Core.KeyLast));
+                        pVM->pgm.s.pLastPhysHandlerR0 = 0;
+                        pVM->pgm.s.pLastPhysHandlerR3 = 0;
+                        PGMHandlerPhysicalTypeRelease(pVM, pCur2->hType);
+                        MMHyperFree(pVM, pCur2);
+                        PGM_UNLOCK(pVM);
+                        return VINF_SUCCESS;
+                    }
+
+                    Assert(pCur3 == pCur2);
+                    rc = VERR_PGM_PHYS_HANDLER_IPE;
+                }
+                else
+                {
+                    AssertMsgFailed(("mismatching handlers\n"));
+                    rc = VERR_ACCESS_DENIED;
+                }
+            }
+            else
+            {
+                AssertMsgFailed(("not adjacent: %RGp-%RGp %RGp-%RGp\n",
+                                 pCur1->Core.Key, pCur1->Core.KeyLast, pCur2->Core.Key, pCur2->Core.KeyLast));
+                rc = VERR_INVALID_PARAMETER;
+            }
+        }
+        else
+        {
+            AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys2));
+            rc = VERR_PGM_HANDLER_NOT_FOUND;
+        }
+    }
+    else
+    {
+        AssertMsgFailed(("Didn't find range starting at %RGp\n", GCPhys1));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+    PGM_UNLOCK(pVM);
+    return rc;
+
+}
+
+#endif /* unused */
+
+/**
+ * Resets any modifications to individual pages in a physical page access
+ * handler region.
+ *
+ * This is used in pair with PGMHandlerPhysicalPageTempOff(),
+ * PGMHandlerPhysicalPageAliasMmio2() or PGMHandlerPhysicalPageAliasHC().
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The start address of the handler regions, i.e. what you
+ *                      passed to PGMR3HandlerPhysicalRegister(),
+ *                      PGMHandlerPhysicalRegisterEx() or
+ *                      PGMHandlerPhysicalModify().
+ */
+VMMDECL(int) PGMHandlerPhysicalReset(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    LogFlow(("PGMHandlerPhysicalReset GCPhys=%RGp\n", GCPhys));
+    int rc = PGM_LOCK(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Find the handler.
+     */
+    PPGMPHYSHANDLER pCur;
+    rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pCur->Key == GCPhys);
+
+        /*
+         * Validate kind.
+         */
+        PCPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+        switch (pCurType->enmKind)
+        {
+            case PGMPHYSHANDLERKIND_WRITE:
+            case PGMPHYSHANDLERKIND_ALL:
+            case PGMPHYSHANDLERKIND_MMIO: /* NOTE: Only use when clearing MMIO ranges with aliased MMIO2 pages! */
+            {
+                STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysHandlerReset)); /** @todo move out of switch */
+                PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
+                Assert(pRam);
+                Assert(pRam->GCPhys     <= pCur->Key);
+                Assert(pRam->GCPhysLast >= pCur->KeyLast);
+
+                if (pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO)
+                {
+                    /*
+                     * Reset all the PGMPAGETYPE_MMIO2_ALIAS_MMIO pages first and that's it.
+                     * This could probably be optimized a bit wrt to flushing, but I'm too lazy
+                     * to do that now...
+                     */
+                    if (pCur->cAliasedPages)
+                    {
+                        PPGMPAGE    pPage        = &pRam->aPages[(pCur->Key - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
+                        RTGCPHYS    GCPhysPage   = pCur->Key;
+                        uint32_t    cLeft        = pCur->cPages;
+                        bool        fFlushIemTlb = false;
+                        while (cLeft-- > 0)
+                        {
+                            if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
+                                || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO)
+                            {
+                                fFlushIemTlb |= PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO;
+                                Assert(pCur->cAliasedPages > 0);
+                                pgmHandlerPhysicalResetAliasedPage(pVM, pPage, GCPhysPage, pRam,
+                                                                   false /*fDoAccounting*/, false /*fFlushIemTlbs*/);
+                                --pCur->cAliasedPages;
+#ifndef VBOX_STRICT
+                                if (pCur->cAliasedPages == 0)
+                                    break;
+#endif
+                            }
+                            Assert(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO);
+                            GCPhysPage += GUEST_PAGE_SIZE;
+                            pPage++;
+                        }
+                        Assert(pCur->cAliasedPages == 0);
+
+                        /*
+                         * Flush IEM TLBs in case they contain any references to aliased pages.
+                         * This is only necessary for MMIO2 aliases.
+                         */
+                        if (fFlushIemTlb)
+                            IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID);
+                    }
+                }
+                else if (pCur->cTmpOffPages > 0)
+                {
+                    /*
+                     * Set the flags and flush shadow PT entries.
+                     */
+                    rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pCur, pRam, NULL /*pvBitmap*/, 0 /*offBitmap*/);
+                }
+
+                pCur->cAliasedPages = 0;
+                pCur->cTmpOffPages  = 0;
+
+                rc = VINF_SUCCESS;
+                break;
+            }
+
+            /*
+             * Invalid.
+             */
+            default:
+                AssertMsgFailed(("Invalid type %d/%#x! Corruption!\n",  pCurType->enmKind, pCur->hType));
+                rc = VERR_PGM_PHYS_HANDLER_IPE;
+                break;
+        }
+    }
+    else if (rc == VERR_NOT_FOUND)
+    {
+        AssertMsgFailed(("Didn't find MMIO Range starting at %#x\n", GCPhys));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+
+/**
+ * Special version of PGMHandlerPhysicalReset used by MMIO2 w/ dirty page
+ * tracking.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The start address of the handler region.
+ * @param   pvBitmap    Dirty bitmap. Caller has cleared this already, only
+ *                      dirty bits will be set. Caller also made sure it's big
+ *                      enough.
+ * @param   offBitmap   Dirty bitmap offset.
+ * @remarks Caller must own the PGM critical section.
+ */
+DECLHIDDEN(int) pgmHandlerPhysicalResetMmio2WithBitmap(PVMCC pVM, RTGCPHYS GCPhys, void *pvBitmap, uint32_t offBitmap)
+{
+    LogFlow(("pgmHandlerPhysicalResetMmio2WithBitmap GCPhys=%RGp\n", GCPhys));
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Find the handler.
+     */
+    PPGMPHYSHANDLER pCur;
+    int rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pCur->Key == GCPhys);
+
+        /*
+         * Validate kind.
+         */
+        PCPGMPHYSHANDLERTYPEINT pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+        if (   pCurType
+            && pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE)
+        {
+            STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysHandlerReset));
+
+            PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
+            Assert(pRam);
+            Assert(pRam->GCPhys     <= pCur->Key);
+            Assert(pRam->GCPhysLast >= pCur->KeyLast);
+
+            /*
+             * Set the flags and flush shadow PT entries.
+             */
+            if (pCur->cTmpOffPages > 0)
+            {
+                rc = pgmHandlerPhysicalSetRamFlagsAndFlushShadowPTs(pVM, pCur, pRam, pvBitmap, offBitmap);
+                pCur->cTmpOffPages  = 0;
+            }
+            else
+                rc = VINF_SUCCESS;
+        }
+        else
+        {
+            AssertFailed();
+            rc = VERR_WRONG_TYPE;
+        }
+    }
+    else if (rc == VERR_NOT_FOUND)
+    {
+        AssertMsgFailed(("Didn't find MMIO Range starting at %#x\n", GCPhys));
+        rc = VERR_PGM_HANDLER_NOT_FOUND;
+    }
+
+    return rc;
+}
+
+
+/**
+ * Temporarily turns off the access monitoring of a page within a monitored
+ * physical write/all page access handler region.
+ *
+ * Use this when no further \#PFs are required for that page. Be aware that
+ * a page directory sync might reset the flags, and turn on access monitoring
+ * for the page.
+ *
+ * The caller must do required page table modifications.
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   GCPhys              The start address of the access handler. This
+ *                              must be a fully page aligned range or we risk
+ *                              messing up other handlers installed for the
+ *                              start and end pages.
+ * @param   GCPhysPage          The physical address of the page to turn off
+ *                              access monitoring for.
+ */
+VMMDECL(int)  PGMHandlerPhysicalPageTempOff(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysPage)
+{
+    LogFlow(("PGMHandlerPhysicalPageTempOff GCPhysPage=%RGp\n", GCPhysPage));
+    int rc = PGM_LOCK(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Validate the range.
+     */
+    PPGMPHYSHANDLER pCur;
+    rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pCur->Key == GCPhys);
+        if (RT_LIKELY(   GCPhysPage >= pCur->Key
+                      && GCPhysPage <= pCur->KeyLast))
+        {
+            Assert(!(pCur->Key & GUEST_PAGE_OFFSET_MASK));
+            Assert((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK);
+
+            PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE(pVM, pCur);
+            AssertReturnStmt(   pCurType
+                             && (   pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE
+                                 || pCurType->enmKind == PGMPHYSHANDLERKIND_ALL),
+                             PGM_UNLOCK(pVM), VERR_ACCESS_DENIED);
+
+            /*
+             * Change the page status.
+             */
+            PPGMPAGE     pPage;
+            PPGMRAMRANGE pRam;
+            rc = pgmPhysGetPageAndRangeEx(pVM, GCPhysPage, &pPage, &pRam);
+            AssertReturnStmt(RT_SUCCESS_NP(rc), PGM_UNLOCK(pVM), rc);
+            if (PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
+            {
+                PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(pPage, PGM_PAGE_HNDL_PHYS_STATE_DISABLED);
+                pCur->cTmpOffPages++;
+
+#ifdef VBOX_WITH_NATIVE_NEM
+                /* Tell NEM about the protection change (VGA is using this to track dirty pages). */
+                if (VM_IS_NEM_ENABLED(pVM))
+                {
+                    uint8_t     u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                    PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+                    NEMHCNotifyPhysPageProtChanged(pVM, GCPhysPage, PGM_PAGE_GET_HCPHYS(pPage),
+                                                   PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage),
+                                                   pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+                    PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+                }
+#endif
+            }
+            PGM_UNLOCK(pVM);
+            return VINF_SUCCESS;
+        }
+        PGM_UNLOCK(pVM);
+        AssertMsgFailed(("The page %#x is outside the range %#x-%#x\n", GCPhysPage, pCur->Key, pCur->KeyLast));
+        return VERR_INVALID_PARAMETER;
+    }
+    PGM_UNLOCK(pVM);
+
+    if (rc == VERR_NOT_FOUND)
+    {
+        AssertMsgFailed(("Specified physical handler start address %#x is invalid.\n", GCPhys));
+        return VERR_PGM_HANDLER_NOT_FOUND;
+    }
+    return rc;
+}
+
+
+/**
+ * Resolves an MMIO2 page.
+ *
+ * Caller as taken the PGM lock.
+ *
+ * @returns Pointer to the page if valid, NULL otherwise
+ * @param   pVM             The cross context VM structure.
+ * @param   pDevIns         The device owning it.
+ * @param   hMmio2          The MMIO2 region.
+ * @param   offMmio2Page    The offset into the region.
+ */
+static PPGMPAGE pgmPhysResolveMmio2PageLocked(PVMCC pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS offMmio2Page)
+{
+    /* Only works if the handle is in the handle table! */
+    AssertReturn(hMmio2 != 0, NULL);
+    hMmio2--;
+
+    /* Must check the first one for PGMREGMMIO2RANGE_F_FIRST_CHUNK. */
+    AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3), NULL);
+    PPGMREGMMIO2RANGE pCur = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[hMmio2];
+    AssertReturn(pCur, NULL);
+    AssertReturn(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, NULL);
+
+    /* Loop thru the sub-ranges till we find the one covering offMmio2. */
+    for (;;)
+    {
+#ifdef IN_RING3
+        AssertReturn(pCur->pDevInsR3 == pDevIns, NULL);
+#else
+        AssertReturn(pCur->pDevInsR3 == pDevIns->pDevInsForR3, NULL);
+#endif
+
+        /* Does it match the offset? */
+        if (offMmio2Page < pCur->cbReal)
+            return &pCur->RamRange.aPages[offMmio2Page >> GUEST_PAGE_SHIFT];
+
+        /* Advance if we can. */
+        AssertReturn(!(pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK), NULL);
+        offMmio2Page -= pCur->cbReal;
+        hMmio2++;
+        AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3), NULL);
+        pCur = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[hMmio2];
+        AssertReturn(pCur, NULL);
+    }
+}
+
+
+/**
+ * Replaces an MMIO page with an MMIO2 page.
+ *
+ * This is a worker for IOMMMIOMapMMIO2Page that works in a similar way to
+ * PGMHandlerPhysicalPageTempOff but for an MMIO page. Since an MMIO page has no
+ * backing, the caller must provide a replacement page. For various reasons the
+ * replacement page must be an MMIO2 page.
+ *
+ * The caller must do required page table modifications. You can get away
+ * without making any modifications since it's an MMIO page, the cost is an extra
+ * \#PF which will the resync the page.
+ *
+ * Call PGMHandlerPhysicalReset() to restore the MMIO page.
+ *
+ * The caller may still get handler callback even after this call and must be
+ * able to deal correctly with such calls. The reason for these callbacks are
+ * either that we're executing in the recompiler (which doesn't know about this
+ * arrangement) or that we've been restored from saved state (where we won't
+ * save the change).
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   GCPhys              The start address of the access handler. This
+ *                              must be a fully page aligned range or we risk
+ *                              messing up other handlers installed for the
+ *                              start and end pages.
+ * @param   GCPhysPage          The physical address of the page to turn off
+ *                              access monitoring for and replace with the MMIO2
+ *                              page.
+ * @param   pDevIns             The device instance owning @a hMmio2.
+ * @param   hMmio2              Handle to the MMIO2 region containing the page
+ *                              to remap in the the MMIO page at @a GCPhys.
+ * @param   offMmio2PageRemap   The offset into @a hMmio2 of the MMIO2 page that
+ *                              should serve as backing memory.
+ *
+ * @remark  May cause a page pool flush if used on a page that is already
+ *          aliased.
+ *
+ * @note    This trick does only work reliably if the two pages are never ever
+ *          mapped in the same page table. If they are the page pool code will
+ *          be confused should either of them be flushed. See the special case
+ *          of zero page aliasing mentioned in #3170.
+ *
+ */
+VMMDECL(int)  PGMHandlerPhysicalPageAliasMmio2(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysPage,
+                                               PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS offMmio2PageRemap)
+{
+#ifdef VBOX_WITH_PGM_NEM_MODE
+    AssertReturn(!VM_IS_NEM_ENABLED(pVM) || !pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
+#endif
+    int rc = PGM_LOCK(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Resolve the MMIO2 reference.
+     */
+    PPGMPAGE pPageRemap = pgmPhysResolveMmio2PageLocked(pVM, pDevIns, hMmio2, offMmio2PageRemap);
+    if (RT_LIKELY(pPageRemap))
+        AssertMsgReturnStmt(PGM_PAGE_GET_TYPE(pPageRemap) == PGMPAGETYPE_MMIO2,
+                            ("hMmio2=%RU64 offMmio2PageRemap=%RGp %R[pgmpage]\n", hMmio2, offMmio2PageRemap, pPageRemap),
+                            PGM_UNLOCK(pVM), VERR_PGM_PHYS_NOT_MMIO2);
+    else
+    {
+        PGM_UNLOCK(pVM);
+        return VERR_OUT_OF_RANGE;
+    }
+
+    /*
+     * Lookup and validate the range.
+     */
+    PPGMPHYSHANDLER pCur;
+    rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pCur->Key == GCPhys);
+        if (RT_LIKELY(   GCPhysPage >= pCur->Key
+                      && GCPhysPage <= pCur->KeyLast))
+        {
+            PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+            AssertReturnStmt(pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO, PGM_UNLOCK(pVM), VERR_ACCESS_DENIED);
+            AssertReturnStmt(!(pCur->Key & GUEST_PAGE_OFFSET_MASK), PGM_UNLOCK(pVM), VERR_INVALID_PARAMETER);
+            AssertReturnStmt((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK,
+                             PGM_UNLOCK(pVM), VERR_INVALID_PARAMETER);
+
+            /*
+             * Validate the page.
+             */
+            PPGMPAGE     pPage;
+            PPGMRAMRANGE pRam;
+            rc = pgmPhysGetPageAndRangeEx(pVM, GCPhysPage, &pPage, &pRam);
+            AssertReturnStmt(RT_SUCCESS_NP(rc), PGM_UNLOCK(pVM), rc);
+            if (PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO)
+            {
+                AssertMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO,
+                                ("GCPhysPage=%RGp %R[pgmpage]\n", GCPhysPage, pPage),
+                                VERR_PGM_PHYS_NOT_MMIO2);
+                if (PGM_PAGE_GET_HCPHYS(pPage) == PGM_PAGE_GET_HCPHYS(pPageRemap))
+                {
+                    PGM_UNLOCK(pVM);
+                    return VINF_PGM_HANDLER_ALREADY_ALIASED;
+                }
+
+                /*
+                 * The page is already mapped as some other page, reset it
+                 * to an MMIO/ZERO page before doing the new mapping.
+                 */
+                Log(("PGMHandlerPhysicalPageAliasMmio2: GCPhysPage=%RGp (%R[pgmpage]; %RHp -> %RHp\n",
+                     GCPhysPage, pPage, PGM_PAGE_GET_HCPHYS(pPage), PGM_PAGE_GET_HCPHYS(pPageRemap)));
+                pgmHandlerPhysicalResetAliasedPage(pVM, pPage, GCPhysPage, pRam,
+                                                   false /*fDoAccounting*/, false /*fFlushIemTlbs*/);
+                pCur->cAliasedPages--;
+
+                /* Since this may be present in the TLB and now be wrong, invalid
+                   the guest physical address part of the IEM TLBs.  Note, we do
+                   this here as we will not invalid */
+                IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID);
+            }
+            Assert(PGM_PAGE_IS_ZERO(pPage));
+
+            /*
+             * Do the actual remapping here.
+             * This page now serves as an alias for the backing memory specified.
+             */
+            LogFlow(("PGMHandlerPhysicalPageAliasMmio2: %RGp (%R[pgmpage]) alias for %RU64/%RGp (%R[pgmpage])\n",
+                     GCPhysPage, pPage, hMmio2, offMmio2PageRemap, pPageRemap ));
+            PGM_PAGE_SET_HCPHYS(pVM, pPage, PGM_PAGE_GET_HCPHYS(pPageRemap));
+            PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_MMIO2_ALIAS_MMIO);
+            PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+            PGM_PAGE_SET_PAGEID(pVM, pPage, PGM_PAGE_GET_PAGEID(pPageRemap));
+            PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(pPage, PGM_PAGE_HNDL_PHYS_STATE_DISABLED);
+            pCur->cAliasedPages++;
+            Assert(pCur->cAliasedPages <= pCur->cPages);
+
+            /*
+             * Flush its TLB entry.
+             *
+             * Not calling IEMTlbInvalidateAllPhysicalAllCpus here to conserve
+             * all the other IEM TLB entires.  When this one is kicked out and
+             * reloaded, it will be using the MMIO2 alias, but till then we'll
+             * continue doing MMIO.
+             */
+            pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhysPage);
+            /** @todo Do some preformance checks of calling
+             *        IEMTlbInvalidateAllPhysicalAllCpus when in IEM mode, to see if it
+             *        actually makes sense or not.  Screen updates are typically massive
+             *        and important when this kind of aliasing is used, so it may pay of... */
+
+#ifdef VBOX_WITH_NATIVE_NEM
+            /* Tell NEM about the backing and protection change. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                NEMHCNotifyPhysPageChanged(pVM, GCPhysPage, pVM->pgm.s.HCPhysZeroPg, PGM_PAGE_GET_HCPHYS(pPage),
+                                           PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage),
+                                           pgmPhysPageCalcNemProtection(pPage, PGMPAGETYPE_MMIO2_ALIAS_MMIO),
+                                           PGMPAGETYPE_MMIO2_ALIAS_MMIO, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+#endif
+            LogFlow(("PGMHandlerPhysicalPageAliasMmio2: => %R[pgmpage]\n", pPage));
+            PGM_UNLOCK(pVM);
+            return VINF_SUCCESS;
+        }
+
+        PGM_UNLOCK(pVM);
+        AssertMsgFailed(("The page %#x is outside the range %#x-%#x\n", GCPhysPage, pCur->Key, pCur->KeyLast));
+        return VERR_INVALID_PARAMETER;
+    }
+
+    PGM_UNLOCK(pVM);
+    if (rc == VERR_NOT_FOUND)
+    {
+        AssertMsgFailed(("Specified physical handler start address %#x is invalid.\n", GCPhys));
+        return VERR_PGM_HANDLER_NOT_FOUND;
+    }
+    return rc;
+}
+
+
+/**
+ * Replaces an MMIO page with an arbitrary HC page in the shadow page tables.
+ *
+ * This differs from PGMHandlerPhysicalPageAliasMmio2 in that the page doesn't
+ * need to be a known MMIO2 page and that only shadow paging may access the
+ * page. The latter distinction is important because the only use for this
+ * feature is for mapping the special APIC access page that VT-x uses to detect
+ * APIC MMIO operations, the page is shared between all guest CPUs and actually
+ * not written to. At least at the moment.
+ *
+ * The caller must do required page table modifications. You can get away
+ * without making any modifications since it's an MMIO page, the cost is an extra
+ * \#PF which will the resync the page.
+ *
+ * Call PGMHandlerPhysicalReset() to restore the MMIO page.
+ *
+ *
+ * @returns VBox status code.
+ * @param   pVM                 The cross context VM structure.
+ * @param   GCPhys              The start address of the access handler. This
+ *                              must be a fully page aligned range or we risk
+ *                              messing up other handlers installed for the
+ *                              start and end pages.
+ * @param   GCPhysPage          The physical address of the page to turn off
+ *                              access monitoring for.
+ * @param   HCPhysPageRemap     The physical address of the HC page that
+ *                              serves as backing memory.
+ *
+ * @remark  May cause a page pool flush if used on a page that is already
+ *          aliased.
+ */
+VMMDECL(int)  PGMHandlerPhysicalPageAliasHC(PVMCC pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysPage, RTHCPHYS HCPhysPageRemap)
+{
+///    Assert(!IOMIsLockOwner(pVM)); /* We mustn't own any other locks when calling this */
+#ifdef VBOX_WITH_PGM_NEM_MODE
+    AssertReturn(!VM_IS_NEM_ENABLED(pVM) || !pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
+#endif
+    int rc = PGM_LOCK(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Lookup and validate the range.
+     */
+    PPGMPHYSHANDLER pCur;
+    rc = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, GCPhys, &pCur);
+    if (RT_SUCCESS(rc))
+    {
+        Assert(pCur->Key == GCPhys);
+        if (RT_LIKELY(    GCPhysPage >= pCur->Key
+                      &&  GCPhysPage <= pCur->KeyLast))
+        {
+            PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+            AssertReturnStmt(pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO, PGM_UNLOCK(pVM), VERR_ACCESS_DENIED);
+            AssertReturnStmt(!(pCur->Key & GUEST_PAGE_OFFSET_MASK), PGM_UNLOCK(pVM), VERR_INVALID_PARAMETER);
+            AssertReturnStmt((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK,
+                             PGM_UNLOCK(pVM), VERR_INVALID_PARAMETER);
+
+            /*
+             * Get and validate the pages.
+             */
+            PPGMPAGE pPage;
+            rc = pgmPhysGetPageEx(pVM, GCPhysPage, &pPage);
+            AssertReturnStmt(RT_SUCCESS_NP(rc), PGM_UNLOCK(pVM), rc);
+            if (PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_MMIO)
+            {
+                PGM_UNLOCK(pVM);
+                AssertMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO,
+                                ("GCPhysPage=%RGp %R[pgmpage]\n", GCPhysPage, pPage),
+                                VERR_PGM_PHYS_NOT_MMIO2);
+                return VINF_PGM_HANDLER_ALREADY_ALIASED;
+            }
+            Assert(PGM_PAGE_IS_ZERO(pPage));
+
+            /*
+             * Do the actual remapping here.
+             * This page now serves as an alias for the backing memory
+             * specified as far as shadow paging is concerned.
+             */
+            LogFlow(("PGMHandlerPhysicalPageAliasHC: %RGp (%R[pgmpage]) alias for %RHp\n",
+                     GCPhysPage, pPage, HCPhysPageRemap));
+            PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhysPageRemap);
+            PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_SPECIAL_ALIAS_MMIO);
+            PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+            PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
+            PGM_PAGE_SET_HNDL_PHYS_STATE_ONLY(pPage, PGM_PAGE_HNDL_PHYS_STATE_DISABLED);
+            pCur->cAliasedPages++;
+            Assert(pCur->cAliasedPages <= pCur->cPages);
+
+            /*
+             * Flush its TLB entry.
+             *
+             * Not calling IEMTlbInvalidateAllPhysicalAllCpus here as special
+             * aliased MMIO pages are handled like MMIO by the IEM TLB.
+             */
+            pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhysPage);
+
+#ifdef VBOX_WITH_NATIVE_NEM
+            /* Tell NEM about the backing and protection change. */
+            if (VM_IS_NEM_ENABLED(pVM))
+            {
+                PPGMRAMRANGE pRam    = pgmPhysGetRange(pVM, GCPhysPage);
+                uint8_t      u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+                NEMHCNotifyPhysPageChanged(pVM, GCPhysPage, pVM->pgm.s.HCPhysZeroPg, PGM_PAGE_GET_HCPHYS(pPage),
+                                           PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage),
+                                           pgmPhysPageCalcNemProtection(pPage, PGMPAGETYPE_SPECIAL_ALIAS_MMIO),
+                                           PGMPAGETYPE_SPECIAL_ALIAS_MMIO, &u2State);
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            }
+#endif
+            LogFlow(("PGMHandlerPhysicalPageAliasHC: => %R[pgmpage]\n", pPage));
+            PGM_UNLOCK(pVM);
+            return VINF_SUCCESS;
+        }
+        PGM_UNLOCK(pVM);
+        AssertMsgFailed(("The page %#x is outside the range %#x-%#x\n", GCPhysPage, pCur->Key, pCur->KeyLast));
+        return VERR_INVALID_PARAMETER;
+    }
+    PGM_UNLOCK(pVM);
+
+    if (rc == VERR_NOT_FOUND)
+    {
+        AssertMsgFailed(("Specified physical handler start address %#x is invalid.\n", GCPhys));
+        return VERR_PGM_HANDLER_NOT_FOUND;
+    }
+    return rc;
+}
+
+
+/**
+ * Checks if a physical range is handled
+ *
+ * @returns boolean
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      Start physical address earlier passed to PGMR3HandlerPhysicalRegister().
+ * @remarks Caller must take the PGM lock...
+ * @thread  EMT.
+ */
+VMMDECL(bool) PGMHandlerPhysicalIsRegistered(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    /*
+     * Find the handler.
+     */
+    PGM_LOCK_VOID(pVM);
+    PPGMPHYSHANDLER pCur;
+    int rc = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur);
+    if (RT_SUCCESS(rc))
+    {
+#ifdef VBOX_STRICT
+        Assert(GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast);
+        PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+        Assert(   pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE
+               || pCurType->enmKind == PGMPHYSHANDLERKIND_ALL
+               || pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO);
+#endif
+        PGM_UNLOCK(pVM);
+        return true;
+    }
+    PGM_UNLOCK(pVM);
+    return false;
+}
+
+
+/**
+ * Checks if it's an disabled all access handler or write access handler at the
+ * given address.
+ *
+ * @returns true if it's an all access handler, false if it's a write access
+ *          handler.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The address of the page with a disabled handler.
+ *
+ * @remarks The caller, PGMR3PhysTlbGCPhys2Ptr, must hold the PGM lock.
+ */
+bool pgmHandlerPhysicalIsAll(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PGM_LOCK_VOID(pVM);
+    PPGMPHYSHANDLER pCur;
+    int rc = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur);
+    AssertRCReturnStmt(rc, PGM_UNLOCK(pVM), true);
+
+    /* Only whole pages can be disabled. */
+    Assert(   pCur->Key     <= (GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK)
+           && pCur->KeyLast >= (GCPhys | GUEST_PAGE_OFFSET_MASK));
+
+    PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+    Assert(   pCurType->enmKind == PGMPHYSHANDLERKIND_WRITE
+           || pCurType->enmKind == PGMPHYSHANDLERKIND_ALL
+           || pCurType->enmKind == PGMPHYSHANDLERKIND_MMIO); /* sanity */
+    bool const fRet = pCurType->enmKind != PGMPHYSHANDLERKIND_WRITE;
+    PGM_UNLOCK(pVM);
+    return fRet;
+}
+
+#ifdef VBOX_STRICT
+
+/**
+ * State structure used by the PGMAssertHandlerAndFlagsInSync() function
+ * and its AVL enumerators.
+ */
+typedef struct PGMAHAFIS
+{
+    /** The current physical address. */
+    RTGCPHYS    GCPhys;
+    /** Number of errors. */
+    unsigned    cErrors;
+    /** Pointer to the VM. */
+    PVM         pVM;
+} PGMAHAFIS, *PPGMAHAFIS;
+
+
+/**
+ * Asserts that the handlers+guest-page-tables == ramrange-flags and
+ * that the physical addresses associated with virtual handlers are correct.
+ *
+ * @returns Number of mismatches.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(unsigned) PGMAssertHandlerAndFlagsInSync(PVMCC pVM)
+{
+    PPGM        pPGM = &pVM->pgm.s;
+    PGMAHAFIS   State;
+    State.GCPhys  = 0;
+    State.cErrors = 0;
+    State.pVM     = pVM;
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Check the RAM flags against the handlers.
+     */
+    PPGMPHYSHANDLERTREE const pPhysHandlerTree = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree;
+    for (PPGMRAMRANGE pRam = pPGM->CTX_SUFF(pRamRangesX); pRam; pRam = pRam->CTX_SUFF(pNext))
+    {
+        const uint32_t cPages = pRam->cb >> GUEST_PAGE_SHIFT;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            PGMPAGE const *pPage = &pRam->aPages[iPage];
+            if (PGM_PAGE_HAS_ANY_HANDLERS(pPage))
+            {
+                State.GCPhys = pRam->GCPhys + (iPage << GUEST_PAGE_SHIFT);
+
+                /*
+                 * Physical first - calculate the state based on the handlers
+                 *                  active on the page, then compare.
+                 */
+                if (PGM_PAGE_HAS_ANY_PHYSICAL_HANDLERS(pPage))
+                {
+                    /* the first */
+                    PPGMPHYSHANDLER pPhys;
+                    int rc = pPhysHandlerTree->lookup(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator, State.GCPhys, &pPhys);
+                    if (rc == VERR_NOT_FOUND)
+                    {
+                        rc = pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator,
+                                                                     State.GCPhys, &pPhys);
+                        if (RT_SUCCESS(rc))
+                        {
+                            Assert(pPhys->Key >= State.GCPhys);
+                            if (pPhys->Key > (State.GCPhys + GUEST_PAGE_SIZE - 1))
+                                pPhys = NULL;
+                        }
+                        else
+                            AssertLogRelMsgReturn(rc == VERR_NOT_FOUND, ("rc=%Rrc GCPhys=%RGp\n", rc, State.GCPhys), 999);
+                    }
+                    else
+                        AssertLogRelMsgReturn(RT_SUCCESS(rc), ("rc=%Rrc GCPhys=%RGp\n", rc, State.GCPhys), 999);
+
+                    if (pPhys)
+                    {
+                        PCPGMPHYSHANDLERTYPEINT pPhysType = pgmHandlerPhysicalTypeHandleToPtr(pVM, pPhys->hType);
+                        unsigned   uState   = pPhysType->uState;
+                        bool const fNotInHm = pPhysType->fNotInHm; /* whole pages, so no need to accumulate sub-page configs. */
+
+                        /* more? */
+                        while (pPhys->KeyLast < (State.GCPhys | GUEST_PAGE_OFFSET_MASK))
+                        {
+                            PPGMPHYSHANDLER pPhys2;
+                            rc = pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator,
+                                                                         pPhys->KeyLast + 1, &pPhys2);
+                            if (rc == VERR_NOT_FOUND)
+                                break;
+                            AssertLogRelMsgReturn(RT_SUCCESS(rc), ("rc=%Rrc KeyLast+1=%RGp\n", rc, pPhys->KeyLast + 1), 999);
+                            if (pPhys2->Key > (State.GCPhys | GUEST_PAGE_OFFSET_MASK))
+                                break;
+                            PCPGMPHYSHANDLERTYPEINT pPhysType2 = pgmHandlerPhysicalTypeHandleToPtr(pVM, pPhys2->hType);
+                            uState = RT_MAX(uState, pPhysType2->uState);
+                            pPhys = pPhys2;
+                        }
+
+                        /* compare.*/
+                        if (    PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != uState
+                            &&  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_DISABLED)
+                        {
+                            AssertMsgFailed(("ram range vs phys handler flags mismatch. GCPhys=%RGp state=%d expected=%d %s\n",
+                                             State.GCPhys, PGM_PAGE_GET_HNDL_PHYS_STATE(pPage), uState, pPhysType->pszDesc));
+                            State.cErrors++;
+                        }
+                        AssertMsgStmt(PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage) == fNotInHm,
+                                      ("ram range vs phys handler flags mismatch. GCPhys=%RGp fNotInHm=%d, %d %s\n",
+                                       State.GCPhys, PGM_PAGE_IS_HNDL_PHYS_NOT_IN_HM(pPage), fNotInHm, pPhysType->pszDesc),
+                                      State.cErrors++);
+                    }
+                    else
+                    {
+                        AssertMsgFailed(("ram range vs phys handler mismatch. no handler for GCPhys=%RGp\n", State.GCPhys));
+                        State.cErrors++;
+                    }
+                }
+            }
+        } /* foreach page in ram range. */
+    } /* foreach ram range. */
+
+    /*
+     * Do the reverse check for physical handlers.
+     */
+    /** @todo */
+
+    return State.cErrors;
+}
+
+#endif /* VBOX_STRICT */
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllPhys.cpp b/src/VBox/VMM/VMMAll/PGMAllPhys.cpp
new file mode 100644
index 00000000..457f7de0
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllPhys.cpp
@@ -0,0 +1,4151 @@
+/* $Id: PGMAllPhys.cpp $ */
+/** @file
+ * PGM - Page Manager and Monitor, Physical Memory Addressing.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_PHYS
+#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/iem.h>
+#include <VBox/vmm/iom.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/nem.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include "PGMInline.h"
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <VBox/log.h>
+#ifdef IN_RING3
+# include <iprt/thread.h>
+#endif
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+/** Enable the physical TLB. */
+#define PGM_WITH_PHYS_TLB
+
+/** @def PGM_HANDLER_PHYS_IS_VALID_STATUS
+ * Checks if valid physical access handler return code (normal handler, not PF).
+ *
+ * Checks if the given strict status code is one of the expected ones for a
+ * physical access handler in the current context.
+ *
+ * @returns true or false.
+ * @param   a_rcStrict      The status code.
+ * @param   a_fWrite        Whether it is a write or read being serviced.
+ *
+ * @remarks We wish to keep the list of statuses here as short as possible.
+ *          When changing, please make sure to update the PGMPhysRead,
+ *          PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
+ */
+#ifdef IN_RING3
+# define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
+    (   (a_rcStrict) == VINF_SUCCESS \
+     || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
+#elif defined(IN_RING0)
+#define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
+    (   (a_rcStrict) == VINF_SUCCESS \
+     || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT \
+     \
+     || (a_rcStrict) == ((a_fWrite) ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ) \
+     || (a_rcStrict) == VINF_IOM_R3_MMIO_READ_WRITE \
+     || ((a_rcStrict) == VINF_IOM_R3_MMIO_COMMIT_WRITE && (a_fWrite)) \
+     \
+     || (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR  \
+     || (a_rcStrict) == VINF_EM_DBG_STOP \
+     || (a_rcStrict) == VINF_EM_DBG_EVENT \
+     || (a_rcStrict) == VINF_EM_DBG_BREAKPOINT \
+     || (a_rcStrict) == VINF_EM_OFF \
+     || (a_rcStrict) == VINF_EM_SUSPEND \
+     || (a_rcStrict) == VINF_EM_RESET \
+    )
+#else
+# error "Context?"
+#endif
+
+/** @def PGM_HANDLER_VIRT_IS_VALID_STATUS
+ * Checks if valid virtual access handler return code (normal handler, not PF).
+ *
+ * Checks if the given strict status code is one of the expected ones for a
+ * virtual access handler in the current context.
+ *
+ * @returns true or false.
+ * @param   a_rcStrict      The status code.
+ * @param   a_fWrite        Whether it is a write or read being serviced.
+ *
+ * @remarks We wish to keep the list of statuses here as short as possible.
+ *          When changing, please make sure to update the PGMPhysRead,
+ *          PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
+ */
+#ifdef IN_RING3
+# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
+    (   (a_rcStrict) == VINF_SUCCESS \
+     || (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
+#elif defined(IN_RING0)
+# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
+    (false /* no virtual handlers in ring-0! */ )
+#else
+# error "Context?"
+#endif
+
+
+
+/**
+ * Calculate the actual table size.
+ *
+ * The memory is layed out like this:
+ *      - PGMPHYSHANDLERTREE (8 bytes)
+ *      - Allocation bitmap (8-byte size align)
+ *      - Slab of PGMPHYSHANDLER. Start is 64 byte aligned.
+ */
+uint32_t pgmHandlerPhysicalCalcTableSizes(uint32_t *pcEntries, uint32_t *pcbTreeAndBitmap)
+{
+    /*
+     * A minimum of 64 entries and a maximum of ~64K.
+     */
+    uint32_t cEntries = *pcEntries;
+    if (cEntries <= 64)
+        cEntries = 64;
+    else if (cEntries >= _64K)
+        cEntries = _64K;
+    else
+        cEntries = RT_ALIGN_32(cEntries, 16);
+
+    /*
+     * Do the initial calculation.
+     */
+    uint32_t cbBitmap        = RT_ALIGN_32(cEntries, 64) / 8;
+    uint32_t cbTreeAndBitmap = RT_ALIGN_32(sizeof(PGMPHYSHANDLERTREE) + cbBitmap, 64);
+    uint32_t cbTable         = cEntries * sizeof(PGMPHYSHANDLER);
+    uint32_t cbTotal         = cbTreeAndBitmap + cbTable;
+
+    /*
+     * Align the total and try use up extra space from that.
+     */
+    uint32_t cbTotalAligned = RT_ALIGN_32(cbTotal, RT_MAX(HOST_PAGE_SIZE, _16K));
+    uint32_t cAvail         = cbTotalAligned - cbTotal;
+    cAvail /= sizeof(PGMPHYSHANDLER);
+    if (cAvail >= 1)
+        for (;;)
+        {
+            cbBitmap        = RT_ALIGN_32(cEntries, 64) / 8;
+            cbTreeAndBitmap = RT_ALIGN_32(sizeof(PGMPHYSHANDLERTREE) + cbBitmap, 64);
+            cbTable         = cEntries * sizeof(PGMPHYSHANDLER);
+            cbTotal         = cbTreeAndBitmap + cbTable;
+            if (cbTotal <= cbTotalAligned)
+                break;
+            cEntries--;
+            Assert(cEntries >= 16);
+        }
+
+    /*
+     * Return the result.
+     */
+    *pcbTreeAndBitmap = cbTreeAndBitmap;
+    *pcEntries        = cEntries;
+    return cbTotalAligned;
+}
+
+
+/**
+ * Looks up a ROM range by its PGMROMRANGE::GCPhys value.
+ */
+DECLINLINE(PPGMROMRANGE) pgmPhysRomLookupByBase(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    for (PPGMROMRANGE pRom = pVM->pgm.s.CTX_SUFF(pRomRanges); pRom; pRom = pRom->CTX_SUFF(pNext))
+        if (pRom->GCPhys == GCPhys)
+            return pRom;
+    return NULL;
+}
+
+#ifndef IN_RING3
+
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      \#PF access handler callback for guest ROM range write access.}
+ *
+ * @remarks The @a uUser argument is the PGMROMRANGE::GCPhys value.
+ */
+DECLCALLBACK(VBOXSTRICTRC) pgmPhysRomWritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
+                                                    RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
+
+{
+    PPGMROMRANGE const  pRom  = pgmPhysRomLookupByBase(pVM, uUser);
+    AssertReturn(pRom, VINF_EM_RAW_EMULATE_INSTR);
+    uint32_t const      iPage = (GCPhysFault - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
+    int                 rc;
+    RT_NOREF(uErrorCode, pvFault);
+
+    Assert(uErrorCode & X86_TRAP_PF_RW); /* This shall not be used for read access! */
+
+    Assert(iPage < (pRom->cb >> GUEST_PAGE_SHIFT));
+    switch (pRom->aPages[iPage].enmProt)
+    {
+        case PGMROMPROT_READ_ROM_WRITE_IGNORE:
+        case PGMROMPROT_READ_RAM_WRITE_IGNORE:
+        {
+            /*
+             * If it's a simple instruction which doesn't change the cpu state
+             * we will simply skip it. Otherwise we'll have to defer it to REM.
+             */
+            uint32_t     cbOp;
+            PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
+            rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbOp);
+            if (     RT_SUCCESS(rc)
+                &&   pDis->uCpuMode == DISCPUMODE_32BIT  /** @todo why does this matter? */
+                &&  !(pDis->fPrefix & (DISPREFIX_REPNE | DISPREFIX_REP | DISPREFIX_SEG)))
+            {
+                switch (pDis->bOpCode)
+                {
+                    /** @todo Find other instructions we can safely skip, possibly
+                     * adding this kind of detection to DIS or EM. */
+                    case OP_MOV:
+                        pCtx->rip += cbOp;
+                        STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZGuestROMWriteHandled);
+                        return VINF_SUCCESS;
+                }
+            }
+            break;
+        }
+
+        case PGMROMPROT_READ_RAM_WRITE_RAM:
+            pRom->aPages[iPage].LiveSave.fWrittenTo = true;
+            rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & X86_PTE_PG_MASK);
+            AssertRC(rc);
+            break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
+
+        case PGMROMPROT_READ_ROM_WRITE_RAM:
+            /* Handle it in ring-3 because it's *way* easier there. */
+            pRom->aPages[iPage].LiveSave.fWrittenTo = true;
+            break;
+
+        default:
+            AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
+                                   pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
+                                  VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+    }
+
+    STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZGuestROMWriteUnhandled);
+    return VINF_EM_RAW_EMULATE_INSTR;
+}
+
+#endif /* !IN_RING3 */
+
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER,
+ *      Access handler callback for ROM write accesses.}
+ *
+ * @remarks The @a uUser argument is the PGMROMRANGE::GCPhys value.
+ */
+DECLCALLBACK(VBOXSTRICTRC)
+pgmPhysRomWriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
+                       PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
+{
+    PPGMROMRANGE const  pRom  = pgmPhysRomLookupByBase(pVM, uUser);
+    AssertReturn(pRom, VERR_INTERNAL_ERROR_3);
+    uint32_t const      iPage    = (GCPhys - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
+    Assert(iPage < (pRom->cb >> GUEST_PAGE_SHIFT));
+    PPGMROMPAGE const   pRomPage = &pRom->aPages[iPage];
+
+    Log5(("pgmPhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
+    RT_NOREF(pVCpu, pvPhys, enmOrigin);
+
+    if (enmAccessType == PGMACCESSTYPE_READ)
+    {
+        switch (pRomPage->enmProt)
+        {
+            /*
+             * Take the default action.
+             */
+            case PGMROMPROT_READ_ROM_WRITE_IGNORE:
+            case PGMROMPROT_READ_RAM_WRITE_IGNORE:
+            case PGMROMPROT_READ_ROM_WRITE_RAM:
+            case PGMROMPROT_READ_RAM_WRITE_RAM:
+                return VINF_PGM_HANDLER_DO_DEFAULT;
+
+            default:
+                AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
+                                       pRom->aPages[iPage].enmProt, iPage, GCPhys),
+                                      VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+        }
+    }
+    else
+    {
+        Assert(enmAccessType == PGMACCESSTYPE_WRITE);
+        switch (pRomPage->enmProt)
+        {
+            /*
+             * Ignore writes.
+             */
+            case PGMROMPROT_READ_ROM_WRITE_IGNORE:
+            case PGMROMPROT_READ_RAM_WRITE_IGNORE:
+                return VINF_SUCCESS;
+
+            /*
+             * Write to the RAM page.
+             */
+            case PGMROMPROT_READ_ROM_WRITE_RAM:
+            case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's *way* simpler that way. */
+            {
+                /* This should be impossible now, pvPhys doesn't work cross page anylonger. */
+                Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> GUEST_PAGE_SHIFT) == iPage);
+
+                /*
+                 * Take the lock, do lazy allocation, map the page and copy the data.
+                 *
+                 * Note that we have to bypass the mapping TLB since it works on
+                 * guest physical addresses and entering the shadow page would
+                 * kind of screw things up...
+                 */
+                PGM_LOCK_VOID(pVM);
+
+                PPGMPAGE pShadowPage = &pRomPage->Shadow;
+                if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
+                {
+                    pShadowPage = pgmPhysGetPage(pVM, GCPhys);
+                    AssertLogRelMsgReturnStmt(pShadowPage, ("%RGp\n", GCPhys), PGM_UNLOCK(pVM), VERR_PGM_PHYS_PAGE_GET_IPE);
+                }
+
+                void *pvDstPage;
+                int rc;
+#if defined(VBOX_WITH_PGM_NEM_MODE) && defined(IN_RING3)
+                if (PGM_IS_IN_NEM_MODE(pVM) && PGMROMPROT_IS_ROM(pRomPage->enmProt))
+                {
+                    pvDstPage = &pRom->pbR3Alternate[GCPhys - pRom->GCPhys];
+                    rc = VINF_SUCCESS;
+                }
+                else
+#endif
+                {
+                    rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & X86_PTE_PG_MASK, &pvDstPage);
+                    if (RT_SUCCESS(rc))
+                        pvDstPage = (uint8_t *)pvDstPage + (GCPhys & GUEST_PAGE_OFFSET_MASK);
+                }
+                if (RT_SUCCESS(rc))
+                {
+                    memcpy((uint8_t *)pvDstPage + (GCPhys & GUEST_PAGE_OFFSET_MASK), pvBuf, cbBuf);
+                    pRomPage->LiveSave.fWrittenTo = true;
+
+                    AssertMsg(    rc == VINF_SUCCESS
+                              || (  rc == VINF_PGM_SYNC_CR3
+                                  && VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
+                              , ("%Rrc\n", rc));
+                    rc = VINF_SUCCESS;
+                }
+
+                PGM_UNLOCK(pVM);
+                return rc;
+            }
+
+            default:
+                AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
+                                       pRom->aPages[iPage].enmProt, iPage, GCPhys),
+                                      VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+        }
+    }
+}
+
+
+/**
+ * Common worker for pgmPhysMmio2WriteHandler and pgmPhysMmio2WritePfHandler.
+ */
+static VBOXSTRICTRC pgmPhysMmio2WriteHandlerCommon(PVMCC pVM, PVMCPUCC pVCpu, uint64_t hMmio2, RTGCPHYS GCPhys, RTGCPTR GCPtr)
+{
+    /*
+     * Get the MMIO2 range.
+     */
+    AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.apMmio2RangesR3), VERR_INTERNAL_ERROR_3);
+    AssertReturn(hMmio2 != 0, VERR_INTERNAL_ERROR_3);
+    PPGMREGMMIO2RANGE pMmio2 = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[hMmio2 - 1];
+    Assert(pMmio2->idMmio2 == hMmio2);
+    AssertReturn((pMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES) == PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES,
+                 VERR_INTERNAL_ERROR_4);
+
+    /*
+     * Get the page and make sure it's an MMIO2 page.
+     */
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    AssertReturn(pPage, VINF_EM_RAW_EMULATE_INSTR);
+    AssertReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2, VINF_EM_RAW_EMULATE_INSTR);
+
+    /*
+     * Set the dirty flag so we can avoid scanning all the pages when it isn't dirty.
+     * (The PGM_PAGE_HNDL_PHYS_STATE_DISABLED handler state indicates that a single
+     * page is dirty, saving the need for additional storage (bitmap).)
+     */
+    pMmio2->fFlags |= PGMREGMMIO2RANGE_F_IS_DIRTY;
+
+    /*
+     * Disable the handler for this page.
+     */
+    int rc = PGMHandlerPhysicalPageTempOff(pVM, pMmio2->RamRange.GCPhys, GCPhys & X86_PTE_PG_MASK);
+    AssertRC(rc);
+#ifndef IN_RING3
+    if (RT_SUCCESS(rc) && GCPtr != ~(RTGCPTR)0)
+    {
+        rc = PGMShwMakePageWritable(pVCpu, GCPtr, PGM_MK_PG_IS_MMIO2 | PGM_MK_PG_IS_WRITE_FAULT);
+        AssertMsgReturn(rc == VINF_SUCCESS || rc == VERR_PAGE_TABLE_NOT_PRESENT,
+                        ("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", GCPtr, rc), rc);
+    }
+#else
+    RT_NOREF(pVCpu, GCPtr);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+#ifndef IN_RING3
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      \#PF access handler callback for guest MMIO2 dirty page tracing.}
+ *
+ * @remarks The @a uUser is the MMIO2 index.
+ */
+DECLCALLBACK(VBOXSTRICTRC) pgmPhysMmio2WritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
+                                                      RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
+{
+    RT_NOREF(pVCpu, uErrorCode, pCtx);
+    VBOXSTRICTRC rcStrict = PGM_LOCK(pVM); /* We should already have it, but just make sure we do. */
+    if (RT_SUCCESS(rcStrict))
+    {
+        rcStrict = pgmPhysMmio2WriteHandlerCommon(pVM, pVCpu, uUser, GCPhysFault, pvFault);
+        PGM_UNLOCK(pVM);
+    }
+    return rcStrict;
+}
+#endif /* !IN_RING3 */
+
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER,
+ *      Access handler callback for MMIO2 dirty page tracing.}
+ *
+ * @remarks The @a uUser is the MMIO2 index.
+ */
+DECLCALLBACK(VBOXSTRICTRC)
+pgmPhysMmio2WriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
+                         PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
+{
+    VBOXSTRICTRC rcStrict = PGM_LOCK(pVM); /* We should already have it, but just make sure we do. */
+    if (RT_SUCCESS(rcStrict))
+    {
+        rcStrict = pgmPhysMmio2WriteHandlerCommon(pVM, pVCpu, uUser, GCPhys, ~(RTGCPTR)0);
+        PGM_UNLOCK(pVM);
+        if (rcStrict == VINF_SUCCESS)
+            rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
+    }
+    RT_NOREF(pvPhys, pvBuf, cbBuf, enmAccessType, enmOrigin);
+    return rcStrict;
+}
+
+
+/**
+ * Invalidates the RAM range TLBs.
+ *
+ * @param   pVM                 The cross context VM structure.
+ */
+void pgmPhysInvalidRamRangeTlbs(PVMCC pVM)
+{
+    PGM_LOCK_VOID(pVM);
+    RT_ZERO(pVM->pgm.s.apRamRangesTlbR3);
+    RT_ZERO(pVM->pgm.s.apRamRangesTlbR0);
+    PGM_UNLOCK(pVM);
+}
+
+
+/**
+ * Tests if a value of type RTGCPHYS is negative if the type had been signed
+ * instead of unsigned.
+ *
+ * @returns @c true if negative, @c false if positive or zero.
+ * @param   a_GCPhys        The value to test.
+ * @todo    Move me to iprt/types.h.
+ */
+#define RTGCPHYS_IS_NEGATIVE(a_GCPhys)  ((a_GCPhys) & ((RTGCPHYS)1 << (sizeof(RTGCPHYS)*8 - 1)))
+
+
+/**
+ * Slow worker for pgmPhysGetRange.
+ *
+ * @copydoc pgmPhysGetRange
+ */
+PPGMRAMRANGE pgmPhysGetRangeSlow(PVM pVM, RTGCPHYS GCPhys)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
+
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
+    while (pRam)
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            return pRam;
+        }
+        if (RTGCPHYS_IS_NEGATIVE(off))
+            pRam = pRam->CTX_SUFF(pLeft);
+        else
+            pRam = pRam->CTX_SUFF(pRight);
+    }
+    return NULL;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetRangeAtOrAbove.
+ *
+ * @copydoc pgmPhysGetRangeAtOrAbove
+ */
+PPGMRAMRANGE pgmPhysGetRangeAtOrAboveSlow(PVM pVM, RTGCPHYS GCPhys)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
+
+    PPGMRAMRANGE pLastLeft = NULL;
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
+    while (pRam)
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            return pRam;
+        }
+        if (RTGCPHYS_IS_NEGATIVE(off))
+        {
+            pLastLeft = pRam;
+            pRam = pRam->CTX_SUFF(pLeft);
+        }
+        else
+            pRam = pRam->CTX_SUFF(pRight);
+    }
+    return pLastLeft;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetPage.
+ *
+ * @copydoc pgmPhysGetPage
+ */
+PPGMPAGE pgmPhysGetPageSlow(PVM pVM, RTGCPHYS GCPhys)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
+
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
+    while (pRam)
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            return &pRam->aPages[off >> GUEST_PAGE_SHIFT];
+        }
+
+        if (RTGCPHYS_IS_NEGATIVE(off))
+            pRam = pRam->CTX_SUFF(pLeft);
+        else
+            pRam = pRam->CTX_SUFF(pRight);
+    }
+    return NULL;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetPageEx.
+ *
+ * @copydoc pgmPhysGetPageEx
+ */
+int pgmPhysGetPageExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
+
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
+    while (pRam)
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            *ppPage = &pRam->aPages[off >> GUEST_PAGE_SHIFT];
+            return VINF_SUCCESS;
+        }
+
+        if (RTGCPHYS_IS_NEGATIVE(off))
+            pRam = pRam->CTX_SUFF(pLeft);
+        else
+            pRam = pRam->CTX_SUFF(pRight);
+    }
+
+    *ppPage = NULL;
+    return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+}
+
+
+/**
+ * Slow worker for pgmPhysGetPageAndRangeEx.
+ *
+ * @copydoc pgmPhysGetPageAndRangeEx
+ */
+int pgmPhysGetPageAndRangeExSlow(PVM pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
+{
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
+
+    PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangeTree);
+    while (pRam)
+    {
+        RTGCPHYS off = GCPhys - pRam->GCPhys;
+        if (off < pRam->cb)
+        {
+            pVM->pgm.s.CTX_SUFF(apRamRangesTlb)[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRam;
+            *ppRam  = pRam;
+            *ppPage = &pRam->aPages[off >> GUEST_PAGE_SHIFT];
+            return VINF_SUCCESS;
+        }
+
+        if (RTGCPHYS_IS_NEGATIVE(off))
+            pRam = pRam->CTX_SUFF(pLeft);
+        else
+            pRam = pRam->CTX_SUFF(pRight);
+    }
+
+    *ppRam  = NULL;
+    *ppPage = NULL;
+    return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+}
+
+
+/**
+ * Checks if Address Gate 20 is enabled or not.
+ *
+ * @returns true if enabled.
+ * @returns false if disabled.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
+{
+    LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu->pgm.s.fA20Enabled));
+    return pVCpu->pgm.s.fA20Enabled;
+}
+
+
+/**
+ * Validates a GC physical address.
+ *
+ * @returns true if valid.
+ * @returns false if invalid.
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  The physical address to validate.
+ */
+VMMDECL(bool) PGMPhysIsGCPhysValid(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    return pPage != NULL;
+}
+
+
+/**
+ * Checks if a GC physical address is a normal page,
+ * i.e. not ROM, MMIO or reserved.
+ *
+ * @returns true if normal.
+ * @returns false if invalid, ROM, MMIO or reserved page.
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  The physical address to check.
+ */
+VMMDECL(bool) PGMPhysIsGCPhysNormal(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    return pPage
+        && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
+}
+
+
+/**
+ * Converts a GC physical address to a HC physical address.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
+ *          page but has no physical backing.
+ * @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
+ *          GC physical address.
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  The GC physical address to convert.
+ * @param   pHCPhys Where to store the HC physical address on success.
+ */
+VMM_INT_DECL(int) PGMPhysGCPhys2HCPhys(PVMCC pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
+{
+    PGM_LOCK_VOID(pVM);
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_SUCCESS(rc))
+        *pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) | (GCPhys & GUEST_PAGE_OFFSET_MASK);
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+
+/**
+ * Invalidates all page mapping TLBs.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+void pgmPhysInvalidatePageMapTLB(PVMCC pVM)
+{
+    PGM_LOCK_VOID(pVM);
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatPageMapTlbFlushes);
+
+    /* Clear the R3 & R0 TLBs completely. */
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR0.aEntries); i++)
+    {
+        pVM->pgm.s.PhysTlbR0.aEntries[i].GCPhys = NIL_RTGCPHYS;
+        pVM->pgm.s.PhysTlbR0.aEntries[i].pPage = 0;
+        pVM->pgm.s.PhysTlbR0.aEntries[i].pv = 0;
+    }
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
+    {
+        pVM->pgm.s.PhysTlbR3.aEntries[i].GCPhys = NIL_RTGCPHYS;
+        pVM->pgm.s.PhysTlbR3.aEntries[i].pPage = 0;
+        pVM->pgm.s.PhysTlbR3.aEntries[i].pMap = 0;
+        pVM->pgm.s.PhysTlbR3.aEntries[i].pv = 0;
+    }
+
+    IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID);
+    PGM_UNLOCK(pVM);
+}
+
+
+/**
+ * Invalidates a page mapping TLB entry
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  GCPhys entry to flush
+ *
+ * @note    Caller is responsible for calling IEMTlbInvalidateAllPhysicalAllCpus
+ *          when needed.
+ */
+void pgmPhysInvalidatePageMapTLBEntry(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatPageMapTlbFlushEntry);
+
+    unsigned const idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
+
+    pVM->pgm.s.PhysTlbR0.aEntries[idx].GCPhys = NIL_RTGCPHYS;
+    pVM->pgm.s.PhysTlbR0.aEntries[idx].pPage = 0;
+    pVM->pgm.s.PhysTlbR0.aEntries[idx].pv = 0;
+
+    pVM->pgm.s.PhysTlbR3.aEntries[idx].GCPhys = NIL_RTGCPHYS;
+    pVM->pgm.s.PhysTlbR3.aEntries[idx].pPage = 0;
+    pVM->pgm.s.PhysTlbR3.aEntries[idx].pMap = 0;
+    pVM->pgm.s.PhysTlbR3.aEntries[idx].pv = 0;
+}
+
+
+/**
+ * Makes sure that there is at least one handy page ready for use.
+ *
+ * This will also take the appropriate actions when reaching water-marks.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_EM_NO_MEMORY if we're really out of memory.
+ *
+ * @param   pVM     The cross context VM structure.
+ *
+ * @remarks Must be called from within the PGM critical section. It may
+ *          nip back to ring-3/0 in some cases.
+ */
+static int pgmPhysEnsureHandyPage(PVMCC pVM)
+{
+    AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
+
+    /*
+     * Do we need to do anything special?
+     */
+#ifdef IN_RING3
+    if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
+#else
+    if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
+#endif
+    {
+        /*
+         * Allocate pages only if we're out of them, or in ring-3, almost out.
+         */
+#ifdef IN_RING3
+        if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
+#else
+        if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
+#endif
+        {
+            Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
+                 pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) ));
+#ifdef IN_RING3
+            int rc = PGMR3PhysAllocateHandyPages(pVM);
+#else
+            int rc = pgmR0PhysAllocateHandyPages(pVM, VMMGetCpuId(pVM), false /*fRing3*/);
+#endif
+            if (RT_UNLIKELY(rc != VINF_SUCCESS))
+            {
+                if (RT_FAILURE(rc))
+                    return rc;
+                AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
+                if (!pVM->pgm.s.cHandyPages)
+                {
+                    LogRel(("PGM: no more handy pages!\n"));
+                    return VERR_EM_NO_MEMORY;
+                }
+                Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
+                Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY));
+#ifndef IN_RING3
+                VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
+#endif
+            }
+            AssertMsgReturn(    pVM->pgm.s.cHandyPages > 0
+                            &&  pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
+                            ("%u\n", pVM->pgm.s.cHandyPages),
+                            VERR_PGM_HANDY_PAGE_IPE);
+        }
+        else
+        {
+            if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
+                VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
+#ifndef IN_RING3
+            if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
+            {
+                Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
+                VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
+            }
+#endif
+        }
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Replace a zero or shared page with new page that we can write to.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success, pPage is modified.
+ * @retval  VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
+ * @retval  VERR_EM_NO_MEMORY if we're totally out of memory.
+ *
+ * @todo    Propagate VERR_EM_NO_MEMORY up the call tree.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure. This will
+ *                      be modified on success.
+ * @param   GCPhys      The address of the page.
+ *
+ * @remarks Must be called from within the PGM critical section. It may
+ *          nip back to ring-3/0 in some cases.
+ *
+ * @remarks This function shouldn't really fail, however if it does
+ *          it probably means we've screwed up the size of handy pages and/or
+ *          the low-water mark. Or, that some device I/O is causing a lot of
+ *          pages to be allocated while while the host is in a low-memory
+ *          condition. This latter should be handled elsewhere and in a more
+ *          controlled manner, it's on the @bugref{3170} todo list...
+ */
+int pgmPhysAllocPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
+{
+    LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
+
+    /*
+     * Prereqs.
+     */
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    AssertMsg(PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
+    Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
+
+# ifdef PGM_WITH_LARGE_PAGES
+    /*
+     * Try allocate a large page if applicable.
+     */
+    if (   PGMIsUsingLargePages(pVM)
+        && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
+        && !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Implement large pages support. */
+    {
+        RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
+        PPGMPAGE pBasePage;
+
+        int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
+        AssertRCReturn(rc, rc);     /* paranoia; can't happen. */
+        if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE)
+        {
+            rc = pgmPhysAllocLargePage(pVM, GCPhys);
+            if (rc == VINF_SUCCESS)
+                return rc;
+        }
+        /* Mark the base as type page table, so we don't check over and over again. */
+        PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT);
+
+        /* fall back to 4KB pages. */
+    }
+# endif
+
+    /*
+     * Flush any shadow page table mappings of the page.
+     * When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
+     */
+    bool fFlushTLBs = false;
+    int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /*fFlushTLBs*/, &fFlushTLBs);
+    AssertMsgReturn(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
+
+    /*
+     * Ensure that we've got a page handy, take it and use it.
+     */
+    int rc2 = pgmPhysEnsureHandyPage(pVM);
+    if (RT_FAILURE(rc2))
+    {
+        if (fFlushTLBs)
+            PGM_INVL_ALL_VCPU_TLBS(pVM);
+        Assert(rc2 == VERR_EM_NO_MEMORY);
+        return rc2;
+    }
+    /* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    AssertMsg(PGM_PAGE_IS_ZERO(pPage) || PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
+    Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
+
+    uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
+    AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
+    Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_GMMPAGEDESC_PHYS);
+    Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
+    Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
+    Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
+
+    /*
+     * There are one or two action to be taken the next time we allocate handy pages:
+     *      - Tell the GMM (global memory manager) what the page is being used for.
+     *        (Speeds up replacement operations - sharing and defragmenting.)
+     *      - If the current backing is shared, it must be freed.
+     */
+    const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
+    pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+
+    void const *pvSharedPage = NULL;
+    if (PGM_PAGE_IS_SHARED(pPage))
+    {
+        /* Mark this shared page for freeing/dereferencing. */
+        pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
+        Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
+
+        Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
+             GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageReplaceShared));
+        pVM->pgm.s.cSharedPages--;
+
+        /* Grab the address of the page so we can make a copy later on. (safe) */
+        rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
+        AssertRC(rc);
+    }
+    else
+    {
+        Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatRZPageReplaceZero);
+        pVM->pgm.s.cZeroPages--;
+    }
+
+    /*
+     * Do the PGMPAGE modifications.
+     */
+    pVM->pgm.s.cPrivatePages++;
+    PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
+    PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
+    PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+    PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT);
+    pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
+    IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID);
+
+    /* Copy the shared page contents to the replacement page. */
+    if (pvSharedPage)
+    {
+        /* Get the virtual address of the new page. */
+        PGMPAGEMAPLOCK  PgMpLck;
+        void           *pvNewPage;
+        rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc);
+        if (RT_SUCCESS(rc))
+        {
+            memcpy(pvNewPage, pvSharedPage, GUEST_PAGE_SIZE); /** @todo todo write ASMMemCopyPage */
+            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+        }
+    }
+
+    if (    fFlushTLBs
+        &&  rc != VINF_PGM_GCPHYS_ALIASED)
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+
+    /*
+     * Notify NEM about the mapping change for this page.
+     *
+     * Note! Shadow ROM pages are complicated as they can definitely be
+     *       allocated while not visible, so play safe.
+     */
+    if (VM_IS_NEM_ENABLED(pVM))
+    {
+        PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+        if (   enmType != PGMPAGETYPE_ROM_SHADOW
+            || pgmPhysGetPage(pVM, GCPhys) == pPage)
+        {
+            uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+            rc2 = NEMHCNotifyPhysPageAllocated(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, HCPhys,
+                                               pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+            if (RT_SUCCESS(rc))
+                PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+            else
+                rc = rc2;
+        }
+    }
+
+    return rc;
+}
+
+#ifdef PGM_WITH_LARGE_PAGES
+
+/**
+ * Replace a 2 MB range of zero pages with new pages that we can write to.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success, pPage is modified.
+ * @retval  VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
+ * @retval  VERR_EM_NO_MEMORY if we're totally out of memory.
+ *
+ * @todo    Propagate VERR_EM_NO_MEMORY up the call tree.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The address of the page.
+ *
+ * @remarks Must be called from within the PGM critical section.  It may block
+ *          on GMM and host mutexes/locks, leaving HM context.
+ */
+int pgmPhysAllocLargePage(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
+    LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
+    Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
+
+    /*
+     * Check Prereqs.
+     */
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    Assert(PGMIsUsingLargePages(pVM));
+
+    /*
+     * All the pages must be unallocated RAM pages, i.e. mapping the ZERO page.
+     */
+    PPGMPAGE pFirstPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
+    if (   RT_SUCCESS(rc)
+        && PGM_PAGE_GET_TYPE(pFirstPage)  == PGMPAGETYPE_RAM
+        && PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ZERO)
+    {
+        /*
+         * Further they should have PDE type set to PGM_PAGE_PDE_TYPE_DONTCARE,
+         * since they are unallocated.
+         */
+        unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pFirstPage);
+        Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
+        if (uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE)
+        {
+            /*
+             * Now, make sure all the other pages in the 2 MB is in the same state.
+             */
+            GCPhys = GCPhysBase;
+            unsigned cLeft = _2M / GUEST_PAGE_SIZE;
+            while (cLeft-- > 0)
+            {
+                PPGMPAGE pSubPage = pgmPhysGetPage(pVM, GCPhys);
+                if (   pSubPage
+                    && PGM_PAGE_GET_TYPE(pSubPage)  == PGMPAGETYPE_RAM      /* Anything other than ram implies monitoring. */
+                    && PGM_PAGE_GET_STATE(pSubPage) == PGM_PAGE_STATE_ZERO) /* Allocated, monitored or shared means we can't use a large page here */
+                {
+                    Assert(PGM_PAGE_GET_PDE_TYPE(pSubPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
+                    GCPhys += GUEST_PAGE_SIZE;
+                }
+                else
+                {
+                    LogFlow(("pgmPhysAllocLargePage: Found page %RGp with wrong attributes (type=%d; state=%d); cancel check.\n",
+                             GCPhys, pSubPage ? PGM_PAGE_GET_TYPE(pSubPage) : -1, pSubPage ? PGM_PAGE_GET_STATE(pSubPage) : -1));
+
+                    /* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
+                    STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
+                    PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT);
+                    return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
+                }
+            }
+
+            /*
+             * Do the allocation.
+             */
+# ifdef IN_RING3
+            rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_PAGE, GCPhysBase, NULL);
+# elif defined(IN_RING0)
+            rc = pgmR0PhysAllocateLargePage(pVM, VMMGetCpuId(pVM), GCPhysBase);
+# else
+#  error "Port me"
+# endif
+            if (RT_SUCCESS(rc))
+            {
+                Assert(PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ALLOCATED);
+                pVM->pgm.s.cLargePages++;
+                return VINF_SUCCESS;
+            }
+
+            /* If we fail once, it most likely means the host's memory is too
+               fragmented; don't bother trying again. */
+            LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
+            return rc;
+        }
+    }
+    return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
+}
+
+
+/**
+ * Recheck the entire 2 MB range to see if we can use it again as a large page.
+ *
+ * @returns The following VBox status codes.
+ * @retval  VINF_SUCCESS on success, the large page can be used again
+ * @retval  VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The address of the page.
+ * @param   pLargePage  Page structure of the base page
+ */
+int pgmPhysRecheckLargePage(PVMCC pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
+{
+    STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
+
+    Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
+
+    AssertCompile(X86_PDE2M_PAE_PG_MASK == EPT_PDE2M_PG_MASK);  /* Paranoia: Caller uses this for guest EPT tables as well. */
+    GCPhys &= X86_PDE2M_PAE_PG_MASK;
+
+    /* Check the base page. */
+    Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
+    if (    PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
+        ||  PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
+        ||  PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
+    {
+        LogFlow(("pgmPhysRecheckLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
+        return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
+    }
+
+    STAM_PROFILE_START(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,IsValidLargePage), a);
+    /* Check all remaining pages in the 2 MB range. */
+    unsigned i;
+    GCPhys += GUEST_PAGE_SIZE;
+    for (i = 1; i < _2M / GUEST_PAGE_SIZE; i++)
+    {
+        PPGMPAGE pPage;
+        int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+        AssertRCBreak(rc);
+
+        if (    PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
+            ||  PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
+            ||  PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
+            ||  PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
+        {
+            LogFlow(("pgmPhysRecheckLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
+            break;
+        }
+
+        GCPhys += GUEST_PAGE_SIZE;
+    }
+    STAM_PROFILE_STOP(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,IsValidLargePage), a);
+
+    if (i == _2M / GUEST_PAGE_SIZE)
+    {
+        PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE);
+        pVM->pgm.s.cLargePagesDisabled--;
+        Log(("pgmPhysRecheckLargePage: page %RGp can be reused!\n", GCPhys - _2M));
+        return VINF_SUCCESS;
+    }
+
+    return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
+}
+
+#endif /* PGM_WITH_LARGE_PAGES */
+
+
+/**
+ * Deal with a write monitored page.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure.
+ * @param   GCPhys      The guest physical address of the page.
+ *                      PGMPhysReleasePageMappingLock() passes NIL_RTGCPHYS in a
+ *                      very unlikely situation where it is okay that we let NEM
+ *                      fix the page access in a lazy fasion.
+ *
+ * @remarks Called from within the PGM critical section.
+ */
+void pgmPhysPageMakeWriteMonitoredWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
+{
+    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
+    PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
+    PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
+    Assert(pVM->pgm.s.cMonitoredPages > 0);
+    pVM->pgm.s.cMonitoredPages--;
+    pVM->pgm.s.cWrittenToPages++;
+
+#ifdef VBOX_WITH_NATIVE_NEM
+    /*
+     * Notify NEM about the protection change so we won't spin forever.
+     *
+     * Note! NEM need to be handle to lazily correct page protection as we cannot
+     *       really get it 100% right here it seems.  The page pool does this too.
+     */
+    if (VM_IS_NEM_ENABLED(pVM) && GCPhys != NIL_RTGCPHYS)
+    {
+        uint8_t      u2State = PGM_PAGE_GET_NEM_STATE(pPage);
+        PGMPAGETYPE  enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+        PPGMRAMRANGE pRam    = pgmPhysGetRange(pVM, GCPhys);
+        NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
+                                       pRam ? PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhys) : NULL,
+                                       pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
+        PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+    }
+#else
+    RT_NOREF(GCPhys);
+#endif
+}
+
+
+/**
+ * Deal with pages that are not writable, i.e. not in the ALLOCATED state.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure.
+ * @param   GCPhys      The address of the page.
+ *
+ * @remarks Called from within the PGM critical section.
+ */
+int pgmPhysPageMakeWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    switch (PGM_PAGE_GET_STATE(pPage))
+    {
+        case PGM_PAGE_STATE_WRITE_MONITORED:
+            pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
+            RT_FALL_THRU();
+        default: /* to shut up GCC */
+        case PGM_PAGE_STATE_ALLOCATED:
+            return VINF_SUCCESS;
+
+        /*
+         * Zero pages can be dummy pages for MMIO or reserved memory,
+         * so we need to check the flags before joining cause with
+         * shared page replacement.
+         */
+        case PGM_PAGE_STATE_ZERO:
+            if (PGM_PAGE_IS_MMIO(pPage))
+                return VERR_PGM_PHYS_PAGE_RESERVED;
+            RT_FALL_THRU();
+        case PGM_PAGE_STATE_SHARED:
+            return pgmPhysAllocPage(pVM, pPage, GCPhys);
+
+        /* Not allowed to write to ballooned pages. */
+        case PGM_PAGE_STATE_BALLOONED:
+            return VERR_PGM_PHYS_PAGE_BALLOONED;
+    }
+}
+
+
+/**
+ * Internal usage: Map the page specified by its GMM ID.
+ *
+ * This is similar to pgmPhysPageMap
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idPage      The Page ID.
+ * @param   HCPhys      The physical address (for SUPR0HCPhysToVirt).
+ * @param   ppv         Where to store the mapping address.
+ *
+ * @remarks Called from within the PGM critical section.  The mapping is only
+ *          valid while you are inside this section.
+ */
+int pgmPhysPageMapByPageID(PVMCC pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
+{
+    /*
+     * Validation.
+     */
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    AssertReturn(HCPhys && !(HCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
+    const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
+    AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
+
+#ifdef IN_RING0
+# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
+    return SUPR0HCPhysToVirt(HCPhys & ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK, ppv);
+# else
+    return GMMR0PageIdToVirt(pVM, idPage, ppv);
+# endif
+
+#else
+    /*
+     * Find/make Chunk TLB entry for the mapping chunk.
+     */
+    PPGMCHUNKR3MAP pMap;
+    PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
+    if (pTlbe->idChunk == idChunk)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
+        pMap = pTlbe->pChunk;
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
+
+        /*
+         * Find the chunk, map it if necessary.
+         */
+        pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
+        if (pMap)
+            pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
+        else
+        {
+            int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
+            if (RT_FAILURE(rc))
+                return rc;
+        }
+
+        /*
+         * Enter it into the Chunk TLB.
+         */
+        pTlbe->idChunk = idChunk;
+        pTlbe->pChunk = pMap;
+    }
+
+    *ppv = (uint8_t *)pMap->pv + ((idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT);
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * Maps a page into the current virtual address space so it can be accessed.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure.
+ * @param   GCPhys      The address of the page.
+ * @param   ppMap       Where to store the address of the mapping tracking structure.
+ * @param   ppv         Where to store the mapping address of the page. The page
+ *                      offset is masked off!
+ *
+ * @remarks Called from within the PGM critical section.
+ */
+static int pgmPhysPageMapCommon(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    NOREF(GCPhys);
+
+    /*
+     * Special cases: MMIO2, ZERO and specially aliased MMIO pages.
+     */
+    if (   PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2
+        || PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
+    {
+        /* Decode the page id to a page in a MMIO2 ram range. */
+        uint8_t  idMmio2 = PGM_MMIO2_PAGEID_GET_MMIO2_ID(PGM_PAGE_GET_PAGEID(pPage));
+        uint32_t iPage   = PGM_MMIO2_PAGEID_GET_IDX(PGM_PAGE_GET_PAGEID(pPage));
+        AssertLogRelMsgReturn((uint8_t)(idMmio2 - 1U) < RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)),
+                              ("idMmio2=%u size=%u type=%u GCPHys=%#RGp Id=%u State=%u", idMmio2,
+                               RT_ELEMENTS(pVM->pgm.s.CTX_SUFF(apMmio2Ranges)), PGM_PAGE_GET_TYPE(pPage), GCPhys,
+                               pPage->s.idPage, pPage->s.uStateY),
+                              VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
+        PPGMREGMMIO2RANGE pMmio2Range = pVM->pgm.s.CTX_SUFF(apMmio2Ranges)[idMmio2 - 1];
+        AssertLogRelReturn(pMmio2Range, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
+        AssertLogRelReturn(pMmio2Range->idMmio2 == idMmio2, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
+        AssertLogRelReturn(iPage < (pMmio2Range->RamRange.cb >> GUEST_PAGE_SHIFT), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
+        *ppMap = NULL;
+# if   defined(IN_RING0) && defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+        return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv);
+# elif defined(IN_RING0)
+        *ppv = (uint8_t *)pMmio2Range->pvR0 + ((uintptr_t)iPage << GUEST_PAGE_SHIFT);
+        return VINF_SUCCESS;
+# else
+        *ppv = (uint8_t *)pMmio2Range->RamRange.pvR3 + ((uintptr_t)iPage << GUEST_PAGE_SHIFT);
+        return VINF_SUCCESS;
+# endif
+    }
+
+# ifdef VBOX_WITH_PGM_NEM_MODE
+    if (pVM->pgm.s.fNemMode)
+    {
+#  ifdef IN_RING3
+        /*
+         * Find the corresponding RAM range and use that to locate the mapping address.
+         */
+        /** @todo Use the page ID for some kind of indexing as we do with MMIO2 above. */
+        PPGMRAMRANGE const pRam = pgmPhysGetRange(pVM, GCPhys);
+        AssertLogRelMsgReturn(pRam, ("%RTGp\n", GCPhys), VERR_INTERNAL_ERROR_3);
+        size_t const idxPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
+        Assert(pPage == &pRam->aPages[idxPage]);
+        *ppMap = NULL;
+        *ppv   = (uint8_t *)pRam->pvR3 + (idxPage << GUEST_PAGE_SHIFT);
+        return VINF_SUCCESS;
+#  else
+        AssertFailedReturn(VERR_INTERNAL_ERROR_2);
+#  endif
+    }
+# endif
+
+    const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
+    if (idChunk == NIL_GMM_CHUNKID)
+    {
+        AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage),
+                        VERR_PGM_PHYS_PAGE_MAP_IPE_1);
+        if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+        {
+            AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage),
+                            VERR_PGM_PHYS_PAGE_MAP_IPE_3);
+            AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage),
+                            VERR_PGM_PHYS_PAGE_MAP_IPE_4);
+            *ppv = pVM->pgm.s.abZeroPg;
+        }
+        else
+            *ppv = pVM->pgm.s.abZeroPg;
+        *ppMap = NULL;
+        return VINF_SUCCESS;
+    }
+
+# if defined(IN_RING0) && defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
+    /*
+     * Just use the physical address.
+     */
+    *ppMap = NULL;
+    return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv);
+
+# elif defined(IN_RING0)
+    /*
+     * Go by page ID thru GMMR0.
+     */
+    *ppMap = NULL;
+    return GMMR0PageIdToVirt(pVM, PGM_PAGE_GET_PAGEID(pPage), ppv);
+
+# else
+    /*
+     * Find/make Chunk TLB entry for the mapping chunk.
+     */
+    PPGMCHUNKR3MAP pMap;
+    PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
+    if (pTlbe->idChunk == idChunk)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
+        pMap = pTlbe->pChunk;
+        AssertPtr(pMap->pv);
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
+
+        /*
+         * Find the chunk, map it if necessary.
+         */
+        pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
+        if (pMap)
+        {
+            AssertPtr(pMap->pv);
+            pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
+        }
+        else
+        {
+            int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
+            if (RT_FAILURE(rc))
+                return rc;
+            AssertPtr(pMap->pv);
+        }
+
+        /*
+         * Enter it into the Chunk TLB.
+         */
+        pTlbe->idChunk = idChunk;
+        pTlbe->pChunk = pMap;
+    }
+
+    *ppv = (uint8_t *)pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << GUEST_PAGE_SHIFT);
+    *ppMap = pMap;
+    return VINF_SUCCESS;
+# endif /* !IN_RING0 */
+}
+
+
+/**
+ * Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
+ *
+ * This is typically used is paths where we cannot use the TLB methods (like ROM
+ * pages) or where there is no point in using them since we won't get many hits.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure.
+ * @param   GCPhys      The address of the page.
+ * @param   ppv         Where to store the mapping address of the page. The page
+ *                      offset is masked off!
+ *
+ * @remarks Called from within the PGM critical section.  The mapping is only
+ *          valid while you are inside section.
+ */
+int pgmPhysPageMakeWritableAndMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
+{
+    int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+    if (RT_SUCCESS(rc))
+    {
+        AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
+        PPGMPAGEMAP pMapIgnore;
+        int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
+        if (RT_FAILURE(rc2)) /* preserve rc */
+            rc = rc2;
+    }
+    return rc;
+}
+
+
+/**
+ * Maps a page into the current virtual address space so it can be accessed for
+ * both writing and reading.
+ *
+ * This is typically used is paths where we cannot use the TLB methods (like ROM
+ * pages) or where there is no point in using them since we won't get many hits.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure. Must be in the
+ *                      allocated state.
+ * @param   GCPhys      The address of the page.
+ * @param   ppv         Where to store the mapping address of the page. The page
+ *                      offset is masked off!
+ *
+ * @remarks Called from within the PGM critical section.  The mapping is only
+ *          valid while you are inside section.
+ */
+int pgmPhysPageMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
+{
+    Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
+    PPGMPAGEMAP pMapIgnore;
+    return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
+}
+
+
+/**
+ * Maps a page into the current virtual address space so it can be accessed for
+ * reading.
+ *
+ * This is typically used is paths where we cannot use the TLB methods (like ROM
+ * pages) or where there is no point in using them since we won't get many hits.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The physical page tracking structure.
+ * @param   GCPhys      The address of the page.
+ * @param   ppv         Where to store the mapping address of the page. The page
+ *                      offset is masked off!
+ *
+ * @remarks Called from within the PGM critical section.  The mapping is only
+ *          valid while you are inside this section.
+ */
+int pgmPhysPageMapReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
+{
+    PPGMPAGEMAP pMapIgnore;
+    return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
+}
+
+
+/**
+ * Load a guest page into the ring-3 physical TLB.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address in question.
+ */
+int pgmPhysPageLoadIntoTlb(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Find the ram range and page and hand it over to the with-page function.
+     * 99.8% of requests are expected to be in the first range.
+     */
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    if (!pPage)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbMisses));
+        return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
+    }
+
+    return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
+}
+
+
+/**
+ * Load a guest page into the ring-3 physical TLB.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       Pointer to the PGMPAGE structure corresponding to
+ *                      GCPhys.
+ * @param   GCPhys      The guest physical address in question.
+ */
+int pgmPhysPageLoadIntoTlbWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbMisses));
+
+    /*
+     * Map the page.
+     * Make a special case for the zero page as it is kind of special.
+     */
+    PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTX_SUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
+    if (    !PGM_PAGE_IS_ZERO(pPage)
+        &&  !PGM_PAGE_IS_BALLOONED(pPage))
+    {
+        void *pv;
+        PPGMPAGEMAP pMap;
+        int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
+        if (RT_FAILURE(rc))
+            return rc;
+# ifndef IN_RING0
+        pTlbe->pMap = pMap;
+# endif
+        pTlbe->pv = pv;
+        Assert(!((uintptr_t)pTlbe->pv & GUEST_PAGE_OFFSET_MASK));
+    }
+    else
+    {
+        AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage));
+# ifndef IN_RING0
+        pTlbe->pMap = NULL;
+# endif
+        pTlbe->pv = pVM->pgm.s.abZeroPg;
+    }
+# ifdef PGM_WITH_PHYS_TLB
+    if (    PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW
+        ||  PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM)
+        pTlbe->GCPhys = GCPhys & X86_PTE_PAE_PG_MASK;
+    else
+        pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */
+# else
+    pTlbe->GCPhys = NIL_RTGCPHYS;
+# endif
+    pTlbe->pPage = pPage;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
+ * own the PGM lock and therefore not need to lock the mapped page.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be mapped.
+ * @param   pPage       Pointer to the PGMPAGE structure for the page.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ *
+ * @internal
+ * @deprecated Use pgmPhysGCPhys2CCPtrInternalEx.
+ */
+int pgmPhysGCPhys2CCPtrInternalDepr(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
+{
+    int rc;
+    AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    pVM->pgm.s.cDeprecatedPageLocks++;
+
+    /*
+     * Make sure the page is writable.
+     */
+    if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
+    {
+        rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+        if (RT_FAILURE(rc))
+            return rc;
+        AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
+    }
+    Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
+
+    /*
+     * Get the mapping address.
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+    if (RT_FAILURE(rc))
+        return rc;
+    *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Locks a page mapping for writing.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pPage               The page.
+ * @param   pTlbe               The mapping TLB entry for the page.
+ * @param   pLock               The lock structure (output).
+ */
+DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
+{
+# ifndef IN_RING0
+    PPGMPAGEMAP pMap = pTlbe->pMap;
+    if (pMap)
+        pMap->cRefs++;
+# else
+    RT_NOREF(pTlbe);
+# endif
+
+    unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
+    if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
+    {
+        if (cLocks == 0)
+            pVM->pgm.s.cWriteLockedPages++;
+        PGM_PAGE_INC_WRITE_LOCKS(pPage);
+    }
+    else if (cLocks != PGM_PAGE_MAX_LOCKS)
+    {
+        PGM_PAGE_INC_WRITE_LOCKS(pPage);
+        AssertMsgFailed(("%R[pgmpage] is entering permanent write locked state!\n", pPage));
+# ifndef IN_RING0
+        if (pMap)
+            pMap->cRefs++; /* Extra ref to prevent it from going away. */
+# endif
+    }
+
+    pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_WRITE;
+# ifndef IN_RING0
+    pLock->pvMap = pMap;
+# else
+    pLock->pvMap = NULL;
+# endif
+}
+
+/**
+ * Locks a page mapping for reading.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pPage               The page.
+ * @param   pTlbe               The mapping TLB entry for the page.
+ * @param   pLock               The lock structure (output).
+ */
+DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
+{
+# ifndef IN_RING0
+    PPGMPAGEMAP pMap = pTlbe->pMap;
+    if (pMap)
+        pMap->cRefs++;
+# else
+    RT_NOREF(pTlbe);
+# endif
+
+    unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
+    if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
+    {
+        if (cLocks == 0)
+            pVM->pgm.s.cReadLockedPages++;
+        PGM_PAGE_INC_READ_LOCKS(pPage);
+    }
+    else if (cLocks != PGM_PAGE_MAX_LOCKS)
+    {
+        PGM_PAGE_INC_READ_LOCKS(pPage);
+        AssertMsgFailed(("%R[pgmpage] is entering permanent read locked state!\n", pPage));
+# ifndef IN_RING0
+        if (pMap)
+            pMap->cRefs++; /* Extra ref to prevent it from going away. */
+# endif
+    }
+
+    pLock->uPageAndType = (uintptr_t)pPage | PGMPAGEMAPLOCK_TYPE_READ;
+# ifndef IN_RING0
+    pLock->pvMap = pMap;
+# else
+    pLock->pvMap = NULL;
+# endif
+}
+
+
+/**
+ * Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
+ * own the PGM lock and have access to the page structure.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be mapped.
+ * @param   pPage       Pointer to the PGMPAGE structure for the page.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that
+ *                      pgmPhysReleaseInternalPageMappingLock needs.
+ *
+ * @internal
+ */
+int pgmPhysGCPhys2CCPtrInternal(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    int rc;
+    AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Make sure the page is writable.
+     */
+    if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
+    {
+        rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+        if (RT_FAILURE(rc))
+            return rc;
+        AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
+    }
+    Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
+
+    /*
+     * Do the job.
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+    if (RT_FAILURE(rc))
+        return rc;
+    pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
+    *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
+ * own the PGM lock and have access to the page structure.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be mapped.
+ * @param   pPage       Pointer to the PGMPAGE structure for the page.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that
+ *                      pgmPhysReleaseInternalPageMappingLock needs.
+ *
+ * @internal
+ */
+int pgmPhysGCPhys2CCPtrInternalReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0);
+
+    /*
+     * Do the job.
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+    if (RT_FAILURE(rc))
+        return rc;
+    pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
+    *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Requests the mapping of a guest page into the current context.
+ *
+ * This API should only be used for very short term, as it will consume scarse
+ * resources (R0 and GC) in the mapping cache. When you're done with the page,
+ * call PGMPhysReleasePageMappingLock() ASAP to release it.
+ *
+ * This API will assume your intention is to write to the page, and will
+ * therefore replace shared and zero pages. If you do not intend to modify
+ * the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be
+ *                      mapped.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that
+ *                      PGMPhysReleasePageMappingLock needs.
+ *
+ * @remarks The caller is responsible for dealing with access handlers.
+ * @todo    Add an informational return code for pages with access handlers?
+ *
+ * @remark  Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk. External threads may
+ *          need to delegate jobs to the EMTs.
+ * @remarks Only one page is mapped!  Make no assumption about what's after or
+ *          before the returned page!
+ * @thread  Any thread.
+ */
+VMM_INT_DECL(int) PGMPhysGCPhys2CCPtr(PVMCC pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    int rc = PGM_LOCK(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Query the Physical TLB entry for the page (may fail).
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * If the page is shared, the zero page, or being write monitored
+         * it must be converted to a page that's writable if possible.
+         */
+        PPGMPAGE pPage = pTlbe->pPage;
+        if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
+        {
+            rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+            if (RT_SUCCESS(rc))
+            {
+                AssertMsg(rc == VINF_SUCCESS || rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
+                rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+            }
+        }
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * Now, just perform the locking and calculate the return address.
+             */
+            pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
+        }
+    }
+
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of a guest page into the current context.
+ *
+ * This API should only be used for very short term, as it will consume scarse
+ * resources (R0 and GC) in the mapping cache.  When you're done with the page,
+ * call PGMPhysReleasePageMappingLock() ASAP to release it.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The guest physical address of the page that should be
+ *                      mapped.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that
+ *                      PGMPhysReleasePageMappingLock needs.
+ *
+ * @remarks The caller is responsible for dealing with access handlers.
+ * @todo    Add an informational return code for pages with access handlers?
+ *
+ * @remarks Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk.
+ * @remarks Only one page is mapped!  Make no assumption about what's after or
+ *          before the returned page!
+ * @thread  Any thread.
+ */
+VMM_INT_DECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVMCC pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    int rc = PGM_LOCK(pVM);
+    AssertRCReturn(rc, rc);
+
+    /*
+     * Query the Physical TLB entry for the page (may fail).
+     */
+    PPGMPAGEMAPTLBE pTlbe;
+    rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
+    if (RT_SUCCESS(rc))
+    {
+        /* MMIO pages doesn't have any readable backing. */
+        PPGMPAGE pPage = pTlbe->pPage;
+        if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
+            rc = VERR_PGM_PHYS_PAGE_RESERVED;
+        else
+        {
+            /*
+             * Now, just perform the locking and calculate the return address.
+             */
+            pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
+        }
+    }
+
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of a guest page given by virtual address into the current context.
+ *
+ * This API should only be used for very short term, as it will consume
+ * scarse resources (R0 and GC) in the mapping cache. When you're done
+ * with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
+ *
+ * This API will assume your intention is to write to the page, and will
+ * therefore replace shared and zero pages. If you do not intend to modify
+ * the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
+ * @retval  VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The guest physical address of the page that should be
+ *                      mapped.
+ * @param   ppv         Where to store the address corresponding to GCPhys.
+ * @param   pLock       Where to store the lock information that PGMPhysReleasePageMappingLock needs.
+ *
+ * @remark  Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk.
+ * @thread  EMT
+ */
+VMM_INT_DECL(int) PGMPhysGCPtr2CCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
+    RTGCPHYS GCPhys;
+    int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
+    if (RT_SUCCESS(rc))
+        rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
+    return rc;
+}
+
+
+/**
+ * Requests the mapping of a guest page given by virtual address into the current context.
+ *
+ * This API should only be used for very short term, as it will consume
+ * scarse resources (R0 and GC) in the mapping cache. When you're done
+ * with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
+ * @retval  VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
+ * @retval  VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
+ * @retval  VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The guest physical address of the page that should be
+ *                      mapped.
+ * @param   ppv         Where to store the address corresponding to GCPtr.
+ * @param   pLock       Where to store the lock information that PGMPhysReleasePageMappingLock needs.
+ *
+ * @remark  Avoid calling this API from within critical sections (other than
+ *          the PGM one) because of the deadlock risk.
+ * @thread  EMT
+ */
+VMM_INT_DECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPUCC pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
+    RTGCPHYS GCPhys;
+    int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
+    if (RT_SUCCESS(rc))
+        rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
+    return rc;
+}
+
+
+/**
+ * Release the mapping of a guest page.
+ *
+ * This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
+ * PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pLock       The lock structure initialized by the mapping function.
+ */
+VMMDECL(void) PGMPhysReleasePageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock)
+{
+# ifndef IN_RING0
+    PPGMPAGEMAP pMap       = (PPGMPAGEMAP)pLock->pvMap;
+# endif
+    PPGMPAGE    pPage      = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
+    bool        fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
+
+    pLock->uPageAndType = 0;
+    pLock->pvMap = NULL;
+
+    PGM_LOCK_VOID(pVM);
+    if (fWriteLock)
+    {
+        unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
+        Assert(cLocks > 0);
+        if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
+        {
+            if (cLocks == 1)
+            {
+                Assert(pVM->pgm.s.cWriteLockedPages > 0);
+                pVM->pgm.s.cWriteLockedPages--;
+            }
+            PGM_PAGE_DEC_WRITE_LOCKS(pPage);
+        }
+
+        if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
+        { /* probably extremely likely */ }
+        else
+            pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
+    }
+    else
+    {
+        unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
+        Assert(cLocks > 0);
+        if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
+        {
+            if (cLocks == 1)
+            {
+                Assert(pVM->pgm.s.cReadLockedPages > 0);
+                pVM->pgm.s.cReadLockedPages--;
+            }
+            PGM_PAGE_DEC_READ_LOCKS(pPage);
+        }
+    }
+
+# ifndef IN_RING0
+    if (pMap)
+    {
+        Assert(pMap->cRefs >= 1);
+        pMap->cRefs--;
+    }
+# endif
+    PGM_UNLOCK(pVM);
+}
+
+
+#ifdef IN_RING3
+/**
+ * Release the mapping of multiple guest pages.
+ *
+ * This is the counter part to PGMR3PhysBulkGCPhys2CCPtrExternal() and
+ * PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal().
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   cPages      Number of pages to unlock.
+ * @param   paLocks     Array of locks lock structure initialized by the mapping
+ *                      function.
+ */
+VMMDECL(void) PGMPhysBulkReleasePageMappingLocks(PVMCC pVM, uint32_t cPages, PPGMPAGEMAPLOCK paLocks)
+{
+    Assert(cPages > 0);
+    bool const fWriteLock = (paLocks[0].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
+#ifdef VBOX_STRICT
+    for (uint32_t i = 1; i < cPages; i++)
+    {
+        Assert(fWriteLock == ((paLocks[i].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE));
+        AssertPtr(paLocks[i].uPageAndType);
+    }
+#endif
+
+    PGM_LOCK_VOID(pVM);
+    if (fWriteLock)
+    {
+        /*
+         * Write locks:
+         */
+        for (uint32_t i = 0; i < cPages; i++)
+        {
+            PPGMPAGE pPage  = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
+            unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
+            Assert(cLocks > 0);
+            if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
+            {
+                if (cLocks == 1)
+                {
+                    Assert(pVM->pgm.s.cWriteLockedPages > 0);
+                    pVM->pgm.s.cWriteLockedPages--;
+                }
+                PGM_PAGE_DEC_WRITE_LOCKS(pPage);
+            }
+
+            if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
+            { /* probably extremely likely */ }
+            else
+                pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
+
+            PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap;
+            if (pMap)
+            {
+                Assert(pMap->cRefs >= 1);
+                pMap->cRefs--;
+            }
+
+            /* Yield the lock: */
+            if ((i & 1023) == 1023 && i + 1 < cPages)
+            {
+                PGM_UNLOCK(pVM);
+                PGM_LOCK_VOID(pVM);
+            }
+        }
+    }
+    else
+    {
+        /*
+         * Read locks:
+         */
+        for (uint32_t i = 0; i < cPages; i++)
+        {
+            PPGMPAGE pPage  = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
+            unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
+            Assert(cLocks > 0);
+            if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
+            {
+                if (cLocks == 1)
+                {
+                    Assert(pVM->pgm.s.cReadLockedPages > 0);
+                    pVM->pgm.s.cReadLockedPages--;
+                }
+                PGM_PAGE_DEC_READ_LOCKS(pPage);
+            }
+
+            PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap;
+            if (pMap)
+            {
+                Assert(pMap->cRefs >= 1);
+                pMap->cRefs--;
+            }
+
+            /* Yield the lock: */
+            if ((i & 1023) == 1023 && i + 1 < cPages)
+            {
+                PGM_UNLOCK(pVM);
+                PGM_LOCK_VOID(pVM);
+            }
+        }
+    }
+    PGM_UNLOCK(pVM);
+
+    RT_BZERO(paLocks, sizeof(paLocks[0]) * cPages);
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Release the internal mapping of a guest page.
+ *
+ * This is the counter part of pgmPhysGCPhys2CCPtrInternalEx and
+ * pgmPhysGCPhys2CCPtrInternalReadOnly.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pLock       The lock structure initialized by the mapping function.
+ *
+ * @remarks Caller must hold the PGM lock.
+ */
+void pgmPhysReleaseInternalPageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */
+}
+
+
+/**
+ * Converts a GC physical address to a HC ring-3 pointer.
+ *
+ * @returns VINF_SUCCESS on success.
+ * @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
+ *          page but has no physical backing.
+ * @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
+ *          GC physical address.
+ * @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
+ *          a dynamic ram chunk boundary
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address to convert.
+ * @param   pR3Ptr      Where to store the R3 pointer on success.
+ *
+ * @deprecated  Avoid when possible!
+ */
+int pgmPhysGCPhys2R3Ptr(PVMCC pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
+{
+/** @todo this is kind of hacky and needs some more work. */
+#ifndef DEBUG_sandervl
+    VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
+#endif
+
+    Log(("pgmPhysGCPhys2R3Ptr(,%RGp,): dont use this API!\n", GCPhys)); /** @todo eliminate this API! */
+    PGM_LOCK_VOID(pVM);
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+        rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
+
+    PGM_UNLOCK(pVM);
+    Assert(rc <= VINF_SUCCESS);
+    return rc;
+}
+
+
+/**
+ * Converts a guest pointer to a GC physical address.
+ *
+ * This uses the current CR3/CR0/CR4 of the guest.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The guest pointer to convert.
+ * @param   pGCPhys     Where to store the GC physical address.
+ */
+VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
+{
+    PGMPTWALK Walk;
+    int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, &Walk);
+    if (pGCPhys && RT_SUCCESS(rc))
+        *pGCPhys = Walk.GCPhys | ((RTGCUINTPTR)GCPtr & GUEST_PAGE_OFFSET_MASK);
+    return rc;
+}
+
+
+/**
+ * Converts a guest pointer to a HC physical address.
+ *
+ * This uses the current CR3/CR0/CR4 of the guest.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       The guest pointer to convert.
+ * @param   pHCPhys     Where to store the HC physical address.
+ */
+VMM_INT_DECL(int) PGMPhysGCPtr2HCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
+{
+    PVMCC     pVM = pVCpu->CTX_SUFF(pVM);
+    PGMPTWALK Walk;
+    int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, &Walk);
+    if (RT_SUCCESS(rc))
+        rc = PGMPhysGCPhys2HCPhys(pVM, Walk.GCPhys | ((RTGCUINTPTR)GCPtr & GUEST_PAGE_OFFSET_MASK), pHCPhys);
+    return rc;
+}
+
+
+
+#undef LOG_GROUP
+#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
+
+
+#if defined(IN_RING3) && defined(SOME_UNUSED_FUNCTION)
+/**
+ * Cache PGMPhys memory access
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pCache          Cache structure pointer
+ * @param   GCPhys          GC physical address
+ * @param   pbR3            HC pointer corresponding to physical page
+ *
+ * @thread  EMT.
+ */
+static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE *pCache, RTGCPHYS GCPhys, uint8_t *pbR3)
+{
+    uint32_t iCacheIndex;
+
+    Assert(VM_IS_EMT(pVM));
+
+    GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+    pbR3   = (uint8_t *)((uintptr_t)pbR3 & ~(uintptr_t)GUEST_PAGE_OFFSET_MASK);
+
+    iCacheIndex = ((GCPhys >> GUEST_PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
+
+    ASMBitSet(&pCache->aEntries, iCacheIndex);
+
+    pCache->Entry[iCacheIndex].GCPhys = GCPhys;
+    pCache->Entry[iCacheIndex].pbR3   = pbR3;
+}
+#endif /* IN_RING3 */
+
+
+/**
+ * Deals with reading from a page with one or more ALL access handlers.
+ *
+ * @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
+ *          See PGM_HANDLER_PHYS_IS_VALID_STATUS and
+ *          PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The page descriptor.
+ * @param   GCPhys      The physical address to start reading at.
+ * @param   pvBuf       Where to put the bits we read.
+ * @param   cb          How much to read - less or equal to a page.
+ * @param   enmOrigin   The origin of this call.
+ */
+static VBOXSTRICTRC pgmPhysReadHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb,
+                                       PGMACCESSORIGIN enmOrigin)
+{
+    /*
+     * The most frequent access here is MMIO and shadowed ROM.
+     * The current code ASSUMES all these access handlers covers full pages!
+     */
+
+    /*
+     * Whatever we do we need the source page, map it first.
+     */
+    PGMPAGEMAPLOCK PgMpLck;
+    const void    *pvSrc = NULL;
+    int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck);
+/** @todo Check how this can work for MMIO pages? */
+    if (RT_FAILURE(rc))
+    {
+        AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                               GCPhys, pPage, rc));
+        memset(pvBuf, 0xff, cb);
+        return VINF_SUCCESS;
+    }
+
+    VBOXSTRICTRC rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
+
+    /*
+     * Deal with any physical handlers.
+     */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    if (   PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL
+        || PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
+    {
+        PPGMPHYSHANDLER pCur;
+        rc = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur);
+        if (RT_SUCCESS(rc))
+        {
+            Assert(pCur && GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast);
+            Assert((pCur->Key     & GUEST_PAGE_OFFSET_MASK) == 0);
+            Assert((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK);
+#ifndef IN_RING3
+            if (enmOrigin != PGMACCESSORIGIN_IEM)
+            {
+                /* Cannot reliably handle informational status codes in this context */
+                pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                return VERR_PGM_PHYS_WR_HIT_HANDLER;
+            }
+#endif
+            PCPGMPHYSHANDLERTYPEINT const pCurType   = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+            PFNPGMPHYSHANDLER const       pfnHandler = pCurType->pfnHandler; Assert(pfnHandler);
+            uint64_t const                uUser      = !pCurType->fRing0DevInsIdx ? pCur->uUser
+                                                     : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
+
+            Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pCur->pszDesc) ));
+            STAM_PROFILE_START(&pCur->Stat, h);
+            PGM_LOCK_ASSERT_OWNER(pVM);
+
+            /* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
+            PGM_UNLOCK(pVM);
+            rcStrict = pfnHandler(pVM, pVCpu, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, enmOrigin, uUser);
+            PGM_LOCK_VOID(pVM);
+
+            STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
+            pCur = NULL; /* might not be valid anymore. */
+            AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, false),
+                            ("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys));
+            if (   rcStrict != VINF_PGM_HANDLER_DO_DEFAULT
+                && !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+            {
+                pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                return rcStrict;
+            }
+        }
+        else if (rc == VERR_NOT_FOUND)
+            AssertLogRelMsgFailed(("rc=%Rrc GCPhys=%RGp cb=%#x\n", rc, GCPhys, cb));
+        else
+            AssertLogRelMsgFailedReturn(("rc=%Rrc GCPhys=%RGp cb=%#x\n", rc, GCPhys, cb), rc);
+    }
+
+    /*
+     * Take the default action.
+     */
+    if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
+    {
+        memcpy(pvBuf, pvSrc, cb);
+        rcStrict = VINF_SUCCESS;
+    }
+    pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+    return rcStrict;
+}
+
+
+/**
+ * Read physical memory.
+ *
+ * This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
+ * want to ignore those.
+ *
+ * @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
+ *          code in ring-3.  Use PGM_PHYS_RW_IS_SUCCESS to check.
+ * @retval  VINF_SUCCESS in all context - read completed.
+ *
+ * @retval  VINF_EM_OFF in RC and R0 - read completed.
+ * @retval  VINF_EM_SUSPEND in RC and R0 - read completed.
+ * @retval  VINF_EM_RESET in RC and R0 - read completed.
+ * @retval  VINF_EM_HALT in RC and R0 - read completed.
+ * @retval  VINF_SELM_SYNC_GDT in RC only - read completed.
+ *
+ * @retval  VINF_EM_DBG_STOP in RC and R0 - read completed.
+ * @retval  VINF_EM_DBG_BREAKPOINT in RC and R0 - read completed.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
+ *
+ * @retval  VINF_IOM_R3_MMIO_READ in RC and R0.
+ * @retval  VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
+ *
+ * @retval  VINF_PATM_CHECK_PATCH_PAGE in RC only.
+ *
+ * @retval  VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
+ *          haven't been cleared for strict status codes yet.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Physical address start reading from.
+ * @param   pvBuf           Where to put the read bits.
+ * @param   cbRead          How many bytes to read.
+ * @param   enmOrigin       The origin of this call.
+ */
+VMMDECL(VBOXSTRICTRC) PGMPhysRead(PVMCC pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
+{
+    AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
+    LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
+
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysRead));
+    STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysReadBytes), cbRead);
+
+    PGM_LOCK_VOID(pVM);
+
+    /*
+     * Copy loop on ram ranges.
+     */
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
+    for (;;)
+    {
+        /* Inside range or not? */
+        if (pRam && GCPhys >= pRam->GCPhys)
+        {
+            /*
+             * Must work our way thru this page by page.
+             */
+            RTGCPHYS off = GCPhys - pRam->GCPhys;
+            while (off < pRam->cb)
+            {
+                unsigned iPage = off >> GUEST_PAGE_SHIFT;
+                PPGMPAGE pPage = &pRam->aPages[iPage];
+                size_t   cb    = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
+                if (cb > cbRead)
+                    cb = cbRead;
+
+                /*
+                 * Normal page? Get the pointer to it.
+                 */
+                if (   !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
+                    && !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+                {
+                    /*
+                     * Get the pointer to the page.
+                     */
+                    PGMPAGEMAPLOCK PgMpLck;
+                    const void    *pvSrc;
+                    int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
+                    if (RT_SUCCESS(rc))
+                    {
+                        memcpy(pvBuf, pvSrc, cb);
+                        pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                    }
+                    else
+                    {
+                        AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                                               pRam->GCPhys + off, pPage, rc));
+                        memset(pvBuf, 0xff, cb);
+                    }
+                }
+                /*
+                 * Have ALL/MMIO access handlers.
+                 */
+                else
+                {
+                    VBOXSTRICTRC rcStrict2 = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
+                    if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
+                        PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
+                    else
+                    {
+                        /* Set the remaining buffer to a known value. */
+                        memset(pvBuf, 0xff, cbRead);
+                        PGM_UNLOCK(pVM);
+                        return rcStrict2;
+                    }
+                }
+
+                /* next page */
+                if (cb >= cbRead)
+                {
+                    PGM_UNLOCK(pVM);
+                    return rcStrict;
+                }
+                cbRead -= cb;
+                off    += cb;
+                pvBuf   = (char *)pvBuf + cb;
+            } /* walk pages in ram range. */
+
+            GCPhys = pRam->GCPhysLast + 1;
+        }
+        else
+        {
+            LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
+
+            /*
+             * Unassigned address space.
+             */
+            size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
+            if (cb >= cbRead)
+            {
+                memset(pvBuf, 0xff, cbRead);
+                break;
+            }
+            memset(pvBuf, 0xff, cb);
+
+            cbRead -= cb;
+            pvBuf   = (char *)pvBuf + cb;
+            GCPhys += cb;
+        }
+
+        /* Advance range if necessary. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
+    } /* Ram range walk */
+
+    PGM_UNLOCK(pVM);
+    return rcStrict;
+}
+
+
+/**
+ * Deals with writing to a page with one or more WRITE or ALL access handlers.
+ *
+ * @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
+ *          See PGM_HANDLER_PHYS_IS_VALID_STATUS and
+ *          PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPage       The page descriptor.
+ * @param   GCPhys      The physical address to start writing at.
+ * @param   pvBuf       What to write.
+ * @param   cbWrite     How much to write - less or equal to a page.
+ * @param   enmOrigin   The origin of this call.
+ */
+static VBOXSTRICTRC pgmPhysWriteHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite,
+                                        PGMACCESSORIGIN enmOrigin)
+{
+    PGMPAGEMAPLOCK  PgMpLck;
+    void           *pvDst = NULL;
+    VBOXSTRICTRC    rcStrict;
+
+    /*
+     * Give priority to physical handlers (like #PF does).
+     *
+     * Hope for a lonely physical handler first that covers the whole write
+     * area.  This should be a pretty frequent case with MMIO and the heavy
+     * usage of full page handlers in the page pool.
+     */
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    PPGMPHYSHANDLER pCur;
+    rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur);
+    if (RT_SUCCESS(rcStrict))
+    {
+        Assert(GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast);
+#ifndef IN_RING3
+        if (enmOrigin != PGMACCESSORIGIN_IEM)
+            /* Cannot reliably handle informational status codes in this context */
+            return VERR_PGM_PHYS_WR_HIT_HANDLER;
+#endif
+        size_t cbRange = pCur->KeyLast - GCPhys + 1;
+        if (cbRange > cbWrite)
+            cbRange = cbWrite;
+
+        Assert(PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->pfnHandler);
+        Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n",
+              GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
+        if (!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
+            rcStrict = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
+        else
+            rcStrict = VINF_SUCCESS;
+        if (RT_SUCCESS(rcStrict))
+        {
+            PCPGMPHYSHANDLERTYPEINT const pCurType   = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
+            PFNPGMPHYSHANDLER const       pfnHandler = pCurType->pfnHandler;
+            uint64_t const                uUser      = !pCurType->fRing0DevInsIdx ? pCur->uUser
+                                                     : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
+            STAM_PROFILE_START(&pCur->Stat, h);
+
+            /* Most handlers will want to release the PGM lock for deadlock prevention
+               (esp. MMIO), though some PGM internal ones like the page pool and MMIO2
+               dirty page trackers will want to keep it for performance reasons. */
+            PGM_LOCK_ASSERT_OWNER(pVM);
+            if (pCurType->fKeepPgmLock)
+                rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
+            else
+            {
+                PGM_UNLOCK(pVM);
+                rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
+                PGM_LOCK_VOID(pVM);
+            }
+
+            STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
+            pCur = NULL; /* might not be valid anymore. */
+            if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
+            {
+                if (pvDst)
+                    memcpy(pvDst, pvBuf, cbRange);
+                rcStrict = VINF_SUCCESS;
+            }
+            else
+                AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, true),
+                                ("rcStrict=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n",
+                                 VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pCur ? R3STRING(pCur->pszDesc) : ""));
+        }
+        else
+            AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                                         GCPhys, pPage, VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
+        if (RT_LIKELY(cbRange == cbWrite) || !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
+        {
+            if (pvDst)
+                pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+            return rcStrict;
+        }
+
+        /* more fun to be had below */
+        cbWrite -= cbRange;
+        GCPhys  += cbRange;
+        pvBuf    = (uint8_t *)pvBuf + cbRange;
+        pvDst    = (uint8_t *)pvDst + cbRange;
+    }
+    else if (rcStrict == VERR_NOT_FOUND) /* The handler is somewhere else in the page, deal with it below. */
+        rcStrict = VINF_SUCCESS;
+    else
+        AssertMsgFailedReturn(("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
+    Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); /* MMIO handlers are all GUEST_PAGE_SIZEed! */
+
+    /*
+     * Deal with all the odd ends (used to be deal with virt+phys).
+     */
+    Assert(rcStrict != VINF_PGM_HANDLER_DO_DEFAULT);
+
+    /* We need a writable destination page. */
+    if (!pvDst)
+    {
+        int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
+        AssertLogRelMsgReturn(RT_SUCCESS(rc2),
+                              ("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n", GCPhys, pPage, rc2),
+                              rc2);
+    }
+
+    /** @todo clean up this code some more now there are no virtual handlers any
+     *        more. */
+    /* The loop state (big + ugly). */
+    PPGMPHYSHANDLER pPhys       = NULL;
+    uint32_t        offPhys     = GUEST_PAGE_SIZE;
+    uint32_t        offPhysLast = GUEST_PAGE_SIZE;
+    bool            fMorePhys   = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
+
+    /* The loop. */
+    for (;;)
+    {
+        if (fMorePhys && !pPhys)
+        {
+            rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhys, &pPhys);
+            if (RT_SUCCESS_NP(rcStrict))
+            {
+                offPhys = 0;
+                offPhysLast = pPhys->KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
+            }
+            else
+            {
+                AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
+
+                rcStrict = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator,
+                                                                                        GCPhys, &pPhys);
+                AssertMsgReturn(RT_SUCCESS(rcStrict) || rcStrict == VERR_NOT_FOUND,
+                                ("%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
+
+                if (   RT_SUCCESS(rcStrict)
+                    && pPhys->Key <= GCPhys + (cbWrite - 1))
+                {
+                    offPhys     = pPhys->Key     - GCPhys;
+                    offPhysLast = pPhys->KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
+                    Assert(pPhys->KeyLast - pPhys->Key < _4G);
+                }
+                else
+                {
+                    pPhys     = NULL;
+                    fMorePhys = false;
+                    offPhys   = offPhysLast = GUEST_PAGE_SIZE;
+                }
+            }
+        }
+
+        /*
+         * Handle access to space without handlers (that's easy).
+         */
+        VBOXSTRICTRC rcStrict2 = VINF_PGM_HANDLER_DO_DEFAULT;
+        uint32_t cbRange = (uint32_t)cbWrite;
+        Assert(cbRange == cbWrite);
+
+        /*
+         * Physical handler.
+         */
+        if (!offPhys)
+        {
+#ifndef IN_RING3
+            if (enmOrigin != PGMACCESSORIGIN_IEM)
+                /* Cannot reliably handle informational status codes in this context */
+                return VERR_PGM_PHYS_WR_HIT_HANDLER;
+#endif
+            if (cbRange > offPhysLast + 1)
+                cbRange = offPhysLast + 1;
+
+            PCPGMPHYSHANDLERTYPEINT const pCurType   = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pPhys);
+            PFNPGMPHYSHANDLER const       pfnHandler = pCurType->pfnHandler;
+            uint64_t const                uUser      = !pCurType->fRing0DevInsIdx ? pPhys->uUser
+                                                     : (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pPhys->uUser);
+
+            Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
+            STAM_PROFILE_START(&pPhys->Stat, h);
+
+            /* Most handlers will want to release the PGM lock for deadlock prevention
+               (esp. MMIO), though some PGM internal ones like the page pool and MMIO2
+               dirty page trackers will want to keep it for performance reasons. */
+            PGM_LOCK_ASSERT_OWNER(pVM);
+            if (pCurType->fKeepPgmLock)
+                rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
+            else
+            {
+                PGM_UNLOCK(pVM);
+                rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
+                PGM_LOCK_VOID(pVM);
+            }
+
+            STAM_PROFILE_STOP(&pPhys->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
+            pPhys = NULL; /* might not be valid anymore. */
+            AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict2, true),
+                            ("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
+                             VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage,  pPhys ? R3STRING(pPhys->pszDesc) : ""));
+        }
+
+        /*
+         * Execute the default action and merge the status codes.
+         */
+        if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
+        {
+            memcpy(pvDst, pvBuf, cbRange);
+            rcStrict2 = VINF_SUCCESS;
+        }
+        else if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
+        {
+            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+            return rcStrict2;
+        }
+        else
+            PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
+
+        /*
+         * Advance if we've got more stuff to do.
+         */
+        if (cbRange >= cbWrite)
+        {
+            pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+            return rcStrict;
+        }
+
+
+        cbWrite         -= cbRange;
+        GCPhys          += cbRange;
+        pvBuf            = (uint8_t *)pvBuf + cbRange;
+        pvDst            = (uint8_t *)pvDst + cbRange;
+
+        offPhys         -= cbRange;
+        offPhysLast     -= cbRange;
+    }
+}
+
+
+/**
+ * Write to physical memory.
+ *
+ * This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
+ * want to ignore those.
+ *
+ * @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
+ *          code in ring-3.  Use PGM_PHYS_RW_IS_SUCCESS to check.
+ * @retval  VINF_SUCCESS in all context - write completed.
+ *
+ * @retval  VINF_EM_OFF in RC and R0 - write completed.
+ * @retval  VINF_EM_SUSPEND in RC and R0 - write completed.
+ * @retval  VINF_EM_RESET in RC and R0 - write completed.
+ * @retval  VINF_EM_HALT in RC and R0 - write completed.
+ * @retval  VINF_SELM_SYNC_GDT in RC only - write completed.
+ *
+ * @retval  VINF_EM_DBG_STOP in RC and R0 - write completed.
+ * @retval  VINF_EM_DBG_BREAKPOINT in RC and R0 - write completed.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
+ *
+ * @retval  VINF_IOM_R3_MMIO_WRITE in RC and R0.
+ * @retval  VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
+ * @retval  VINF_IOM_R3_MMIO_COMMIT_WRITE in RC and R0.
+ *
+ * @retval  VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT in RC only - write completed.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT in RC only.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT in RC only.
+ * @retval  VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT in RC only.
+ * @retval  VINF_CSAM_PENDING_ACTION in RC only.
+ * @retval  VINF_PATM_CHECK_PATCH_PAGE in RC only.
+ *
+ * @retval  VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
+ *          haven't been cleared for strict status codes yet.
+ *
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Physical address to write to.
+ * @param   pvBuf           What to write.
+ * @param   cbWrite         How many bytes to write.
+ * @param   enmOrigin       Who is calling.
+ */
+VMMDECL(VBOXSTRICTRC) PGMPhysWrite(PVMCC pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
+{
+    AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()! enmOrigin=%d\n", enmOrigin));
+    AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
+    LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
+
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysWrite));
+    STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
+
+    PGM_LOCK_VOID(pVM);
+
+    /*
+     * Copy loop on ram ranges.
+     */
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
+    for (;;)
+    {
+        /* Inside range or not? */
+        if (pRam && GCPhys >= pRam->GCPhys)
+        {
+            /*
+             * Must work our way thru this page by page.
+             */
+            RTGCPTR off = GCPhys - pRam->GCPhys;
+            while (off < pRam->cb)
+            {
+                RTGCPTR     iPage = off >> GUEST_PAGE_SHIFT;
+                PPGMPAGE    pPage = &pRam->aPages[iPage];
+                size_t      cb    = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
+                if (cb > cbWrite)
+                    cb = cbWrite;
+
+                /*
+                 * Normal page? Get the pointer to it.
+                 */
+                if (   !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
+                    && !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+                {
+                    PGMPAGEMAPLOCK PgMpLck;
+                    void          *pvDst;
+                    int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
+                    if (RT_SUCCESS(rc))
+                    {
+                        Assert(!PGM_PAGE_IS_BALLOONED(pPage));
+                        memcpy(pvDst, pvBuf, cb);
+                        pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
+                    }
+                    /* Ignore writes to ballooned pages. */
+                    else if (!PGM_PAGE_IS_BALLOONED(pPage))
+                        AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
+                                                pRam->GCPhys + off, pPage, rc));
+                }
+                /*
+                 * Active WRITE or ALL access handlers.
+                 */
+                else
+                {
+                    VBOXSTRICTRC rcStrict2 = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
+                    if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
+                        PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
+                    else
+                    {
+                        PGM_UNLOCK(pVM);
+                        return rcStrict2;
+                    }
+                }
+
+                /* next page */
+                if (cb >= cbWrite)
+                {
+                    PGM_UNLOCK(pVM);
+                    return rcStrict;
+                }
+
+                cbWrite -= cb;
+                off     += cb;
+                pvBuf    = (const char *)pvBuf + cb;
+            } /* walk pages in ram range */
+
+            GCPhys = pRam->GCPhysLast + 1;
+        }
+        else
+        {
+            /*
+             * Unassigned address space, skip it.
+             */
+            if (!pRam)
+                break;
+            size_t cb = pRam->GCPhys - GCPhys;
+            if (cb >= cbWrite)
+                break;
+            cbWrite -=                       cb;
+            pvBuf    = (const char *)pvBuf + cb;
+            GCPhys  +=                       cb;
+        }
+
+        /* Advance range if necessary. */
+        while (pRam && GCPhys > pRam->GCPhysLast)
+            pRam = pRam->CTX_SUFF(pNext);
+    } /* Ram range walk */
+
+    PGM_UNLOCK(pVM);
+    return rcStrict;
+}
+
+
+/**
+ * Read from guest physical memory by GC physical address, bypassing
+ * MMIO and access handlers.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvDst       The destination address.
+ * @param   GCPhysSrc   The source address (GC physical address).
+ * @param   cb          The number of bytes to read.
+ */
+VMMDECL(int) PGMPhysSimpleReadGCPhys(PVMCC pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
+{
+    /*
+     * Treat the first page as a special case.
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    /* map the 1st page */
+    void const *pvSrc;
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* optimize for the case where access is completely within the first page. */
+    size_t cbPage = GUEST_PAGE_SIZE - (GCPhysSrc & GUEST_PAGE_OFFSET_MASK);
+    if (RT_LIKELY(cb <= cbPage))
+    {
+        memcpy(pvDst, pvSrc, cb);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        return VINF_SUCCESS;
+    }
+
+    /* copy to the end of the page. */
+    memcpy(pvDst, pvSrc, cbPage);
+    PGMPhysReleasePageMappingLock(pVM, &Lock);
+    GCPhysSrc += cbPage;
+    pvDst = (uint8_t *)pvDst + cbPage;
+    cb -= cbPage;
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* map the page */
+        rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* last page? */
+        if (cb <= GUEST_PAGE_SIZE)
+        {
+            memcpy(pvDst, pvSrc, cb);
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+            return VINF_SUCCESS;
+        }
+
+        /* copy the entire page and advance */
+        memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        GCPhysSrc +=                    GUEST_PAGE_SIZE;
+        pvDst      = (uint8_t *)pvDst + GUEST_PAGE_SIZE;
+        cb        -=                    GUEST_PAGE_SIZE;
+    }
+    /* won't ever get here. */
+}
+
+
+/**
+ * Write to guest physical memory referenced by GC pointer.
+ * Write memory to GC physical address in guest physical memory.
+ *
+ * This will bypass MMIO and access handlers.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhysDst   The GC physical address of the destination.
+ * @param   pvSrc       The source buffer.
+ * @param   cb          The number of bytes to write.
+ */
+VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVMCC pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
+{
+    LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
+
+    /*
+     * Treat the first page as a special case.
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    /* map the 1st page */
+    void *pvDst;
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* optimize for the case where access is completely within the first page. */
+    size_t cbPage = GUEST_PAGE_SIZE - (GCPhysDst & GUEST_PAGE_OFFSET_MASK);
+    if (RT_LIKELY(cb <= cbPage))
+    {
+        memcpy(pvDst, pvSrc, cb);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        return VINF_SUCCESS;
+    }
+
+    /* copy to the end of the page. */
+    memcpy(pvDst, pvSrc, cbPage);
+    PGMPhysReleasePageMappingLock(pVM, &Lock);
+    GCPhysDst += cbPage;
+    pvSrc = (const uint8_t *)pvSrc + cbPage;
+    cb -= cbPage;
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* map the page */
+        rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* last page? */
+        if (cb <= GUEST_PAGE_SIZE)
+        {
+            memcpy(pvDst, pvSrc, cb);
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+            return VINF_SUCCESS;
+        }
+
+        /* copy the entire page and advance */
+        memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        GCPhysDst +=                          GUEST_PAGE_SIZE;
+        pvSrc      = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE;
+        cb        -=                          GUEST_PAGE_SIZE;
+    }
+    /* won't ever get here. */
+}
+
+
+/**
+ * Read from guest physical memory referenced by GC pointer.
+ *
+ * This function uses the current CR3/CR0/CR4 of the guest and will
+ * bypass access handlers and not set any accessed bits.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pvDst       The destination address.
+ * @param   GCPtrSrc    The source address (GC pointer).
+ * @param   cb          The number of bytes to read.
+ */
+VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+/** @todo fix the macro / state handling: VMCPU_ASSERT_EMT_OR_GURU(pVCpu); */
+
+    /*
+     * Treat the first page as a special case.
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleRead));
+    STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
+
+    /* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
+     * when many VCPUs are fighting for the lock.
+     */
+    PGM_LOCK_VOID(pVM);
+
+    /* map the 1st page */
+    void const *pvSrc;
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
+    if (RT_FAILURE(rc))
+    {
+        PGM_UNLOCK(pVM);
+        return rc;
+    }
+
+    /* optimize for the case where access is completely within the first page. */
+    size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
+    if (RT_LIKELY(cb <= cbPage))
+    {
+        memcpy(pvDst, pvSrc, cb);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        PGM_UNLOCK(pVM);
+        return VINF_SUCCESS;
+    }
+
+    /* copy to the end of the page. */
+    memcpy(pvDst, pvSrc, cbPage);
+    PGMPhysReleasePageMappingLock(pVM, &Lock);
+    GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
+    pvDst = (uint8_t *)pvDst + cbPage;
+    cb -= cbPage;
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* map the page */
+        rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
+        if (RT_FAILURE(rc))
+        {
+            PGM_UNLOCK(pVM);
+            return rc;
+        }
+
+        /* last page? */
+        if (cb <= GUEST_PAGE_SIZE)
+        {
+            memcpy(pvDst, pvSrc, cb);
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+            PGM_UNLOCK(pVM);
+            return VINF_SUCCESS;
+        }
+
+        /* copy the entire page and advance */
+        memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + GUEST_PAGE_SIZE);
+        pvDst    = (uint8_t *)pvDst                + GUEST_PAGE_SIZE;
+        cb      -=                                   GUEST_PAGE_SIZE;
+    }
+    /* won't ever get here. */
+}
+
+
+/**
+ * Write to guest physical memory referenced by GC pointer.
+ *
+ * This function uses the current CR3/CR0/CR4 of the guest and will
+ * bypass access handlers and not set dirty or accessed bits.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtrDst    The destination address (GC pointer).
+ * @param   pvSrc       The source address.
+ * @param   cb          The number of bytes to write.
+ */
+VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Treat the first page as a special case.
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleWrite));
+    STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
+
+    /* map the 1st page */
+    void *pvDst;
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* optimize for the case where access is completely within the first page. */
+    size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
+    if (RT_LIKELY(cb <= cbPage))
+    {
+        memcpy(pvDst, pvSrc, cb);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        return VINF_SUCCESS;
+    }
+
+    /* copy to the end of the page. */
+    memcpy(pvDst, pvSrc, cbPage);
+    PGMPhysReleasePageMappingLock(pVM, &Lock);
+    GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
+    pvSrc = (const uint8_t *)pvSrc + cbPage;
+    cb -= cbPage;
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* map the page */
+        rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* last page? */
+        if (cb <= GUEST_PAGE_SIZE)
+        {
+            memcpy(pvDst, pvSrc, cb);
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+            return VINF_SUCCESS;
+        }
+
+        /* copy the entire page and advance */
+        memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + GUEST_PAGE_SIZE);
+        pvSrc    = (const uint8_t *)pvSrc          + GUEST_PAGE_SIZE;
+        cb      -=                                   GUEST_PAGE_SIZE;
+    }
+    /* won't ever get here. */
+}
+
+
+/**
+ * Write to guest physical memory referenced by GC pointer and update the PTE.
+ *
+ * This function uses the current CR3/CR0/CR4 of the guest and will
+ * bypass access handlers but will set any dirty and accessed bits in the PTE.
+ *
+ * If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtrDst    The destination address (GC pointer).
+ * @param   pvSrc       The source address.
+ * @param   cb          The number of bytes to write.
+ */
+VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Treat the first page as a special case.
+     * Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    /* map the 1st page */
+    void *pvDst;
+    PGMPAGEMAPLOCK Lock;
+    int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    /* optimize for the case where access is completely within the first page. */
+    size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
+    if (RT_LIKELY(cb <= cbPage))
+    {
+        memcpy(pvDst, pvSrc, cb);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
+        return VINF_SUCCESS;
+    }
+
+    /* copy to the end of the page. */
+    memcpy(pvDst, pvSrc, cbPage);
+    PGMPhysReleasePageMappingLock(pVM, &Lock);
+    rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
+    GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
+    pvSrc = (const uint8_t *)pvSrc + cbPage;
+    cb -= cbPage;
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* map the page */
+        rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        /* last page? */
+        if (cb <= GUEST_PAGE_SIZE)
+        {
+            memcpy(pvDst, pvSrc, cb);
+            PGMPhysReleasePageMappingLock(pVM, &Lock);
+            rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
+            return VINF_SUCCESS;
+        }
+
+        /* copy the entire page and advance */
+        memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
+        PGMPhysReleasePageMappingLock(pVM, &Lock);
+        rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D)); AssertRC(rc);
+        GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + GUEST_PAGE_SIZE);
+        pvSrc    = (const uint8_t *)pvSrc          + GUEST_PAGE_SIZE;
+        cb      -=                                   GUEST_PAGE_SIZE;
+    }
+    /* won't ever get here. */
+}
+
+
+/**
+ * Read from guest physical memory referenced by GC pointer.
+ *
+ * This function uses the current CR3/CR0/CR4 of the guest and will
+ * respect access handlers and set accessed bits.
+ *
+ * @returns Strict VBox status, see PGMPhysRead for details.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
+ *          specified virtual address.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pvDst       The destination address.
+ * @param   GCPtrSrc    The source address (GC pointer).
+ * @param   cb          The number of bytes to read.
+ * @param   enmOrigin   Who is calling.
+ * @thread  EMT(pVCpu)
+ */
+VMMDECL(VBOXSTRICTRC) PGMPhysReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
+{
+    int         rc;
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Anything to do?
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
+
+    /*
+     * Optimize reads within a single page.
+     */
+    if (((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK) + cb <= GUEST_PAGE_SIZE)
+    {
+        /* Convert virtual to physical address + flags */
+        PGMPTWALK Walk;
+        rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &Walk);
+        AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
+        RTGCPHYS const GCPhys = Walk.GCPhys | ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
+
+        /* mark the guest page as accessed. */
+        if (!(Walk.fEffective & X86_PTE_A))
+        {
+            rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
+            AssertRC(rc);
+        }
+
+        return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
+    }
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* Convert virtual to physical address + flags */
+        PGMPTWALK Walk;
+        rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &Walk);
+        AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
+        RTGCPHYS const GCPhys = Walk.GCPhys | ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
+
+        /* mark the guest page as accessed. */
+        if (!(Walk.fEffective & X86_PTE_A))
+        {
+            rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
+            AssertRC(rc);
+        }
+
+        /* copy */
+        size_t cbRead = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
+        if (cbRead < cb)
+        {
+            VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, GCPhys, pvDst, cbRead, enmOrigin);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+            { /* likely */ }
+            else
+                return rcStrict;
+        }
+        else    /* Last page (cbRead is GUEST_PAGE_SIZE, we only need cb!) */
+            return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
+
+        /* next */
+        Assert(cb > cbRead);
+        cb         -= cbRead;
+        pvDst       = (uint8_t *)pvDst + cbRead;
+        GCPtrSrc   += cbRead;
+    }
+}
+
+
+/**
+ * Write to guest physical memory referenced by GC pointer.
+ *
+ * This function uses the current CR3/CR0/CR4 of the guest and will
+ * respect access handlers and set dirty and accessed bits.
+ *
+ * @returns Strict VBox status, see PGMPhysWrite for details.
+ * @retval  VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
+ *          specified virtual address.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   GCPtrDst    The destination address (GC pointer).
+ * @param   pvSrc       The source address.
+ * @param   cb          The number of bytes to write.
+ * @param   enmOrigin       Who is calling.
+ */
+VMMDECL(VBOXSTRICTRC) PGMPhysWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
+{
+    int         rc;
+    PVMCC       pVM = pVCpu->CTX_SUFF(pVM);
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    /*
+     * Anything to do?
+     */
+    if (!cb)
+        return VINF_SUCCESS;
+
+    LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
+
+    /*
+     * Optimize writes within a single page.
+     */
+    if (((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK) + cb <= GUEST_PAGE_SIZE)
+    {
+        /* Convert virtual to physical address + flags */
+        PGMPTWALK Walk;
+        rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &Walk);
+        AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
+        RTGCPHYS const GCPhys = Walk.GCPhys | ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
+
+        /* Mention when we ignore X86_PTE_RW... */
+        if (!(Walk.fEffective & X86_PTE_RW))
+            Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
+
+        /* Mark the guest page as accessed and dirty if necessary. */
+        if ((Walk.fEffective & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D))
+        {
+            rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D));
+            AssertRC(rc);
+        }
+
+        return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
+    }
+
+    /*
+     * Page by page.
+     */
+    for (;;)
+    {
+        /* Convert virtual to physical address + flags */
+        PGMPTWALK Walk;
+        rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &Walk);
+        AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
+        RTGCPHYS const GCPhys = Walk.GCPhys | ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
+
+        /* Mention when we ignore X86_PTE_RW... */
+        if (!(Walk.fEffective & X86_PTE_RW))
+            Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
+
+        /* Mark the guest page as accessed and dirty if necessary. */
+        if ((Walk.fEffective & (X86_PTE_A | X86_PTE_D)) != (X86_PTE_A | X86_PTE_D))
+        {
+            rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A | X86_PTE_D, ~(uint64_t)(X86_PTE_A | X86_PTE_D));
+            AssertRC(rc);
+        }
+
+        /* copy */
+        size_t cbWrite = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
+        if (cbWrite < cb)
+        {
+            VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite, enmOrigin);
+            if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+            { /* likely */ }
+            else
+                return rcStrict;
+        }
+        else    /* Last page (cbWrite is GUEST_PAGE_SIZE, we only need cb!) */
+            return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
+
+        /* next */
+        Assert(cb > cbWrite);
+        cb         -= cbWrite;
+        pvSrc       = (uint8_t *)pvSrc + cbWrite;
+        GCPtrDst   += cbWrite;
+    }
+}
+
+
+/**
+ * Return the page type of the specified physical address.
+ *
+ * @returns The page type.
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Guest physical address
+ */
+VMM_INT_DECL(PGMPAGETYPE) PGMPhysGetPageType(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PGM_LOCK_VOID(pVM);
+    PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
+    PGMPAGETYPE enmPgType = pPage ? (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage) : PGMPAGETYPE_INVALID;
+    PGM_UNLOCK(pVM);
+
+    return enmPgType;
+}
+
+
+/**
+ * Converts a GC physical address to a HC ring-3 pointer, with some
+ * additional checks.
+ *
+ * @returns VBox status code (no informational statuses).
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ * @param   GCPhys          The GC physical address to convert.  This API mask
+ *                          the A20 line when necessary.
+ * @param   puTlbPhysRev    Where to read the physical TLB revision.  Needs to
+ *                          be done while holding the PGM lock.
+ * @param   ppb             Where to store the pointer corresponding to GCPhys
+ *                          on success.
+ * @param   pfTlb           The TLB flags and revision.  We only add stuff.
+ *
+ * @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr and
+ *          PGMPhysIemGCPhys2Ptr.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(int) PGMPhysIemGCPhys2PtrNoLock(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint64_t const volatile *puTlbPhysRev,
+                                             R3R0PTRTYPE(uint8_t *) *ppb,
+                                             uint64_t *pfTlb)
+{
+    PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
+    Assert(!(GCPhys & X86_PAGE_OFFSET_MASK));
+
+    PGM_LOCK_VOID(pVM);
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+    {
+        if (!PGM_PAGE_IS_BALLOONED(pPage))
+        {
+            if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+            {
+                if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
+                {
+                    /*
+                     * No access handler.
+                     */
+                    switch (PGM_PAGE_GET_STATE(pPage))
+                    {
+                        case PGM_PAGE_STATE_ALLOCATED:
+                            *pfTlb |= *puTlbPhysRev;
+                            break;
+                        case PGM_PAGE_STATE_BALLOONED:
+                            AssertFailed();
+                            RT_FALL_THRU();
+                        case PGM_PAGE_STATE_ZERO:
+                        case PGM_PAGE_STATE_SHARED:
+                        case PGM_PAGE_STATE_WRITE_MONITORED:
+                            *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE;
+                            break;
+                    }
+
+                    PPGMPAGEMAPTLBE pTlbe;
+                    rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+                    AssertLogRelRCReturn(rc, rc);
+                    *ppb = (uint8_t *)pTlbe->pv;
+                }
+                else if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
+                {
+                    /*
+                     * MMIO or similar all access handler: Catch all access.
+                     */
+                    *pfTlb |= *puTlbPhysRev
+                           | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
+                    *ppb   = NULL;
+                }
+                else
+                {
+                    /*
+                     * Write access handler: Catch write accesses if active.
+                     */
+                    if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
+                        *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE;
+                    else
+                        switch (PGM_PAGE_GET_STATE(pPage))
+                        {
+                            case PGM_PAGE_STATE_ALLOCATED:
+                                *pfTlb |= *puTlbPhysRev;
+                                break;
+                            case PGM_PAGE_STATE_BALLOONED:
+                                AssertFailed();
+                                RT_FALL_THRU();
+                            case PGM_PAGE_STATE_ZERO:
+                            case PGM_PAGE_STATE_SHARED:
+                            case PGM_PAGE_STATE_WRITE_MONITORED:
+                                *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE;
+                                break;
+                        }
+
+                    PPGMPAGEMAPTLBE pTlbe;
+                    rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+                    AssertLogRelRCReturn(rc, rc);
+                    *ppb = (uint8_t *)pTlbe->pv;
+                }
+            }
+            else
+            {
+                /* Alias MMIO: For now, we catch all access.  */
+                *pfTlb |= *puTlbPhysRev
+                       |  PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
+                *ppb    = NULL;
+            }
+        }
+        else
+        {
+            /* Ballooned: Shouldn't get here, but we read zero page via PGMPhysRead and writes goes to /dev/null. */
+            *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ | PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
+            *ppb    = NULL;
+        }
+        Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp *ppb=%p *pfTlb=%#RX64 pPage=%R[pgmpage]\n", GCPhys, *ppb, *pfTlb, pPage));
+    }
+    else
+    {
+        *pfTlb |= *puTlbPhysRev | PGMIEMGCPHYS2PTR_F_NO_WRITE | PGMIEMGCPHYS2PTR_F_NO_READ
+               |  PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 | PGMIEMGCPHYS2PTR_F_UNASSIGNED;
+        *ppb    = NULL;
+        Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp *ppb=%p *pfTlb=%#RX64 (rc=%Rrc)\n", GCPhys, *ppb, *pfTlb, rc));
+    }
+
+    PGM_UNLOCK(pVM);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Converts a GC physical address to a HC ring-3 pointer, with some
+ * additional checks.
+ *
+ * @returns VBox status code (no informational statuses).
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
+ *          access handler of some kind.
+ * @retval  VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
+ *          accesses or is odd in any way.
+ * @retval  VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ * @param   GCPhys          The GC physical address to convert.  This API mask
+ *                          the A20 line when necessary.
+ * @param   fWritable       Whether write access is required.
+ * @param   fByPassHandlers Whether to bypass access handlers.
+ * @param   ppv             Where to store the pointer corresponding to GCPhys
+ *                          on success.
+ * @param   pLock
+ *
+ * @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr.
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers,
+                                       void **ppv, PPGMPAGEMAPLOCK pLock)
+{
+    PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
+
+    PGM_LOCK_VOID(pVM);
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+    {
+        if (PGM_PAGE_IS_BALLOONED(pPage))
+            rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
+        else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+            rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+        else if (   !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
+                 || (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
+            rc = VINF_SUCCESS;
+        else
+        {
+            if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
+            {
+                Assert(!fByPassHandlers || PGM_PAGE_IS_MMIO(pPage));
+                rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+            }
+            else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
+            {
+                Assert(!fByPassHandlers);
+                rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
+            }
+        }
+        if (RT_SUCCESS(rc))
+        {
+            int rc2;
+
+            /* Make sure what we return is writable. */
+            if (fWritable)
+                switch (PGM_PAGE_GET_STATE(pPage))
+                {
+                    case PGM_PAGE_STATE_ALLOCATED:
+                        break;
+                    case PGM_PAGE_STATE_BALLOONED:
+                        AssertFailed();
+                        break;
+                    case PGM_PAGE_STATE_ZERO:
+                    case PGM_PAGE_STATE_SHARED:
+                    case PGM_PAGE_STATE_WRITE_MONITORED:
+                        rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
+                        AssertLogRelRCReturn(rc2, rc2);
+                        break;
+                }
+
+            /* Get a ring-3 mapping of the address. */
+            PPGMPAGEMAPTLBE pTlbe;
+            rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
+            AssertLogRelRCReturn(rc2, rc2);
+
+            /* Lock it and calculate the address. */
+            if (fWritable)
+                pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
+            else
+                pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
+            *ppv = (void *)((uintptr_t)pTlbe->pv | (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
+
+            Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] *ppv=%p\n", GCPhys, rc, pPage, *ppv));
+        }
+        else
+            Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
+
+        /* else: handler catching all access, no pointer returned. */
+    }
+    else
+        rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
+
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+
+/**
+ * Checks if the give GCPhys page requires special handling for the given access
+ * because it's MMIO or otherwise monitored.
+ *
+ * @returns VBox status code (no informational statuses).
+ * @retval  VINF_SUCCESS on success.
+ * @retval  VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
+ *          access handler of some kind.
+ * @retval  VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
+ *          accesses or is odd in any way.
+ * @retval  VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          The GC physical address to convert.  Since this is
+ *                          only used for filling the REM TLB, the A20 mask must
+ *                          be applied before calling this API.
+ * @param   fWritable       Whether write access is required.
+ * @param   fByPassHandlers Whether to bypass access handlers.
+ *
+ * @remarks This is a watered down version PGMPhysIemGCPhys2Ptr and really just
+ *          a stop gap thing that should be removed once there is a better TLB
+ *          for virtual address accesses.
+ */
+VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVMCC pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers)
+{
+    PGM_LOCK_VOID(pVM);
+    PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
+
+    PPGMRAMRANGE pRam;
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
+    if (RT_SUCCESS(rc))
+    {
+        if (PGM_PAGE_IS_BALLOONED(pPage))
+            rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
+        else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
+            rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+        else if (   !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
+                 || (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
+            rc = VINF_SUCCESS;
+        else
+        {
+            if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
+            {
+                Assert(!fByPassHandlers || PGM_PAGE_IS_MMIO(pPage));
+                rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
+            }
+            else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
+            {
+                Assert(!fByPassHandlers);
+                rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
+            }
+        }
+    }
+
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+#ifdef VBOX_WITH_NATIVE_NEM
+
+/**
+ * Interface used by NEM to check what to do on a memory access exit.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per virtual CPU structure.
+ *                          Optional.
+ * @param   GCPhys          The guest physical address.
+ * @param   fMakeWritable   Whether to try make the page writable or not.  If it
+ *                          cannot be made writable, NEM_PAGE_PROT_WRITE won't
+ *                          be returned and the return code will be unaffected
+ * @param   pInfo           Where to return the page information.  This is
+ *                          initialized even on failure.
+ * @param   pfnChecker      Page in-sync checker callback.  Optional.
+ * @param   pvUser          User argument to pass to pfnChecker.
+ */
+VMM_INT_DECL(int) PGMPhysNemPageInfoChecker(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fMakeWritable, PPGMPHYSNEMPAGEINFO pInfo,
+                                            PFNPGMPHYSNEMCHECKPAGE pfnChecker, void *pvUser)
+{
+    PGM_LOCK_VOID(pVM);
+
+    PPGMPAGE pPage;
+    int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
+    if (RT_SUCCESS(rc))
+    {
+        /* Try make it writable if requested. */
+        pInfo->u2OldNemState = PGM_PAGE_GET_NEM_STATE(pPage);
+        if (fMakeWritable)
+            switch (PGM_PAGE_GET_STATE(pPage))
+            {
+                case PGM_PAGE_STATE_SHARED:
+                case PGM_PAGE_STATE_WRITE_MONITORED:
+                case PGM_PAGE_STATE_ZERO:
+                    rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
+                    if (rc == VERR_PGM_PHYS_PAGE_RESERVED)
+                        rc = VINF_SUCCESS;
+                    break;
+            }
+
+        /* Fill in the info. */
+        pInfo->HCPhys       = PGM_PAGE_GET_HCPHYS(pPage);
+        pInfo->u2NemState   = PGM_PAGE_GET_NEM_STATE(pPage);
+        pInfo->fHasHandlers = PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) ? 1 : 0;
+        PGMPAGETYPE const enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
+        pInfo->enmType      = enmType;
+        pInfo->fNemProt     = pgmPhysPageCalcNemProtection(pPage, enmType);
+        switch (PGM_PAGE_GET_STATE(pPage))
+        {
+            case PGM_PAGE_STATE_ALLOCATED:
+                pInfo->fZeroPage = 0;
+                break;
+
+            case PGM_PAGE_STATE_ZERO:
+                pInfo->fZeroPage = 1;
+                break;
+
+            case PGM_PAGE_STATE_WRITE_MONITORED:
+                pInfo->fZeroPage = 0;
+                break;
+
+            case PGM_PAGE_STATE_SHARED:
+                pInfo->fZeroPage = 0;
+                break;
+
+            case PGM_PAGE_STATE_BALLOONED:
+                pInfo->fZeroPage = 1;
+                break;
+
+            default:
+                pInfo->fZeroPage = 1;
+                AssertFailedStmt(rc = VERR_PGM_PHYS_PAGE_GET_IPE);
+        }
+
+        /* Call the checker and update NEM state. */
+        if (pfnChecker)
+        {
+            rc = pfnChecker(pVM, pVCpu, GCPhys, pInfo, pvUser);
+            PGM_PAGE_SET_NEM_STATE(pPage, pInfo->u2NemState);
+        }
+
+        /* Done. */
+        PGM_UNLOCK(pVM);
+    }
+    else
+    {
+        PGM_UNLOCK(pVM);
+
+        pInfo->HCPhys       = NIL_RTHCPHYS;
+        pInfo->fNemProt     = NEM_PAGE_PROT_NONE;
+        pInfo->u2NemState   = 0;
+        pInfo->fHasHandlers = 0;
+        pInfo->fZeroPage    = 0;
+        pInfo->enmType      = PGMPAGETYPE_INVALID;
+    }
+
+    return rc;
+}
+
+
+/**
+ * NEM helper that performs @a pfnCallback on pages with NEM state @a uMinState
+ * or higher.
+ *
+ * @returns VBox status code from callback.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context per CPU structure.  This is
+ *                          optional as its only for passing to callback.
+ * @param   uMinState       The minimum NEM state value to call on.
+ * @param   pfnCallback     The callback function.
+ * @param   pvUser          User argument for the callback.
+ */
+VMM_INT_DECL(int) PGMPhysNemEnumPagesByState(PVMCC pVM, PVMCPUCC pVCpu, uint8_t uMinState,
+                                             PFNPGMPHYSNEMENUMCALLBACK pfnCallback, void *pvUser)
+{
+    /*
+     * Just brute force this problem.
+     */
+    PGM_LOCK_VOID(pVM);
+    int rc = VINF_SUCCESS;
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX); pRam; pRam = pRam->CTX_SUFF(pNext))
+    {
+        uint32_t const cPages = pRam->cb >> X86_PAGE_SHIFT;
+        for (uint32_t iPage = 0; iPage < cPages; iPage++)
+        {
+            uint8_t u2State = PGM_PAGE_GET_NEM_STATE(&pRam->aPages[iPage]);
+            if (u2State < uMinState)
+            { /* likely */ }
+            else
+            {
+                rc = pfnCallback(pVM, pVCpu, pRam->GCPhys + ((RTGCPHYS)iPage << X86_PAGE_SHIFT), &u2State, pvUser);
+                if (RT_SUCCESS(rc))
+                    PGM_PAGE_SET_NEM_STATE(&pRam->aPages[iPage], u2State);
+                else
+                    break;
+            }
+        }
+    }
+    PGM_UNLOCK(pVM);
+
+    return rc;
+}
+
+
+/**
+ * Helper for setting the NEM state for a range of pages.
+ *
+ * @param   paPages     Array of pages to modify.
+ * @param   cPages      How many pages to modify.
+ * @param   u2State     The new state value.
+ */
+void pgmPhysSetNemStateForPages(PPGMPAGE paPages, RTGCPHYS cPages, uint8_t u2State)
+{
+    PPGMPAGE pPage = paPages;
+    while (cPages-- > 0)
+    {
+        PGM_PAGE_SET_NEM_STATE(pPage, u2State);
+        pPage++;
+    }
+}
+
+#endif /* VBOX_WITH_NATIVE_NEM */
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllPool.cpp b/src/VBox/VMM/VMMAll/PGMAllPool.cpp
new file mode 100644
index 00000000..59f946e5
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllPool.cpp
@@ -0,0 +1,5898 @@
+/* $Id: PGMAllPool.cpp $ */
+/** @file
+ * PGM Shadow Page Pool.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_PGM_POOL
+#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/cpum.h>
+#include "PGMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include "PGMInline.h"
+#include <VBox/disopcode.h>
+#include <VBox/vmm/hm_vmx.h>
+
+#include <VBox/log.h>
+#include <VBox/err.h>
+#include <iprt/asm.h>
+#include <iprt/string.h>
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+#if 0 /* unused */
+DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind);
+DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind);
+#endif /* unused */
+static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable);
+static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage);
+#if defined(LOG_ENABLED) || defined(VBOX_STRICT)
+static const char *pgmPoolPoolKindToStr(uint8_t enmKind);
+#endif
+#if 0 /*defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT)*/
+static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT);
+#endif
+
+int             pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage);
+PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVMCC pVM, uint16_t *piPhysExt);
+void            pgmPoolTrackPhysExtFree(PVMCC pVM, uint16_t iPhysExt);
+void            pgmPoolTrackPhysExtFreeList(PVMCC pVM, uint16_t iPhysExt);
+
+RT_C_DECLS_END
+
+
+#if 0 /* unused */
+/**
+ * Checks if the specified page pool kind is for a 4MB or 2MB guest page.
+ *
+ * @returns true if it's the shadow of a 4MB or 2MB guest page, otherwise false.
+ * @param   enmKind     The page kind.
+ */
+DECLINLINE(bool) pgmPoolIsBigPage(PGMPOOLKIND enmKind)
+{
+    switch (enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+            return true;
+        default:
+            return false;
+    }
+}
+#endif /* unused */
+
+
+/**
+ * Flushes a chain of pages sharing the same access monitor.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       A page in the chain.
+ */
+void pgmPoolMonitorChainFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    LogFlow(("pgmPoolMonitorChainFlush: Flush page %RGp type=%d\n", pPage->GCPhys, pPage->enmKind));
+
+    /*
+     * Find the list head.
+     */
+    uint16_t idx = pPage->idx;
+    if (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
+    {
+        while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
+        {
+            idx = pPage->iMonitoredPrev;
+            Assert(idx != pPage->idx);
+            pPage = &pPool->aPages[idx];
+        }
+    }
+
+    /*
+     * Iterate the list flushing each shadow page.
+     */
+    for (;;)
+    {
+        idx = pPage->iMonitoredNext;
+        Assert(idx != pPage->idx);
+        if (pPage->idx >= PGMPOOL_IDX_FIRST)
+        {
+            int rc2 = pgmPoolFlushPage(pPool, pPage);
+            AssertRC(rc2);
+        }
+        /* next */
+        if (idx == NIL_PGMPOOL_IDX)
+            break;
+        pPage = &pPool->aPages[idx];
+    }
+}
+
+
+/**
+ * Wrapper for getting the current context pointer to the entry being modified.
+ *
+ * @returns VBox status code suitable for scheduling.
+ * @param   pVM         The cross context VM structure.
+ * @param   pvDst       Destination address
+ * @param   pvSrc       Pointer to the mapping of @a GCPhysSrc or NULL depending
+ *                      on the context (e.g. \#PF in R0 & RC).
+ * @param   GCPhysSrc   The source guest physical address.
+ * @param   cb          Size of data to read
+ */
+DECLINLINE(int) pgmPoolPhysSimpleReadGCPhys(PVMCC pVM, void *pvDst, void const *pvSrc, RTGCPHYS GCPhysSrc, size_t cb)
+{
+#if defined(IN_RING3)
+    NOREF(pVM); NOREF(GCPhysSrc);
+    memcpy(pvDst, (RTHCPTR)((uintptr_t)pvSrc & ~(RTHCUINTPTR)(cb - 1)), cb);
+    return VINF_SUCCESS;
+#else
+    /** @todo in RC we could attempt to use the virtual address, although this can cause many faults (PAE Windows XP guest). */
+    NOREF(pvSrc);
+    return PGMPhysSimpleReadGCPhys(pVM, pvDst, GCPhysSrc & ~(RTGCPHYS)(cb - 1), cb);
+#endif
+}
+
+
+/**
+ * Process shadow entries before they are changed by the guest.
+ *
+ * For PT entries we will clear them. For PD entries, we'll simply check
+ * for mapping conflicts and set the SyncCR3 FF if found.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPool       The pool.
+ * @param   pPage       The head page.
+ * @param   GCPhysFault The guest physical fault address.
+ * @param   pvAddress   Pointer to the mapping of @a GCPhysFault or NULL
+ *                      depending on the context (e.g. \#PF in R0 & RC).
+ * @param   cbWrite     Write size; might be zero if the caller knows we're not crossing entry boundaries
+ */
+static void pgmPoolMonitorChainChanging(PVMCPU pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhysFault,
+                                        void const *pvAddress, unsigned cbWrite)
+{
+    AssertMsg(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX, ("%u (idx=%u)\n", pPage->iMonitoredPrev, pPage->idx));
+    const unsigned  off = GCPhysFault & GUEST_PAGE_OFFSET_MASK;
+    PVMCC           pVM = pPool->CTX_SUFF(pVM);
+    NOREF(pVCpu);
+
+    LogFlow(("pgmPoolMonitorChainChanging: %RGv phys=%RGp cbWrite=%d\n",
+             (RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
+
+    if (PGMPOOL_PAGE_IS_NESTED(pPage))
+        Log7Func(("%RGv phys=%RGp cbWrite=%d\n", (RTGCPTR)(CTXTYPE(RTGCPTR, uintptr_t, RTGCPTR))(uintptr_t)pvAddress, GCPhysFault, cbWrite));
+
+    for (;;)
+    {
+       union
+       {
+            void           *pv;
+            PX86PT          pPT;
+            PPGMSHWPTPAE    pPTPae;
+            PX86PD          pPD;
+            PX86PDPAE       pPDPae;
+            PX86PDPT        pPDPT;
+            PX86PML4        pPML4;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            PEPTPDPT        pEptPdpt;
+            PEPTPD          pEptPd;
+            PEPTPT          pEptPt;
+#endif
+       } uShw;
+
+        LogFlow(("pgmPoolMonitorChainChanging: page idx=%d phys=%RGp (next=%d) kind=%s write=%#x\n",
+                 pPage->idx, pPage->GCPhys, pPage->iMonitoredNext, pgmPoolPoolKindToStr(pPage->enmKind), cbWrite));
+
+        uShw.pv = NULL;
+        switch (pPage->enmKind)
+        {
+            case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PTE);
+                LogFlow(("PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT iShw=%x\n", iShw));
+                X86PGUINT const uPde = uShw.pPT->a[iShw].u;
+                if (uPde & X86_PTE_P)
+                {
+                    X86PTE GstPte;
+                    int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
+                    AssertRC(rc);
+                    Log4(("pgmPoolMonitorChainChanging 32_32: deref %016RX64 GCPhys %08RX32\n", uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
+                    pgmPoolTracDerefGCPhysHint(pPool, pPage, uPde & X86_PTE_PG_MASK, GstPte.u & X86_PTE_PG_MASK, iShw);
+                    ASMAtomicWriteU32(&uShw.pPT->a[iShw].u, 0);
+                }
+                break;
+            }
+
+            /* page/2 sized */
+            case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                if (!((off ^ pPage->GCPhys) & (PAGE_SIZE / 2)))
+                {
+                    const unsigned iShw = (off / sizeof(X86PTE)) & (X86_PG_PAE_ENTRIES - 1);
+                    LogFlow(("PGMPOOLKIND_PAE_PT_FOR_32BIT_PT iShw=%x\n", iShw));
+                    if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
+                    {
+                        X86PTE GstPte;
+                        int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
+                        AssertRC(rc);
+
+                        Log4(("pgmPoolMonitorChainChanging pae_32: deref %016RX64 GCPhys %08RX32\n", uShw.pPT->a[iShw].u & X86_PTE_PAE_PG_MASK, GstPte.u & X86_PTE_PG_MASK));
+                        pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                                   PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
+                                                   GstPte.u & X86_PTE_PG_MASK,
+                                                   iShw);
+                        PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+            case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+            case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+            case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+            {
+                unsigned iGst     = off / sizeof(X86PDE);
+                unsigned iShwPdpt = iGst / 256;
+                unsigned iShw     = (iGst % 256) * 2;
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+
+                LogFlow(("pgmPoolMonitorChainChanging PAE for 32 bits: iGst=%x iShw=%x idx = %d page idx=%d\n", iGst, iShw, iShwPdpt, pPage->enmKind - PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD));
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
+                if (iShwPdpt == pPage->enmKind - (unsigned)PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD)
+                {
+                    for (unsigned i = 0; i < 2; i++)
+                    {
+                        X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw + i].u;
+                        if (uPde & X86_PDE_P)
+                        {
+                            LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw + i, uPde));
+                            pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw + i);
+                            ASMAtomicWriteU64(&uShw.pPDPae->a[iShw + i].u, 0);
+                        }
+
+                        /* paranoia / a bit assumptive. */
+                        if (    (off & 3)
+                            &&  (off & 3) + cbWrite > 4)
+                        {
+                            const unsigned iShw2 = iShw + 2 + i;
+                            if (iShw2 < RT_ELEMENTS(uShw.pPDPae->a))
+                            {
+                                X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
+                                if (uPde2 & X86_PDE_P)
+                                {
+                                    LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
+                                    pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
+                                    ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
+                                }
+                            }
+                        }
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PTEPAE);
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
+                if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw]))
+                {
+                    X86PTEPAE GstPte;
+                    int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
+                    AssertRC(rc);
+
+                    Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]), GstPte.u & X86_PTE_PAE_PG_MASK));
+                    pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                               PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw]),
+                                               GstPte.u & X86_PTE_PAE_PG_MASK,
+                                               iShw);
+                    PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw], 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PTEPAE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTEPAE);
+                    AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPTPae->a));
+
+                    if (PGMSHWPTEPAE_IS_P(uShw.pPTPae->a[iShw2]))
+                    {
+                        X86PTEPAE GstPte;
+                        int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
+                                                             pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
+                                                             GCPhysFault + sizeof(GstPte), sizeof(GstPte));
+                        AssertRC(rc);
+                        Log4(("pgmPoolMonitorChainChanging pae: deref %016RX64 GCPhys %016RX64\n", PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]), GstPte.u & X86_PTE_PAE_PG_MASK));
+                        pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                                   PGMSHWPTEPAE_GET_HCPHYS(uShw.pPTPae->a[iShw2]),
+                                                   GstPte.u & X86_PTE_PAE_PG_MASK,
+                                                   iShw2);
+                        PGMSHWPTEPAE_ATOMIC_SET(uShw.pPTPae->a[iShw2], 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_32BIT_PD:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PTE);         // ASSUMING 32-bit guest paging!
+
+                LogFlow(("pgmPoolMonitorChainChanging: PGMPOOLKIND_32BIT_PD %x\n", iShw));
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
+                X86PGUINT const uPde = uShw.pPD->a[iShw].u;
+                if (uPde & X86_PDE_P)
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
+                    pgmPoolFree(pVM, uPde & X86_PDE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 3)
+                    &&  (off & 3) + cbWrite > sizeof(X86PTE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PTE);
+                    if (    iShw2 != iShw
+                        &&  iShw2 < RT_ELEMENTS(uShw.pPD->a))
+                    {
+                        X86PGUINT const uPde2 = uShw.pPD->a[iShw2].u;
+                        if (uPde2 & X86_PDE_P)
+                        {
+                            LogFlow(("pgmPoolMonitorChainChanging: 32 bit pd iShw=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
+                            pgmPoolFree(pVM, uPde2 & X86_PDE_PG_MASK, pPage->idx, iShw2);
+                            ASMAtomicWriteU32(&uShw.pPD->a[iShw2].u, 0);
+                        }
+                    }
+                }
+#if 0 /* useful when running PGMAssertCR3(), a bit too troublesome for general use (TLBs). - not working any longer... */
+                if (    uShw.pPD->a[iShw].n.u1Present
+                    &&  !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3))
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: iShw=%#x: %RX32 -> freeing it!\n", iShw, uShw.pPD->a[iShw].u));
+                    pgmPoolFree(pVM, uShw.pPD->a[iShw].u & X86_PDE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU32(&uShw.pPD->a[iShw].u, 0);
+                }
+#endif
+                break;
+            }
+
+            case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PDEPAE);
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
+
+                /*
+                 * Causes trouble when the guest uses a PDE to refer to the whole page table level
+                 * structure. (Invalidate here; faults later on when it tries to change the page
+                 * table entries -> recheck; probably only applies to the RC case.)
+                 */
+                X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u;
+                if (uPde & X86_PDE_P)
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
+                    pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PDEPAE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
+                    AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
+
+                    X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
+                    if (uPde2 & X86_PDE_P)
+                    {
+                        LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
+                        pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_PAE_PDPT:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
+                /*
+                 * Hopefully this doesn't happen very often:
+                 * - touching unused parts of the page
+                 * - messing with the bits of pd pointers without changing the physical address
+                 */
+                /* PDPT roots are not page aligned; 32 byte only! */
+                const unsigned offPdpt = GCPhysFault - pPage->GCPhys;
+
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = offPdpt / sizeof(X86PDPE);
+                if (iShw < X86_PG_PAE_PDPE_ENTRIES)          /* don't use RT_ELEMENTS(uShw.pPDPT->a), because that's for long mode only */
+                {
+                    X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u;
+                    if (uPdpe & X86_PDPE_P)
+                    {
+                        LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uShw.pPDPT->a[iShw].u));
+                        pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw);
+                        ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
+                    }
+
+                    /* paranoia / a bit assumptive. */
+                    if (    (offPdpt & 7)
+                        &&  (offPdpt & 7) + cbWrite > sizeof(X86PDPE))
+                    {
+                        const unsigned iShw2 = (offPdpt + cbWrite - 1) / sizeof(X86PDPE);
+                        if (    iShw2 != iShw
+                            &&  iShw2 < X86_PG_PAE_PDPE_ENTRIES)
+                        {
+                            X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u;
+                            if (uPdpe2 & X86_PDPE_P)
+                            {
+                                LogFlow(("pgmPoolMonitorChainChanging: pae pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw2, uShw.pPDPT->a[iShw2].u));
+                                pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2);
+                                ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
+                            }
+                        }
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPD));
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PDEPAE);
+                X86PGPAEUINT const uPde = uShw.pPDPae->a[iShw].u;
+                if (uPde & X86_PDE_P)
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw=%#x: %RX64 -> freeing it!\n", iShw, uPde));
+                    pgmPoolFree(pVM, uPde & X86_PDE_PAE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pPDPae->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PDEPAE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDEPAE);
+                    AssertBreak(iShw2 < RT_ELEMENTS(uShw.pPDPae->a));
+                    X86PGPAEUINT const uPde2 = uShw.pPDPae->a[iShw2].u;
+                    if (uPde2 & X86_PDE_P)
+                    {
+                        LogFlow(("pgmPoolMonitorChainChanging: pae pd iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPde2));
+                        pgmPoolFree(pVM, uPde2 & X86_PDE_PAE_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pPDPae->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPDPT));
+                /*
+                 * Hopefully this doesn't happen very often:
+                 * - messing with the bits of pd pointers without changing the physical address
+                 */
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PDPE);
+                X86PGPAEUINT const uPdpe = uShw.pPDPT->a[iShw].u;
+                if (uPdpe & X86_PDPE_P)
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw=%#x: %RX64 -> freeing it!\n", iShw, uPdpe));
+                    pgmPoolFree(pVM, uPdpe & X86_PDPE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pPDPT->a[iShw].u, 0);
+                }
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PDPE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PDPE);
+                    X86PGPAEUINT const uPdpe2 = uShw.pPDPT->a[iShw2].u;
+                    if (uPdpe2 & X86_PDPE_P)
+                    {
+                        LogFlow(("pgmPoolMonitorChainChanging: pdpt iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpe2));
+                        pgmPoolFree(pVM, uPdpe2 & X86_PDPE_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pPDPT->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_64BIT_PML4:
+            {
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPML4));
+                /*
+                 * Hopefully this doesn't happen very often:
+                 * - messing with the bits of pd pointers without changing the physical address
+                 */
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(X86PDPE);
+                X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u;
+                if (uPml4e & X86_PML4E_P)
+                {
+                    LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw=%#x: %RX64 -> freeing it!\n", iShw, uPml4e));
+                    pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
+                }
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PDPE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
+                    X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u;
+                    if (uPml4e2 & X86_PML4E_P)
+                    {
+                        LogFlow(("pgmPoolMonitorChainChanging: pml4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2));
+                        pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+            case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(EPTPML4E);
+                X86PGPAEUINT const uPml4e = uShw.pPML4->a[iShw].u;
+                if (uPml4e & EPT_PRESENT_MASK)
+                {
+                    Log7Func(("PML4 iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPml4e, pPage->GCPhys));
+                    pgmPoolFree(pVM, uPml4e & X86_PML4E_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pPML4->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(X86PML4E))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(X86PML4E);
+                    X86PGPAEUINT const uPml4e2 = uShw.pPML4->a[iShw2].u;
+                    if (uPml4e2 & EPT_PRESENT_MASK)
+                    {
+                        Log7Func(("PML4 iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPml4e2));
+                        pgmPoolFree(pVM, uPml4e2 & X86_PML4E_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pPML4->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(EPTPDPTE);
+                X86PGPAEUINT const uPdpte = uShw.pEptPdpt->a[iShw].u;
+                if (uPdpte & EPT_PRESENT_MASK)
+                {
+                    Log7Func(("EPT PDPT iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPdpte, pPage->GCPhys));
+                    pgmPoolFree(pVM, uPdpte & EPT_PDPTE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pEptPdpt->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(EPTPDPTE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPDPTE);
+                    X86PGPAEUINT const uPdpte2 = uShw.pEptPdpt->a[iShw2].u;
+                    if (uPdpte2 & EPT_PRESENT_MASK)
+                    {
+                        Log7Func(("EPT PDPT iShw2=%#x: %RX64 -> freeing it!\n", iShw2, uPdpte2));
+                        pgmPoolFree(pVM, uPdpte2 & EPT_PDPTE_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pEptPdpt->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(EPTPDE);
+                X86PGPAEUINT const uPde = uShw.pEptPd->a[iShw].u;
+                if (uPde & EPT_PRESENT_MASK)
+                {
+                    Log7Func(("EPT PD iShw=%#x: %RX64 (%RGp) -> freeing it!\n", iShw, uPde, pPage->GCPhys));
+                    pgmPoolFree(pVM, uPde & EPT_PDE_PG_MASK, pPage->idx, iShw);
+                    ASMAtomicWriteU64(&uShw.pEptPd->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(EPTPDE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPDE);
+                    AssertBreak(iShw2 < RT_ELEMENTS(uShw.pEptPd->a));
+                    X86PGPAEUINT const uPde2 = uShw.pEptPd->a[iShw2].u;
+                    if (uPde2 & EPT_PRESENT_MASK)
+                    {
+                        Log7Func(("EPT PD (2): iShw2=%#x: %RX64 (%RGp) -> freeing it!\n", iShw2, uPde2, pPage->GCPhys));
+                        pgmPoolFree(pVM, uPde2 & EPT_PDE_PG_MASK, pPage->idx, iShw2);
+                        ASMAtomicWriteU64(&uShw.pEptPd->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+
+            case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+            {
+                uShw.pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                const unsigned iShw = off / sizeof(EPTPTE);
+                X86PGPAEUINT const uPte = uShw.pEptPt->a[iShw].u;
+                STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitor,FaultPT));
+                if (uPte & EPT_PRESENT_MASK)
+                {
+                    EPTPTE GstPte;
+                    int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvAddress, GCPhysFault, sizeof(GstPte));
+                    AssertRC(rc);
+
+                    Log7Func(("EPT PT: iShw=%#x %RX64 (%RGp)\n", iShw, uPte, pPage->GCPhys));
+                    pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                               uShw.pEptPt->a[iShw].u & EPT_PTE_PG_MASK,
+                                               GstPte.u & EPT_PTE_PG_MASK,
+                                               iShw);
+                    ASMAtomicWriteU64(&uShw.pEptPt->a[iShw].u, 0);
+                }
+
+                /* paranoia / a bit assumptive. */
+                if (    (off & 7)
+                    &&  (off & 7) + cbWrite > sizeof(EPTPTE))
+                {
+                    const unsigned iShw2 = (off + cbWrite - 1) / sizeof(EPTPTE);
+                    AssertBreak(iShw2 < RT_ELEMENTS(uShw.pEptPt->a));
+                    X86PGPAEUINT const uPte2 = uShw.pEptPt->a[iShw2].u;
+                    if (uPte2 & EPT_PRESENT_MASK)
+                    {
+                        EPTPTE GstPte;
+                        int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte,
+                                                             pvAddress ? (uint8_t const *)pvAddress + sizeof(GstPte) : NULL,
+                                                             GCPhysFault + sizeof(GstPte), sizeof(GstPte));
+                        AssertRC(rc);
+                        Log7Func(("EPT PT (2): iShw=%#x %RX64 (%RGp)\n", iShw2, uPte2, pPage->GCPhys));
+                        pgmPoolTracDerefGCPhysHint(pPool, pPage,
+                                                   uShw.pEptPt->a[iShw2].u & EPT_PTE_PG_MASK,
+                                                   GstPte.u & EPT_PTE_PG_MASK,
+                                                   iShw2);
+                        ASMAtomicWriteU64(&uShw.pEptPt->a[iShw2].u, 0);
+                    }
+                }
+                break;
+            }
+#endif  /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+            default:
+                AssertFatalMsgFailed(("enmKind=%d\n", pPage->enmKind));
+        }
+        PGM_DYNMAP_UNUSED_HINT_VM(pVM, uShw.pv);
+
+        /* next */
+        if (pPage->iMonitoredNext == NIL_PGMPOOL_IDX)
+            return;
+        pPage = &pPool->aPages[pPage->iMonitoredNext];
+    }
+}
+
+#ifndef IN_RING3
+
+/**
+ * Checks if a access could be a fork operation in progress.
+ *
+ * Meaning, that the guest is setting up the parent process for Copy-On-Write.
+ *
+ * @returns true if it's likely that we're forking, otherwise false.
+ * @param   pPool       The pool.
+ * @param   pDis        The disassembled instruction.
+ * @param   offFault    The access offset.
+ */
+DECLINLINE(bool) pgmRZPoolMonitorIsForking(PPGMPOOL pPool, PDISCPUSTATE pDis, unsigned offFault)
+{
+    /*
+     * i386 linux is using btr to clear X86_PTE_RW.
+     * The functions involved are (2.6.16 source inspection):
+     *      clear_bit
+     *      ptep_set_wrprotect
+     *      copy_one_pte
+     *      copy_pte_range
+     *      copy_pmd_range
+     *      copy_pud_range
+     *      copy_page_range
+     *      dup_mmap
+     *      dup_mm
+     *      copy_mm
+     *      copy_process
+     *      do_fork
+     */
+    if (    pDis->pCurInstr->uOpcode == OP_BTR
+        &&  !(offFault & 4)
+        /** @todo Validate that the bit index is X86_PTE_RW. */
+            )
+    {
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,Fork)); RT_NOREF_PV(pPool);
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Determine whether the page is likely to have been reused.
+ *
+ * @returns true if we consider the page as being reused for a different purpose.
+ * @returns false if we consider it to still be a paging page.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pCtx        Pointer to the register context for the CPU.
+ * @param   pDis        The disassembly info for the faulting instruction.
+ * @param   pvFault     The fault address.
+ * @param   pPage       The pool page being accessed.
+ *
+ * @remark  The REP prefix check is left to the caller because of STOSD/W.
+ */
+DECLINLINE(bool) pgmRZPoolMonitorIsReused(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTX pCtx, PDISCPUSTATE pDis, RTGCPTR pvFault,
+                                          PPGMPOOLPAGE pPage)
+{
+    /* Locked (CR3, PDPTR*4) should not be reusable.  Considering them as
+       such may cause loops booting tst-ubuntu-15_10-64-efi, ++. */
+    if (pPage->cLocked)
+    {
+        Log2(("pgmRZPoolMonitorIsReused: %RGv (%p) can't have been resued, because it's locked!\n", pvFault, pPage));
+        return false;
+    }
+
+    /** @todo could make this general, faulting close to rsp should be a safe reuse heuristic. */
+    if (   HMHasPendingIrq(pVM)
+        && pCtx->rsp - pvFault < 32)
+    {
+        /* Fault caused by stack writes while trying to inject an interrupt event. */
+        Log(("pgmRZPoolMonitorIsReused: reused %RGv for interrupt stack (rsp=%RGv).\n", pvFault, pCtx->rsp));
+        return true;
+    }
+
+    LogFlow(("Reused instr %RGv %d at %RGv param1.fUse=%llx param1.reg=%d\n", pCtx->rip, pDis->pCurInstr->uOpcode, pvFault, pDis->Param1.fUse,  pDis->Param1.Base.idxGenReg));
+
+    /* Non-supervisor mode write means it's used for something else. */
+    if (CPUMGetGuestCPL(pVCpu) == 3)
+        return true;
+
+    switch (pDis->pCurInstr->uOpcode)
+    {
+        /* call implies the actual push of the return address faulted */
+        case OP_CALL:
+            Log4(("pgmRZPoolMonitorIsReused: CALL\n"));
+            return true;
+        case OP_PUSH:
+            Log4(("pgmRZPoolMonitorIsReused: PUSH\n"));
+            return true;
+        case OP_PUSHF:
+            Log4(("pgmRZPoolMonitorIsReused: PUSHF\n"));
+            return true;
+        case OP_PUSHA:
+            Log4(("pgmRZPoolMonitorIsReused: PUSHA\n"));
+            return true;
+        case OP_FXSAVE:
+            Log4(("pgmRZPoolMonitorIsReused: FXSAVE\n"));
+            return true;
+        case OP_MOVNTI:     /* solaris - block_zero_no_xmm */
+            Log4(("pgmRZPoolMonitorIsReused: MOVNTI\n"));
+            return true;
+        case OP_MOVNTDQ:    /* solaris - hwblkclr & hwblkpagecopy */
+            Log4(("pgmRZPoolMonitorIsReused: MOVNTDQ\n"));
+            return true;
+        case OP_MOVSWD:
+        case OP_STOSWD:
+            if (    pDis->fPrefix == (DISPREFIX_REP|DISPREFIX_REX)
+                &&  pCtx->rcx >= 0x40
+               )
+            {
+                Assert(pDis->uCpuMode == DISCPUMODE_64BIT);
+
+                Log(("pgmRZPoolMonitorIsReused: OP_STOSQ\n"));
+                return true;
+            }
+            break;
+
+        default:
+            /*
+             * Anything having ESP on the left side means stack writes.
+             */
+            if (    (    (pDis->Param1.fUse & DISUSE_REG_GEN32)
+                     ||  (pDis->Param1.fUse & DISUSE_REG_GEN64))
+                &&  (pDis->Param1.Base.idxGenReg == DISGREG_ESP))
+            {
+                Log4(("pgmRZPoolMonitorIsReused: ESP\n"));
+                return true;
+            }
+            break;
+    }
+
+    /*
+     * Page table updates are very very unlikely to be crossing page boundraries,
+     * and we don't want to deal with that in pgmPoolMonitorChainChanging and such.
+     */
+    uint32_t const cbWrite = DISGetParamSize(pDis, &pDis->Param1);
+    if ( (((uintptr_t)pvFault + cbWrite) >> X86_PAGE_SHIFT) != ((uintptr_t)pvFault >> X86_PAGE_SHIFT) )
+    {
+        Log4(("pgmRZPoolMonitorIsReused: cross page write\n"));
+        return true;
+    }
+
+    /*
+     * Nobody does an unaligned 8 byte write to a page table, right.
+     */
+    if (cbWrite >= 8 && ((uintptr_t)pvFault & 7) != 0)
+    {
+        Log4(("pgmRZPoolMonitorIsReused: Unaligned 8+ byte write\n"));
+        return true;
+    }
+
+    return false;
+}
+
+
+/**
+ * Flushes the page being accessed.
+ *
+ * @returns VBox status code suitable for scheduling.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPool       The pool.
+ * @param   pPage       The pool page (head).
+ * @param   pDis        The disassembly of the write instruction.
+ * @param   pCtx        Pointer to the register context for the CPU.
+ * @param   GCPhysFault The fault address as guest physical address.
+ * @todo VBOXSTRICTRC
+ */
+static int pgmRZPoolAccessPfHandlerFlush(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
+                                         PCPUMCTX pCtx, RTGCPHYS GCPhysFault)
+{
+    NOREF(pVM); NOREF(GCPhysFault);
+
+    /*
+     * First, do the flushing.
+     */
+    pgmPoolMonitorChainFlush(pPool, pPage);
+
+    /*
+     * Emulate the instruction (xp/w2k problem, requires pc/cr2/sp detection).
+     * Must do this in raw mode (!); XP boot will fail otherwise.
+     */
+    int rc = VINF_SUCCESS;
+    VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, pCtx->rip);
+    if (rc2 == VINF_SUCCESS)
+    { /* do nothing */ }
+    else if (rc2 == VINF_EM_RESCHEDULE)
+    {
+        rc = VBOXSTRICTRC_VAL(rc2);
+# ifndef IN_RING3
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
+# endif
+    }
+    else if (rc2 == VERR_EM_INTERPRETER)
+    {
+        rc = VINF_EM_RAW_EMULATE_INSTR;
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
+    }
+    else if (RT_FAILURE_NP(rc2))
+        rc = VBOXSTRICTRC_VAL(rc2);
+    else
+        AssertMsgFailed(("%Rrc\n", VBOXSTRICTRC_VAL(rc2))); /* ASSUMES no complicated stuff here. */
+
+    LogFlow(("pgmRZPoolAccessPfHandlerFlush: returns %Rrc (flushed)\n", rc));
+    return rc;
+}
+
+
+/**
+ * Handles the STOSD write accesses.
+ *
+ * @returns VBox status code suitable for scheduling.
+ * @param   pVM         The cross context VM structure.
+ * @param   pPool       The pool.
+ * @param   pPage       The pool page (head).
+ * @param   pDis        The disassembly of the write instruction.
+ * @param   pCtx        Pointer to the register context for the CPU.
+ * @param   GCPhysFault The fault address as guest physical address.
+ * @param   pvFault     The fault address.
+ */
+DECLINLINE(int) pgmRZPoolAccessPfHandlerSTOSD(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
+                                              PCPUMCTX pCtx, RTGCPHYS GCPhysFault, RTGCPTR pvFault)
+{
+    unsigned uIncrement = pDis->Param1.cb;
+    NOREF(pVM);
+
+    Assert(pDis->uCpuMode == DISCPUMODE_32BIT || pDis->uCpuMode == DISCPUMODE_64BIT);
+    Assert(pCtx->rcx <= 0x20);
+
+# ifdef VBOX_STRICT
+    if (pDis->uOpMode == DISCPUMODE_32BIT)
+        Assert(uIncrement == 4);
+    else
+        Assert(uIncrement == 8);
+# endif
+
+    Log3(("pgmRZPoolAccessPfHandlerSTOSD\n"));
+
+    /*
+     * Increment the modification counter and insert it into the list
+     * of modified pages the first time.
+     */
+    if (!pPage->cModifications++)
+        pgmPoolMonitorModifiedInsert(pPool, pPage);
+
+    /*
+     * Execute REP STOSD.
+     *
+     * This ASSUMES that we're not invoked by Trap0e on in a out-of-sync
+     * write situation, meaning that it's safe to write here.
+     */
+    PVMCPUCC    pVCpu = VMMGetCpu(pPool->CTX_SUFF(pVM));
+    RTGCUINTPTR pu32 = (RTGCUINTPTR)pvFault;
+    while (pCtx->rcx)
+    {
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, uIncrement);
+        PGMPhysSimpleWriteGCPhys(pVM, GCPhysFault, &pCtx->rax, uIncrement);
+        pu32           += uIncrement;
+        GCPhysFault    += uIncrement;
+        pCtx->rdi += uIncrement;
+        pCtx->rcx--;
+    }
+    pCtx->rip += pDis->cbInstr;
+
+    LogFlow(("pgmRZPoolAccessPfHandlerSTOSD: returns\n"));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Handles the simple write accesses.
+ *
+ * @returns VBox status code suitable for scheduling.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pPool       The pool.
+ * @param   pPage       The pool page (head).
+ * @param   pDis        The disassembly of the write instruction.
+ * @param   pCtx        Pointer to the register context for the CPU.
+ * @param   GCPhysFault The fault address as guest physical address.
+ * @param   pfReused    Reused state (in/out)
+ */
+DECLINLINE(int) pgmRZPoolAccessPfHandlerSimple(PVMCC pVM, PVMCPUCC pVCpu, PPGMPOOL pPool, PPGMPOOLPAGE pPage, PDISCPUSTATE pDis,
+                                               PCPUMCTX pCtx, RTGCPHYS GCPhysFault, bool *pfReused)
+{
+    Log3(("pgmRZPoolAccessPfHandlerSimple\n"));
+    NOREF(pVM);
+    NOREF(pfReused); /* initialized by caller */
+
+    /*
+     * Increment the modification counter and insert it into the list
+     * of modified pages the first time.
+     */
+    if (!pPage->cModifications++)
+        pgmPoolMonitorModifiedInsert(pPool, pPage);
+
+    /*
+     * Clear all the pages.
+     */
+    uint32_t cbWrite = DISGetParamSize(pDis, &pDis->Param1);
+    if (cbWrite <= 8)
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, cbWrite);
+    else if (cbWrite <= 16)
+    {
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault, NULL, 8);
+        pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + 8, NULL, cbWrite - 8);
+    }
+    else
+    {
+        Assert(cbWrite <= 32);
+        for (uint32_t off = 0; off < cbWrite; off += 8)
+            pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhysFault + off, NULL, RT_MIN(8, cbWrite - off));
+    }
+
+    /*
+     * Interpret the instruction.
+     */
+    VBOXSTRICTRC rc = EMInterpretInstructionDisasState(pVCpu, pDis, pCtx->rip);
+    if (RT_SUCCESS(rc))
+        AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rc))); /* ASSUMES no complicated stuff here. */
+    else if (rc == VERR_EM_INTERPRETER)
+    {
+        LogFlow(("pgmRZPoolAccessPfHandlerSimple: Interpretation failed for %04x:%RGv - opcode=%d\n",
+                 pCtx->cs.Sel, (RTGCPTR)pCtx->rip, pDis->pCurInstr->uOpcode));
+        rc = VINF_EM_RAW_EMULATE_INSTR;
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(StatMonitorPf,EmulateInstr));
+    }
+
+# if 0 /* experimental code */
+    if (rc == VINF_SUCCESS)
+    {
+        switch (pPage->enmKind)
+        {
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        {
+            X86PTEPAE GstPte;
+            int rc = pgmPoolPhysSimpleReadGCPhys(pVM, &GstPte, pvFault, GCPhysFault, sizeof(GstPte));
+            AssertRC(rc);
+
+            /* Check the new value written by the guest. If present and with a bogus physical address, then
+             * it's fairly safe to assume the guest is reusing the PT.
+             */
+            if (GstPte.n.u1Present)
+            {
+                RTHCPHYS HCPhys = -1;
+                int rc = PGMPhysGCPhys2HCPhys(pVM, GstPte.u & X86_PTE_PAE_PG_MASK, &HCPhys);
+                if (rc != VINF_SUCCESS)
+                {
+                    *pfReused = true;
+                    STAM_COUNTER_INC(&pPool->StatForceFlushReused);
+                }
+            }
+            break;
+        }
+        }
+    }
+# endif
+
+    LogFlow(("pgmRZPoolAccessPfHandlerSimple: returns %Rrc\n", VBOXSTRICTRC_VAL(rc)));
+    return VBOXSTRICTRC_VAL(rc);
+}
+
+
+/**
+ * @callback_method_impl{FNPGMRZPHYSPFHANDLER,
+ *      \#PF access handler callback for page table pages.}
+ *
+ * @remarks The @a uUser argument is the index of the PGMPOOLPAGE.
+ */
+DECLCALLBACK(VBOXSTRICTRC) pgmRZPoolAccessPfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
+                                                    RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
+{
+    STAM_PROFILE_START(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorRZ, a);
+    PPGMPOOL const      pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    AssertReturn(uUser < pPool->cCurPages, VERR_PGM_POOL_IPE);
+    PPGMPOOLPAGE const  pPage = &pPool->aPages[uUser];
+    unsigned            cMaxModifications;
+    bool                fForcedFlush = false;
+    RT_NOREF_PV(uErrorCode);
+
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    AssertMsg(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT,
+              ("pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
+# endif
+    LogFlow(("pgmRZPoolAccessPfHandler: pvFault=%RGv pPage=%p:{.idx=%d} GCPhysFault=%RGp\n", pvFault, pPage, pPage->idx, GCPhysFault));
+
+    PGM_LOCK_VOID(pVM);
+    if (PHYS_PAGE_ADDRESS(GCPhysFault) != PHYS_PAGE_ADDRESS(pPage->GCPhys))
+    {
+        /* Pool page changed while we were waiting for the lock; ignore. */
+        Log(("CPU%d: pgmRZPoolAccessPfHandler pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhysFault), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
+        STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
+        PGM_UNLOCK(pVM);
+        return VINF_SUCCESS;
+    }
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    if (pPage->fDirty)
+    {
+#  ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        Assert(!PGMPOOL_PAGE_IS_NESTED(pPage));
+#  endif
+        Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH));
+        PGM_UNLOCK(pVM);
+        return VINF_SUCCESS;    /* SMP guest case where we were blocking on the pgm lock while the same page was being marked dirty. */
+    }
+# endif
+
+# if 0 /* test code defined(VBOX_STRICT) && defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) */
+    if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+    {
+        void *pvShw = PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pPage);
+        void *pvGst;
+        int rc = PGM_GCPHYS_2_PTR(pPool->CTX_SUFF(pVM), pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+        pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
+        PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+        PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
+    }
+# endif
+
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    if (PGMPOOL_PAGE_IS_NESTED(pPage))
+    {
+        Assert(!CPUMIsGuestInVmxNonRootMode(CPUMQueryGuestCtxPtr(pVCpu)));
+        Log7Func(("Flushing pvFault=%RGv GCPhysFault=%RGp\n", pvFault, GCPhysFault));
+        pgmPoolMonitorChainFlush(pPool, pPage);
+        PGM_UNLOCK(pVM);
+        return VINF_SUCCESS;
+    }
+# endif
+
+    /*
+     * Disassemble the faulting instruction.
+     */
+    PDISCPUSTATE pDis = &pVCpu->pgm.s.DisState;
+    int rc = EMInterpretDisasCurrent(pVCpu, pDis, NULL);
+    if (RT_UNLIKELY(rc != VINF_SUCCESS))
+    {
+        AssertMsg(rc == VERR_PAGE_NOT_PRESENT || rc == VERR_PAGE_TABLE_NOT_PRESENT, ("Unexpected rc %d\n", rc));
+        PGM_UNLOCK(pVM);
+        return rc;
+    }
+
+    Assert(pPage->enmKind != PGMPOOLKIND_FREE);
+
+    /*
+     * We should ALWAYS have the list head as user parameter. This
+     * is because we use that page to record the changes.
+     */
+    Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
+
+# ifdef IN_RING0
+    /* Maximum nr of modifications depends on the page type. */
+    if (    pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+        ||  pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
+        cMaxModifications = 4;
+    else
+        cMaxModifications = 24;
+# else
+    cMaxModifications = 48;
+# endif
+
+    /*
+     * Incremental page table updates should weigh more than random ones.
+     * (Only applies when started from offset 0)
+     */
+    pVCpu->pgm.s.cPoolAccessHandler++;
+    if (    pPage->GCPtrLastAccessHandlerRip >= pCtx->rip - 0x40      /* observed loops in Windows 7 x64 */
+        &&  pPage->GCPtrLastAccessHandlerRip <  pCtx->rip + 0x40
+        &&  pvFault == (pPage->GCPtrLastAccessHandlerFault + pDis->Param1.cb)
+        &&  pVCpu->pgm.s.cPoolAccessHandler == pPage->cLastAccessHandler + 1)
+    {
+        Log(("Possible page reuse cMods=%d -> %d (locked=%d type=%s)\n", pPage->cModifications, pPage->cModifications * 2, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
+        Assert(pPage->cModifications < 32000);
+        pPage->cModifications               = pPage->cModifications * 2;
+        pPage->GCPtrLastAccessHandlerFault  = pvFault;
+        pPage->cLastAccessHandler           = pVCpu->pgm.s.cPoolAccessHandler;
+        if (pPage->cModifications >= cMaxModifications)
+        {
+            STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushReinit);
+            fForcedFlush = true;
+        }
+    }
+
+    if (pPage->cModifications >= cMaxModifications)
+        Log(("Mod overflow %RGv cMods=%d (locked=%d type=%s)\n", pvFault, pPage->cModifications, pgmPoolIsPageLocked(pPage), pgmPoolPoolKindToStr(pPage->enmKind)));
+
+    /*
+     * Check if it's worth dealing with.
+     */
+    bool fReused = false;
+    bool fNotReusedNotForking = false;
+    if (    (   pPage->cModifications < cMaxModifications   /** @todo \#define */ /** @todo need to check that it's not mapping EIP. */ /** @todo adjust this! */
+             || pgmPoolIsPageLocked(pPage)
+            )
+        &&  !(fReused = pgmRZPoolMonitorIsReused(pVM, pVCpu, pCtx, pDis, pvFault, pPage))
+        &&  !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
+    {
+        /*
+         * Simple instructions, no REP prefix.
+         */
+        if (!(pDis->fPrefix & (DISPREFIX_REP | DISPREFIX_REPNE)))
+        {
+            rc = pgmRZPoolAccessPfHandlerSimple(pVM, pVCpu, pPool, pPage, pDis, pCtx, GCPhysFault, &fReused);
+            if (fReused)
+                goto flushPage;
+
+            /* A mov instruction to change the first page table entry will be remembered so we can detect
+             * full page table changes early on. This will reduce the amount of unnecessary traps we'll take.
+             */
+            if (   rc == VINF_SUCCESS
+                && !pPage->cLocked                      /* only applies to unlocked pages as we can't free locked ones (e.g. cr3 root). */
+                && pDis->pCurInstr->uOpcode == OP_MOV
+                && (pvFault & PAGE_OFFSET_MASK) == 0)
+            {
+                pPage->GCPtrLastAccessHandlerFault = pvFault;
+                pPage->cLastAccessHandler          = pVCpu->pgm.s.cPoolAccessHandler;
+                pPage->GCPtrLastAccessHandlerRip   = pCtx->rip;
+                /* Make sure we don't kick out a page too quickly. */
+                if (pPage->cModifications > 8)
+                    pPage->cModifications = 2;
+            }
+            else if (pPage->GCPtrLastAccessHandlerFault == pvFault)
+            {
+                /* ignore the 2nd write to this page table entry. */
+                pPage->cLastAccessHandler       = pVCpu->pgm.s.cPoolAccessHandler;
+            }
+            else
+            {
+                pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
+                pPage->GCPtrLastAccessHandlerRip   = 0;
+            }
+
+            STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZHandled, a);
+            PGM_UNLOCK(pVM);
+            return rc;
+        }
+
+        /*
+         * Windows is frequently doing small memset() operations (netio test 4k+).
+         * We have to deal with these or we'll kill the cache and performance.
+         */
+        if (    pDis->pCurInstr->uOpcode == OP_STOSWD
+            &&  !pCtx->eflags.Bits.u1DF
+            &&  pDis->uOpMode == pDis->uCpuMode
+            &&  pDis->uAddrMode == pDis->uCpuMode)
+        {
+            bool fValidStosd = false;
+
+            if (    pDis->uCpuMode == DISCPUMODE_32BIT
+                &&  pDis->fPrefix == DISPREFIX_REP
+                &&  pCtx->ecx <= 0x20
+                &&  pCtx->ecx * 4 <= GUEST_PAGE_SIZE - ((uintptr_t)pvFault & GUEST_PAGE_OFFSET_MASK)
+                &&  !((uintptr_t)pvFault & 3)
+                &&  (pCtx->eax == 0 || pCtx->eax == 0x80) /* the two values observed. */
+                )
+            {
+                fValidStosd = true;
+                pCtx->rcx &= 0xffffffff;   /* paranoia */
+            }
+            else
+            if (    pDis->uCpuMode == DISCPUMODE_64BIT
+                &&  pDis->fPrefix == (DISPREFIX_REP | DISPREFIX_REX)
+                &&  pCtx->rcx <= 0x20
+                &&  pCtx->rcx * 8 <= GUEST_PAGE_SIZE - ((uintptr_t)pvFault & GUEST_PAGE_OFFSET_MASK)
+                &&  !((uintptr_t)pvFault & 7)
+                &&  (pCtx->rax == 0 || pCtx->rax == 0x80) /* the two values observed. */
+                )
+            {
+                fValidStosd = true;
+            }
+
+            if (fValidStosd)
+            {
+                rc = pgmRZPoolAccessPfHandlerSTOSD(pVM, pPool, pPage, pDis, pCtx, GCPhysFault, pvFault);
+                STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZRepStosd, a);
+                PGM_UNLOCK(pVM);
+                return rc;
+            }
+        }
+
+        /* REP prefix, don't bother. */
+        STAM_COUNTER_INC(&pPool->StatMonitorPfRZRepPrefix);
+        Log4(("pgmRZPoolAccessPfHandler: eax=%#x ecx=%#x edi=%#x esi=%#x rip=%RGv opcode=%d prefix=%#x\n",
+              pCtx->eax, pCtx->ecx, pCtx->edi, pCtx->esi, (RTGCPTR)pCtx->rip, pDis->pCurInstr->uOpcode, pDis->fPrefix));
+        fNotReusedNotForking = true;
+    }
+
+# if defined(PGMPOOL_WITH_OPTIMIZED_DIRTY_PT) && defined(IN_RING0)
+    /* E.g. Windows 7 x64 initializes page tables and touches some pages in the table during the process. This
+     * leads to pgm pool trashing and an excessive amount of write faults due to page monitoring.
+     */
+    if (    pPage->cModifications >= cMaxModifications
+        &&  !fForcedFlush
+        &&  (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT || pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
+        &&  (   fNotReusedNotForking
+             || (   !pgmRZPoolMonitorIsReused(pVM, pVCpu, pCtx, pDis, pvFault, pPage)
+                 && !pgmRZPoolMonitorIsForking(pPool, pDis, GCPhysFault & PAGE_OFFSET_MASK))
+            )
+       )
+    {
+        Assert(!pgmPoolIsPageLocked(pPage));
+        Assert(pPage->fDirty == false);
+
+        /* Flush any monitored duplicates as we will disable write protection. */
+        if (    pPage->iMonitoredNext != NIL_PGMPOOL_IDX
+            ||  pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
+        {
+            PPGMPOOLPAGE pPageHead = pPage;
+
+            /* Find the monitor head. */
+            while (pPageHead->iMonitoredPrev != NIL_PGMPOOL_IDX)
+                pPageHead = &pPool->aPages[pPageHead->iMonitoredPrev];
+
+            while (pPageHead)
+            {
+                unsigned idxNext = pPageHead->iMonitoredNext;
+
+                if (pPageHead != pPage)
+                {
+                    STAM_COUNTER_INC(&pPool->StatDirtyPageDupFlush);
+                    Log(("Flush duplicate page idx=%d GCPhys=%RGp type=%s\n", pPageHead->idx, pPageHead->GCPhys, pgmPoolPoolKindToStr(pPageHead->enmKind)));
+                    int rc2 = pgmPoolFlushPage(pPool, pPageHead);
+                    AssertRC(rc2);
+                }
+
+                if (idxNext == NIL_PGMPOOL_IDX)
+                    break;
+
+                pPageHead = &pPool->aPages[idxNext];
+            }
+        }
+
+        /* The flushing above might fail for locked pages, so double check. */
+        if (    pPage->iMonitoredNext == NIL_PGMPOOL_IDX
+            &&  pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
+        {
+            pgmPoolAddDirtyPage(pVM, pPool, pPage);
+
+            /* Temporarily allow write access to the page table again. */
+            rc = PGMHandlerPhysicalPageTempOff(pVM,
+                                               pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK,
+                                               pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
+            if (rc == VINF_SUCCESS)
+            {
+                rc = PGMShwMakePageWritable(pVCpu, pvFault, PGM_MK_PG_IS_WRITE_FAULT);
+                AssertMsg(rc == VINF_SUCCESS
+                        /* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
+                        ||  rc == VERR_PAGE_TABLE_NOT_PRESENT
+                        ||  rc == VERR_PAGE_NOT_PRESENT,
+                        ("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", pvFault, rc));
+#  ifdef VBOX_STRICT
+                pPage->GCPtrDirtyFault = pvFault;
+#  endif
+
+                STAM_PROFILE_STOP(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, a);
+                PGM_UNLOCK(pVM);
+                return rc;
+            }
+        }
+    }
+# endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT && IN_RING0 */
+
+    STAM_COUNTER_INC(&pPool->StatMonitorPfRZFlushModOverflow);
+flushPage:
+    /*
+     * Not worth it, so flush it.
+     *
+     * If we considered it to be reused, don't go back to ring-3
+     * to emulate failed instructions since we usually cannot
+     * interpret then. This may be a bit risky, in which case
+     * the reuse detection must be fixed.
+     */
+    rc = pgmRZPoolAccessPfHandlerFlush(pVM, pVCpu, pPool, pPage, pDis, pCtx, GCPhysFault);
+    if (    rc == VINF_EM_RAW_EMULATE_INSTR
+        &&  fReused)
+    {
+        Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* temporary, remove later. */
+        /* Make sure that the current instruction still has shadow page backing, otherwise we'll end up in a loop. */
+        if (PGMShwGetPage(pVCpu, pCtx->rip, NULL, NULL) == VINF_SUCCESS)
+            rc = VINF_SUCCESS;  /* safe to restart the instruction. */
+    }
+    STAM_PROFILE_STOP_EX(&pVM->pgm.s.CTX_SUFF(pPool)->StatMonitorPfRZ, &pPool->StatMonitorPfRZFlushPage, a);
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+#endif /* !IN_RING3 */
+
+/**
+ * @callback_method_impl{FNPGMPHYSHANDLER,
+ *      Access handler for shadowed page table pages.}
+ *
+ * @remarks Only uses the VINF_PGM_HANDLER_DO_DEFAULT status.
+ * @note    The @a uUser argument is the index of the PGMPOOLPAGE.
+ */
+DECLCALLBACK(VBOXSTRICTRC)
+pgmPoolAccessHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void *pvPhys, void *pvBuf, size_t cbBuf,
+                     PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
+{
+    PPGMPOOL const      pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    STAM_PROFILE_START(&pPool->CTX_SUFF_Z(StatMonitor), a);
+    AssertReturn(uUser < pPool->cCurPages, VERR_PGM_POOL_IPE);
+    PPGMPOOLPAGE const  pPage = &pPool->aPages[uUser];
+    LogFlow(("PGM_ALL_CB_DECL: GCPhys=%RGp %p:{.Core=%RHp, .idx=%d, .GCPhys=%RGp, .enmType=%d}\n",
+             GCPhys, pPage, pPage->Core.Key, pPage->idx, pPage->GCPhys, pPage->enmKind));
+
+    NOREF(pvPhys); NOREF(pvBuf); NOREF(enmAccessType);
+
+    PGM_LOCK_VOID(pVM);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Collect stats on the access.
+     */
+    AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Sizes)) == 19);
+    if (cbBuf <= 16 && cbBuf > 0)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[cbBuf - 1]);
+    else if (cbBuf >= 17 && cbBuf < 32)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[16]);
+    else if (cbBuf >= 32 && cbBuf < 64)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[17]);
+    else if (cbBuf >= 64)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Sizes)[18]);
+
+    uint8_t cbAlign;
+    switch (pPage->enmKind)
+    {
+        default:
+            cbAlign = 7;
+            break;
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            cbAlign = 3;
+            break;
+    }
+    AssertCompile(RT_ELEMENTS(pPool->CTX_MID_Z(aStatMonitor,Misaligned)) == 7);
+    if ((uint8_t)GCPhys & cbAlign)
+        STAM_COUNTER_INC(&pPool->CTX_MID_Z(aStatMonitor,Misaligned)[((uint8_t)GCPhys & cbAlign) - 1]);
+#endif
+
+    /*
+     * Make sure the pool page wasn't modified by a different CPU.
+     */
+    if (PHYS_PAGE_ADDRESS(GCPhys) == PHYS_PAGE_ADDRESS(pPage->GCPhys))
+    {
+        Assert(pPage->enmKind != PGMPOOLKIND_FREE);
+
+        /* The max modification count before flushing depends on the context and page type. */
+#ifdef IN_RING3
+        uint16_t const cMaxModifications = 96; /* it's cheaper here, right? */
+#else
+        uint16_t cMaxModifications;
+        if (    pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT
+            ||  pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
+            cMaxModifications = 4;
+        else
+            cMaxModifications = 24;
+#endif
+
+        /*
+         * We don't have to be very sophisticated about this since there are relativly few calls here.
+         * However, we must try our best to detect any non-cpu accesses (disk / networking).
+         */
+        if (   (   pPage->cModifications < cMaxModifications
+                || pgmPoolIsPageLocked(pPage) )
+            && enmOrigin != PGMACCESSORIGIN_DEVICE
+            && cbBuf <= 16)
+        {
+            /* Clear the shadow entry. */
+            if (!pPage->cModifications++)
+                pgmPoolMonitorModifiedInsert(pPool, pPage);
+
+            if (cbBuf <= 8)
+                pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, (uint32_t)cbBuf);
+            else
+            {
+                pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys, pvBuf, 8);
+                pgmPoolMonitorChainChanging(pVCpu, pPool, pPage, GCPhys + 8, (uint8_t *)pvBuf + 8, (uint32_t)cbBuf - 8);
+            }
+        }
+        else
+            pgmPoolMonitorChainFlush(pPool, pPage);
+
+        STAM_PROFILE_STOP_EX(&pPool->CTX_SUFF_Z(StatMonitor), &pPool->CTX_MID_Z(StatMonitor,FlushPage), a);
+    }
+    else
+        Log(("CPU%d: PGM_ALL_CB_DECL pgm pool page for %RGp changed (to %RGp) while waiting!\n", pVCpu->idCpu, PHYS_PAGE_ADDRESS(GCPhys), PHYS_PAGE_ADDRESS(pPage->GCPhys)));
+    PGM_UNLOCK(pVM);
+    return VINF_PGM_HANDLER_DO_DEFAULT;
+}
+
+
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+
+# if defined(VBOX_STRICT) && !defined(IN_RING3)
+
+/**
+ * Check references to guest physical memory in a PAE / PAE page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ * @param   pGstPT      The guest page table.
+ */
+static void pgmPoolTrackCheckPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
+{
+    unsigned cErrors    = 0;
+    int      LastRc     = -1;           /* initialized to shut up gcc */
+    unsigned LastPTE    = ~0U;          /* initialized to shut up gcc */
+    RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
+    PVMCC    pVM        = pPool->CTX_SUFF(pVM);
+
+#  ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
+        AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]),  pPage->iFirstPresent));
+#  endif
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            RTHCPHYS HCPhys = NIL_RTHCPHYS;
+            int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
+            if (    rc != VINF_SUCCESS
+                ||  PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
+            {
+                Log(("rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
+                LastPTE     = i;
+                LastRc      = rc;
+                LastHCPhys  = HCPhys;
+                cErrors++;
+
+                RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
+                rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
+                AssertRC(rc);
+
+                for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
+                {
+                    PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
+
+                    if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+                    {
+                        PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
+
+                        for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
+                        {
+                            if (   PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
+                                && PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
+                            {
+                                Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
+                            }
+                        }
+
+                        PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
+                    }
+                }
+            }
+        }
+    }
+    AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %RX64 shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
+}
+
+
+/**
+ * Check references to guest physical memory in a PAE / 32-bit page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ * @param   pGstPT      The guest page table.
+ */
+static void pgmPoolTrackCheckPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
+{
+    unsigned cErrors    = 0;
+    int      LastRc     = -1;           /* initialized to shut up gcc */
+    unsigned LastPTE    = ~0U;          /* initialized to shut up gcc */
+    RTHCPHYS LastHCPhys = NIL_RTHCPHYS; /* initialized to shut up gcc */
+    PVMCC    pVM        = pPool->CTX_SUFF(pVM);
+
+#  ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
+        AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]),  pPage->iFirstPresent));
+#  endif
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            RTHCPHYS HCPhys = NIL_RTHCPHYS;
+            int rc = PGMPhysGCPhys2HCPhys(pVM, pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
+            if (    rc != VINF_SUCCESS
+                ||  PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) != HCPhys)
+            {
+                Log(("rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", rc, i, pGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
+                LastPTE     = i;
+                LastRc      = rc;
+                LastHCPhys  = HCPhys;
+                cErrors++;
+
+                RTHCPHYS HCPhysPT = NIL_RTHCPHYS;
+                rc = PGMPhysGCPhys2HCPhys(pVM, pPage->GCPhys, &HCPhysPT);
+                AssertRC(rc);
+
+                for (unsigned iPage = 0; iPage < pPool->cCurPages; iPage++)
+                {
+                    PPGMPOOLPAGE pTempPage = &pPool->aPages[iPage];
+
+                    if (pTempPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_32BIT_PT)
+                    {
+                        PPGMSHWPTPAE pShwPT2 = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pTempPage);
+
+                        for (unsigned j = 0; j < RT_ELEMENTS(pShwPT->a); j++)
+                        {
+                            if (   PGMSHWPTEPAE_IS_P_RW(pShwPT2->a[j])
+                                && PGMSHWPTEPAE_GET_HCPHYS(pShwPT2->a[j]) == HCPhysPT)
+                            {
+                                Log(("GCPhys=%RGp idx=%d %RX64 vs %RX64\n", pTempPage->GCPhys, j, PGMSHWPTEPAE_GET_LOG(pShwPT->a[j]), PGMSHWPTEPAE_GET_LOG(pShwPT2->a[j])));
+                            }
+                        }
+
+                        PGM_DYNMAP_UNUSED_HINT_VM(pVM, pShwPT2);
+                    }
+                }
+            }
+        }
+    }
+    AssertMsg(!cErrors, ("cErrors=%d: last rc=%d idx=%d guest %x shw=%RX64 vs %RHp\n", cErrors, LastRc, LastPTE, pGstPT->a[LastPTE].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[LastPTE]), LastHCPhys));
+}
+
+# endif /* VBOX_STRICT && !IN_RING3 */
+
+/**
+ * Clear references to guest physical memory in a PAE / PAE page table.
+ *
+ * @returns nr of changed PTEs
+ * @param   pPool           The pool.
+ * @param   pPage           The page.
+ * @param   pShwPT          The shadow page table (mapping of the page).
+ * @param   pGstPT          The guest page table.
+ * @param   pOldGstPT       The old cached guest page table.
+ * @param   fAllowRemoval   Bail out as soon as we encounter an invalid PTE
+ * @param   pfFlush         Flush reused page table (out)
+ */
+DECLINLINE(unsigned) pgmPoolTrackFlushPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT,
+                                               PCX86PTPAE pOldGstPT, bool fAllowRemoval, bool *pfFlush)
+{
+    unsigned cChanged = 0;
+
+# ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
+        AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]),  pPage->iFirstPresent));
+# endif
+    *pfFlush = false;
+
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        /* Check the new value written by the guest. If present and with a bogus physical address, then
+         * it's fairly safe to assume the guest is reusing the PT.
+         */
+        if (   fAllowRemoval
+            && (pGstPT->a[i].u & X86_PTE_P))
+        {
+            if (!PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
+            {
+                *pfFlush = true;
+                return ++cChanged;
+            }
+        }
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            /* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
+            if ((pGstPT->a[i].u & X86_PTE_PAE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK))
+            {
+# ifdef VBOX_STRICT
+                RTHCPHYS HCPhys = NIL_RTGCPHYS;
+                int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK, &HCPhys);
+                AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %RX64 old %RX64 shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
+# endif
+                uint64_t uHostAttr  = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G | X86_PTE_PAE_NX);
+                bool     fHostRW    = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
+                uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G | X86_PTE_PAE_NX);
+                bool     fGuestRW   = !!(pGstPT->a[i].u & X86_PTE_RW);
+
+                if (    uHostAttr == uGuestAttr
+                    &&  fHostRW <= fGuestRW)
+                    continue;
+            }
+            cChanged++;
+            /* Something was changed, so flush it. */
+            Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%RX64\n",
+                  i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
+            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PAE_PG_MASK, i);
+            PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
+        }
+    }
+    return cChanged;
+}
+
+
+/**
+ * Clear references to guest physical memory in a PAE / PAE page table.
+ *
+ * @returns nr of changed PTEs
+ * @param   pPool           The pool.
+ * @param   pPage           The page.
+ * @param   pShwPT          The shadow page table (mapping of the page).
+ * @param   pGstPT          The guest page table.
+ * @param   pOldGstPT       The old cached guest page table.
+ * @param   fAllowRemoval   Bail out as soon as we encounter an invalid PTE
+ * @param   pfFlush         Flush reused page table (out)
+ */
+DECLINLINE(unsigned) pgmPoolTrackFlushPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT,
+                                                 PCX86PT pOldGstPT, bool fAllowRemoval, bool *pfFlush)
+{
+    unsigned cChanged = 0;
+
+# ifdef VBOX_STRICT
+    for (unsigned i = 0; i < RT_MIN(RT_ELEMENTS(pShwPT->a), pPage->iFirstPresent); i++)
+        AssertMsg(!PGMSHWPTEPAE_IS_P(pShwPT->a[i]), ("Unexpected PTE: idx=%d %RX64 (first=%d)\n", i, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]),  pPage->iFirstPresent));
+# endif
+    *pfFlush = false;
+
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        /* Check the new value written by the guest. If present and with a bogus physical address, then
+         * it's fairly safe to assume the guest is reusing the PT. */
+        if (fAllowRemoval)
+        {
+            X86PGUINT const uPte = pGstPT->a[i].u;
+            if (   (uPte & X86_PTE_P)
+                && !PGMPhysIsGCPhysValid(pPool->CTX_SUFF(pVM), uPte & X86_PTE_PG_MASK))
+            {
+                *pfFlush = true;
+                return ++cChanged;
+            }
+        }
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            /* If the old cached PTE is identical, then there's no need to flush the shadow copy. */
+            if ((pGstPT->a[i].u & X86_PTE_PG_MASK) == (pOldGstPT->a[i].u & X86_PTE_PG_MASK))
+            {
+# ifdef VBOX_STRICT
+                RTHCPHYS HCPhys = NIL_RTGCPHYS;
+                int rc = PGMPhysGCPhys2HCPhys(pPool->CTX_SUFF(pVM), pGstPT->a[i].u & X86_PTE_PG_MASK, &HCPhys);
+                AssertMsg(rc == VINF_SUCCESS && PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]) == HCPhys, ("rc=%d guest %x old %x shw=%RX64 vs %RHp\n", rc, pGstPT->a[i].u, pOldGstPT->a[i].u, PGMSHWPTEPAE_GET_LOG(pShwPT->a[i]), HCPhys));
+# endif
+                uint64_t uHostAttr  = PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G);
+                bool     fHostRW    = !!(PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & X86_PTE_RW);
+                uint64_t uGuestAttr = pGstPT->a[i].u & (X86_PTE_P | X86_PTE_US | X86_PTE_A | X86_PTE_D | X86_PTE_G);
+                bool     fGuestRW   = !!(pGstPT->a[i].u & X86_PTE_RW);
+
+                if (    uHostAttr == uGuestAttr
+                    &&  fHostRW <= fGuestRW)
+                    continue;
+            }
+            cChanged++;
+            /* Something was changed, so flush it. */
+            Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX64 hint=%x\n",
+                  i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK));
+            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pOldGstPT->a[i].u & X86_PTE_PG_MASK, i);
+            PGMSHWPTEPAE_ATOMIC_SET(pShwPT->a[i], 0);
+        }
+    }
+    return cChanged;
+}
+
+
+/**
+ * Flush a dirty page
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pPool           The pool.
+ * @param   idxSlot         Dirty array slot index
+ * @param   fAllowRemoval   Allow a reused page table to be removed
+ */
+static void pgmPoolFlushDirtyPage(PVMCC pVM, PPGMPOOL pPool, unsigned idxSlot, bool fAllowRemoval = false)
+{
+    AssertCompile(RT_ELEMENTS(pPool->aidxDirtyPages) == RT_ELEMENTS(pPool->aDirtyPages));
+
+    Assert(idxSlot < RT_ELEMENTS(pPool->aDirtyPages));
+    unsigned idxPage = pPool->aidxDirtyPages[idxSlot];
+    if (idxPage == NIL_PGMPOOL_IDX)
+        return;
+
+    PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
+    Assert(pPage->idx == idxPage);
+    Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
+
+    AssertMsg(pPage->fDirty, ("Page %RGp (slot=%d) not marked dirty!", pPage->GCPhys, idxSlot));
+    Log(("Flush dirty page %RGp cMods=%d\n", pPage->GCPhys, pPage->cModifications));
+
+    /* First write protect the page again to catch all write accesses. (before checking for changes -> SMP) */
+    int rc = PGMHandlerPhysicalReset(pVM, pPage->GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
+    Assert(rc == VINF_SUCCESS);
+    pPage->fDirty = false;
+
+# ifdef VBOX_STRICT
+    uint64_t fFlags = 0;
+    RTHCPHYS HCPhys;
+    rc = PGMShwGetPage(VMMGetCpu(pVM), pPage->GCPtrDirtyFault, &fFlags, &HCPhys);
+    AssertMsg(      (   rc == VINF_SUCCESS
+                     && (!(fFlags & X86_PTE_RW) || HCPhys != pPage->Core.Key))
+              /* In the SMP case the page table might be removed while we wait for the PGM lock in the trap handler. */
+              ||    rc == VERR_PAGE_TABLE_NOT_PRESENT
+              ||    rc == VERR_PAGE_NOT_PRESENT,
+              ("PGMShwGetPage -> GCPtr=%RGv rc=%d flags=%RX64\n", pPage->GCPtrDirtyFault, rc, fFlags));
+# endif
+
+    /* Flush those PTEs that have changed. */
+    STAM_PROFILE_START(&pPool->StatTrackDeref,a);
+    void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+    void *pvGst;
+    rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+    bool  fFlush;
+    unsigned cChanges;
+
+    if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+        cChanges = pgmPoolTrackFlushPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst,
+                                             (PCX86PTPAE)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
+    else
+    {
+        Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* temporary, remove later. */
+        cChanges = pgmPoolTrackFlushPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst,
+                                               (PCX86PT)&pPool->aDirtyPages[idxSlot].aPage[0], fAllowRemoval, &fFlush);
+    }
+
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
+    STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
+    /* Note: we might want to consider keeping the dirty page active in case there were many changes. */
+
+    /* This page is likely to be modified again, so reduce the nr of modifications just a bit here. */
+    Assert(pPage->cModifications);
+    if (cChanges < 4)
+        pPage->cModifications = 1;      /* must use > 0 here */
+    else
+        pPage->cModifications = RT_MAX(1, pPage->cModifications / 2);
+
+    STAM_COUNTER_INC(&pPool->StatResetDirtyPages);
+    if (pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages))
+        pPool->idxFreeDirtyPage = idxSlot;
+
+    pPool->cDirtyPages--;
+    pPool->aidxDirtyPages[idxSlot] = NIL_PGMPOOL_IDX;
+    Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
+    if (fFlush)
+    {
+        Assert(fAllowRemoval);
+        Log(("Flush reused page table!\n"));
+        pgmPoolFlushPage(pPool, pPage);
+        STAM_COUNTER_INC(&pPool->StatForceFlushReused);
+    }
+    else
+        Log(("Removed dirty page %RGp cMods=%d cChanges=%d\n", pPage->GCPhys, pPage->cModifications, cChanges));
+}
+
+
+# ifndef IN_RING3
+/**
+ * Add a new dirty page
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ */
+void pgmPoolAddDirtyPage(PVMCC pVM, PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    AssertCompile(RT_ELEMENTS(pPool->aDirtyPages) == 8 || RT_ELEMENTS(pPool->aDirtyPages) == 16);
+    Assert(!pPage->fDirty);
+    Assert(!PGMPOOL_PAGE_IS_NESTED(pPage));
+
+    unsigned idxFree = pPool->idxFreeDirtyPage;
+    Assert(idxFree < RT_ELEMENTS(pPool->aDirtyPages));
+    Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX && pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
+
+    if (pPool->cDirtyPages >= RT_ELEMENTS(pPool->aDirtyPages))
+    {
+        STAM_COUNTER_INC(&pPool->StatDirtyPageOverFlowFlush);
+        pgmPoolFlushDirtyPage(pVM, pPool, idxFree, true /* allow removal of reused page tables*/);
+    }
+    Assert(pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages));
+    AssertMsg(pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX, ("idxFree=%d cDirtyPages=%d\n", idxFree, pPool->cDirtyPages));
+
+    Log(("Add dirty page %RGp (slot=%d)\n", pPage->GCPhys, idxFree));
+
+    /*
+     * Make a copy of the guest page table as we require valid GCPhys addresses
+     * when removing references to physical pages.
+     * (The HCPhys linear lookup is *extremely* expensive!)
+     */
+    void *pvGst;
+    int   rc  = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+    memcpy(&pPool->aDirtyPages[idxFree].aPage[0], pvGst,
+           pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT ? PAGE_SIZE : PAGE_SIZE / 2);
+#  ifdef VBOX_STRICT
+    void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+    if (pPage->enmKind == PGMPOOLKIND_PAE_PT_FOR_PAE_PT)
+        pgmPoolTrackCheckPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
+    else
+        pgmPoolTrackCheckPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
+#  endif
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+
+    STAM_COUNTER_INC(&pPool->StatDirtyPage);
+    pPage->fDirty                    = true;
+    pPage->idxDirtyEntry             = (uint8_t)idxFree; Assert(pPage->idxDirtyEntry == idxFree);
+    pPool->aidxDirtyPages[idxFree]   = pPage->idx;
+    pPool->cDirtyPages++;
+
+    pPool->idxFreeDirtyPage        = (pPool->idxFreeDirtyPage + 1) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
+    if (    pPool->cDirtyPages < RT_ELEMENTS(pPool->aDirtyPages)
+        &&  pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
+    {
+        unsigned i;
+        for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+        {
+            idxFree = (pPool->idxFreeDirtyPage + i) & (RT_ELEMENTS(pPool->aDirtyPages) - 1);
+            if (pPool->aidxDirtyPages[idxFree] == NIL_PGMPOOL_IDX)
+            {
+                pPool->idxFreeDirtyPage = idxFree;
+                break;
+            }
+        }
+        Assert(i != RT_ELEMENTS(pPool->aDirtyPages));
+    }
+
+    Assert(pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages) || pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX);
+
+    /*
+     * Clear all references to this shadow table. See @bugref{7298}.
+     */
+    pgmPoolTrackClearPageUsers(pPool, pPage);
+}
+# endif /* !IN_RING3 */
+
+
+/**
+ * Check if the specified page is dirty (not write monitored)
+ *
+ * @return dirty or not
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhys          Guest physical address
+ */
+bool pgmPoolIsDirtyPageSlow(PVMCC pVM, RTGCPHYS GCPhys)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    if (!pPool->cDirtyPages)
+        return false;
+
+    GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+    {
+        unsigned idxPage = pPool->aidxDirtyPages[i];
+        if (idxPage != NIL_PGMPOOL_IDX)
+        {
+            PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
+            if (pPage->GCPhys == GCPhys)
+                return true;
+        }
+    }
+    return false;
+}
+
+
+/**
+ * Reset all dirty pages by reinstating page monitoring.
+ *
+ * @param   pVM             The cross context VM structure.
+ */
+void pgmPoolResetDirtyPages(PVMCC pVM)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
+
+    if (!pPool->cDirtyPages)
+        return;
+
+    Log(("pgmPoolResetDirtyPages\n"));
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+        pgmPoolFlushDirtyPage(pVM, pPool, i, true /* allow removal of reused page tables*/);
+
+    pPool->idxFreeDirtyPage = 0;
+    if (    pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
+        &&  pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
+    {
+        unsigned i;
+        for (i = 1; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+        {
+            if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
+            {
+                pPool->idxFreeDirtyPage = i;
+                break;
+            }
+        }
+        AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
+    }
+
+    Assert(pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] == NIL_PGMPOOL_IDX || pPool->cDirtyPages == RT_ELEMENTS(pPool->aDirtyPages));
+    return;
+}
+
+
+/**
+ * Invalidate the PT entry for the specified page
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPtrPage       Guest page to invalidate
+ */
+void pgmPoolResetDirtyPage(PVMCC pVM, RTGCPTR GCPtrPage)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
+
+    if (!pPool->cDirtyPages)
+        return;
+
+    Log(("pgmPoolResetDirtyPage %RGv\n", GCPtrPage)); RT_NOREF_PV(GCPtrPage);
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+    {
+    /** @todo What was intended here??? This looks incomplete... */
+    }
+}
+
+
+/**
+ * Reset all dirty pages by reinstating page monitoring.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   GCPhysPT        Physical address of the page table
+ */
+void pgmPoolInvalidateDirtyPage(PVMCC pVM, RTGCPHYS GCPhysPT)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    Assert(pPool->cDirtyPages <= RT_ELEMENTS(pPool->aDirtyPages));
+    unsigned idxDirtyPage = RT_ELEMENTS(pPool->aDirtyPages);
+
+    if (!pPool->cDirtyPages)
+        return;
+
+    GCPhysPT = GCPhysPT & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+    {
+        unsigned idxPage = pPool->aidxDirtyPages[i];
+        if (idxPage != NIL_PGMPOOL_IDX)
+        {
+            PPGMPOOLPAGE pPage = &pPool->aPages[idxPage];
+            if (pPage->GCPhys == GCPhysPT)
+            {
+                idxDirtyPage = i;
+                break;
+            }
+        }
+    }
+
+    if (idxDirtyPage != RT_ELEMENTS(pPool->aDirtyPages))
+    {
+        pgmPoolFlushDirtyPage(pVM, pPool, idxDirtyPage, true /* allow removal of reused page tables*/);
+        if (    pPool->cDirtyPages != RT_ELEMENTS(pPool->aDirtyPages)
+            &&  pPool->aidxDirtyPages[pPool->idxFreeDirtyPage] != NIL_PGMPOOL_IDX)
+        {
+            unsigned i;
+            for (i = 0; i < RT_ELEMENTS(pPool->aDirtyPages); i++)
+            {
+                if (pPool->aidxDirtyPages[i] == NIL_PGMPOOL_IDX)
+                {
+                    pPool->idxFreeDirtyPage = i;
+                    break;
+                }
+            }
+            AssertMsg(i != RT_ELEMENTS(pPool->aDirtyPages), ("cDirtyPages %d", pPool->cDirtyPages));
+        }
+    }
+}
+
+#endif /* PGMPOOL_WITH_OPTIMIZED_DIRTY_PT */
+
+/**
+ * Inserts a page into the GCPhys hash table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ */
+DECLINLINE(void) pgmPoolHashInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Log3(("pgmPoolHashInsert: %RGp\n", pPage->GCPhys));
+    Assert(pPage->GCPhys != NIL_RTGCPHYS); Assert(pPage->iNext == NIL_PGMPOOL_IDX);
+    uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
+    pPage->iNext = pPool->aiHash[iHash];
+    pPool->aiHash[iHash] = pPage->idx;
+}
+
+
+/**
+ * Removes a page from the GCPhys hash table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ */
+DECLINLINE(void) pgmPoolHashRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Log3(("pgmPoolHashRemove: %RGp\n", pPage->GCPhys));
+    uint16_t iHash = PGMPOOL_HASH(pPage->GCPhys);
+    if (pPool->aiHash[iHash] == pPage->idx)
+        pPool->aiHash[iHash] = pPage->iNext;
+    else
+    {
+        uint16_t iPrev = pPool->aiHash[iHash];
+        for (;;)
+        {
+            const int16_t i = pPool->aPages[iPrev].iNext;
+            if (i == pPage->idx)
+            {
+                pPool->aPages[iPrev].iNext = pPage->iNext;
+                break;
+            }
+            if (i == NIL_PGMPOOL_IDX)
+            {
+                AssertReleaseMsgFailed(("GCPhys=%RGp idx=%d\n", pPage->GCPhys, pPage->idx));
+                break;
+            }
+            iPrev = i;
+        }
+    }
+    pPage->iNext = NIL_PGMPOOL_IDX;
+}
+
+
+/**
+ * Frees up one cache page.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @param   pPool       The pool.
+ * @param   iUser       The user index.
+ */
+static int pgmPoolCacheFreeOne(PPGMPOOL pPool, uint16_t iUser)
+{
+    const PVMCC pVM = pPool->CTX_SUFF(pVM);
+    Assert(pPool->iAgeHead != pPool->iAgeTail); /* We shouldn't be here if there < 2 cached entries! */
+    STAM_COUNTER_INC(&pPool->StatCacheFreeUpOne);
+
+    /*
+     * Select one page from the tail of the age list.
+     */
+    PPGMPOOLPAGE    pPage;
+    for (unsigned iLoop = 0; ; iLoop++)
+    {
+        uint16_t iToFree = pPool->iAgeTail;
+        if (iToFree == iUser && iUser != NIL_PGMPOOL_IDX)
+            iToFree = pPool->aPages[iToFree].iAgePrev;
+/* This is the alternative to the SyncCR3 pgmPoolCacheUsed calls.
+        if (pPool->aPages[iToFree].iUserHead != NIL_PGMPOOL_USER_INDEX)
+        {
+            uint16_t i = pPool->aPages[iToFree].iAgePrev;
+            for (unsigned j = 0; j < 10 && i != NIL_PGMPOOL_USER_INDEX; j++, i = pPool->aPages[i].iAgePrev)
+            {
+                if (pPool->aPages[iToFree].iUserHead == NIL_PGMPOOL_USER_INDEX)
+                    continue;
+                iToFree = i;
+                break;
+            }
+        }
+*/
+        Assert(iToFree != iUser);
+        AssertReleaseMsg(iToFree != NIL_PGMPOOL_IDX,
+                         ("iToFree=%#x (iAgeTail=%#x) iUser=%#x iLoop=%u - pPool=%p LB %#zx\n",
+                          iToFree, pPool->iAgeTail, iUser, iLoop, pPool,
+                            RT_UOFFSETOF_DYN(PGMPOOL, aPages[pPool->cMaxPages])
+                          + pPool->cMaxUsers * sizeof(PGMPOOLUSER)
+                          + pPool->cMaxPhysExts * sizeof(PGMPOOLPHYSEXT) ));
+
+        pPage = &pPool->aPages[iToFree];
+
+        /*
+         * Reject any attempts at flushing the currently active shadow CR3 mapping.
+         * Call pgmPoolCacheUsed to move the page to the head of the age list.
+         */
+        if (   !pgmPoolIsPageLocked(pPage)
+            && pPage->idx >= PGMPOOL_IDX_FIRST /* paranoia (#6349) */)
+            break;
+        LogFlow(("pgmPoolCacheFreeOne: refuse CR3 mapping\n"));
+        pgmPoolCacheUsed(pPool, pPage);
+        AssertLogRelReturn(iLoop < 8192, VERR_PGM_POOL_TOO_MANY_LOOPS);
+    }
+
+    /*
+     * Found a usable page, flush it and return.
+     */
+    int rc = pgmPoolFlushPage(pPool, pPage);
+    /* This flush was initiated by us and not the guest, so explicitly flush the TLB. */
+    /** @todo find out why this is necessary; pgmPoolFlushPage should trigger a flush if one is really needed. */
+    if (rc == VINF_SUCCESS)
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+    return rc;
+}
+
+
+/**
+ * Checks if a kind mismatch is really a page being reused
+ * or if it's just normal remappings.
+ *
+ * @returns true if reused and the cached page (enmKind1) should be flushed
+ * @returns false if not reused.
+ * @param   enmKind1    The kind of the cached page.
+ * @param   enmKind2    The kind of the requested page.
+ */
+static bool pgmPoolCacheReusedByKind(PGMPOOLKIND enmKind1, PGMPOOLKIND enmKind2)
+{
+    switch (enmKind1)
+    {
+        /*
+         * Never reuse them. There is no remapping in non-paging mode.
+         */
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT: /* never reuse them for other types */
+            return false;
+
+        /*
+         * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
+         */
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_PAE_PDPT:
+            Assert(!PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2));
+            switch (enmKind2)
+            {
+                case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+                case PGMPOOLKIND_64BIT_PML4:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                    return true;
+                default:
+                    return false;
+            }
+
+        /*
+         * It's perfectly fine to reuse these, except for PAE and non-paging stuff.
+         */
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+            Assert(!PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2));
+            switch (enmKind2)
+            {
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                    return true;
+                default:
+                    return false;
+            }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            return PGMPOOL_PAGE_IS_KIND_NESTED(enmKind2);
+
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            return false;
+#endif
+
+        /*
+         * These cannot be flushed, and it's common to reuse the PDs as PTs.
+         */
+        case PGMPOOLKIND_ROOT_NESTED:
+            return false;
+
+        default:
+            AssertFatalMsgFailed(("enmKind1=%d\n", enmKind1));
+    }
+}
+
+
+/**
+ * Attempts to satisfy a pgmPoolAlloc request from the cache.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_PGM_CACHED_PAGE on success.
+ * @retval  VERR_FILE_NOT_FOUND if not found.
+ * @param   pPool       The pool.
+ * @param   GCPhys      The GC physical address of the page we're gonna shadow.
+ * @param   enmKind     The kind of mapping.
+ * @param   enmAccess   Access type for the mapping (only relevant for big pages)
+ * @param   fA20Enabled Whether the CPU has the A20 gate enabled.
+ * @param   iUser       The shadow page pool index of the user table.  This is
+ *                      NIL_PGMPOOL_IDX for root pages.
+ * @param   iUserTable  The index into the user table (shadowed).  Ignored if
+ *                      root page
+ * @param   ppPage      Where to store the pointer to the page.
+ */
+static int pgmPoolCacheAlloc(PPGMPOOL pPool, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
+                             uint16_t iUser, uint32_t iUserTable, PPPGMPOOLPAGE ppPage)
+{
+    /*
+     * Look up the GCPhys in the hash.
+     */
+    unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
+    Log3(("pgmPoolCacheAlloc: %RGp kind %s iUser=%d iUserTable=%x SLOT=%d\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable, i));
+    if (i != NIL_PGMPOOL_IDX)
+    {
+        do
+        {
+            PPGMPOOLPAGE pPage = &pPool->aPages[i];
+            Log4(("pgmPoolCacheAlloc: slot %d found page %RGp\n", i, pPage->GCPhys));
+            if (pPage->GCPhys == GCPhys)
+            {
+                if (   (PGMPOOLKIND)pPage->enmKind == enmKind
+                    && (PGMPOOLACCESS)pPage->enmAccess == enmAccess
+                    && pPage->fA20Enabled == fA20Enabled)
+                {
+                    /* Put it at the start of the use list to make sure pgmPoolTrackAddUser
+                     * doesn't flush it in case there are no more free use records.
+                     */
+                    pgmPoolCacheUsed(pPool, pPage);
+
+                    int rc = VINF_SUCCESS;
+                    if (iUser != NIL_PGMPOOL_IDX)
+                        rc = pgmPoolTrackAddUser(pPool, pPage, iUser, iUserTable);
+                    if (RT_SUCCESS(rc))
+                    {
+                        Assert((PGMPOOLKIND)pPage->enmKind == enmKind);
+                        *ppPage = pPage;
+                        if (pPage->cModifications)
+                            pPage->cModifications = 1; /* reset counter (can't use 0, or else it will be reinserted in the modified list) */
+                        STAM_COUNTER_INC(&pPool->StatCacheHits);
+                        return VINF_PGM_CACHED_PAGE;
+                    }
+                    return rc;
+                }
+
+                if ((PGMPOOLKIND)pPage->enmKind != enmKind)
+                {
+                    /*
+                     * The kind is different. In some cases we should now flush the page
+                     * as it has been reused, but in most cases this is normal remapping
+                     * of PDs as PT or big pages using the GCPhys field in a slightly
+                     * different way than the other kinds.
+                     */
+                    if (pgmPoolCacheReusedByKind((PGMPOOLKIND)pPage->enmKind, enmKind))
+                    {
+                        STAM_COUNTER_INC(&pPool->StatCacheKindMismatches);
+                        pgmPoolFlushPage(pPool, pPage);
+                        break;
+                    }
+                }
+            }
+
+            /* next */
+            i = pPage->iNext;
+        } while (i != NIL_PGMPOOL_IDX);
+    }
+
+    Log3(("pgmPoolCacheAlloc: Missed GCPhys=%RGp enmKind=%s\n", GCPhys, pgmPoolPoolKindToStr(enmKind)));
+    STAM_COUNTER_INC(&pPool->StatCacheMisses);
+    return VERR_FILE_NOT_FOUND;
+}
+
+
+/**
+ * Inserts a page into the cache.
+ *
+ * @param   pPool           The pool.
+ * @param   pPage           The cached page.
+ * @param   fCanBeCached    Set if the page is fit for caching from the caller's point of view.
+ */
+static void pgmPoolCacheInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fCanBeCached)
+{
+    /*
+     * Insert into the GCPhys hash if the page is fit for that.
+     */
+    Assert(!pPage->fCached);
+    if (fCanBeCached)
+    {
+        pPage->fCached = true;
+        pgmPoolHashInsert(pPool, pPage);
+        Log3(("pgmPoolCacheInsert: Caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
+              pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
+        STAM_COUNTER_INC(&pPool->StatCacheCacheable);
+    }
+    else
+    {
+        Log3(("pgmPoolCacheInsert: Not caching %p:{.Core=%RHp, .idx=%d, .enmKind=%s, GCPhys=%RGp}\n",
+              pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
+        STAM_COUNTER_INC(&pPool->StatCacheUncacheable);
+    }
+
+    /*
+     * Insert at the head of the age list.
+     */
+    pPage->iAgePrev = NIL_PGMPOOL_IDX;
+    pPage->iAgeNext = pPool->iAgeHead;
+    if (pPool->iAgeHead != NIL_PGMPOOL_IDX)
+        pPool->aPages[pPool->iAgeHead].iAgePrev = pPage->idx;
+    else
+        pPool->iAgeTail = pPage->idx;
+    pPool->iAgeHead = pPage->idx;
+}
+
+
+/**
+ * Flushes a cached page.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ */
+static void pgmPoolCacheFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Log3(("pgmPoolCacheFlushPage: %RGp\n", pPage->GCPhys));
+
+    /*
+     * Remove the page from the hash.
+     */
+    if (pPage->fCached)
+    {
+        pPage->fCached = false;
+        pgmPoolHashRemove(pPool, pPage);
+    }
+    else
+        Assert(pPage->iNext == NIL_PGMPOOL_IDX);
+
+    /*
+     * Remove it from the age list.
+     */
+    if (pPage->iAgeNext != NIL_PGMPOOL_IDX)
+        pPool->aPages[pPage->iAgeNext].iAgePrev = pPage->iAgePrev;
+    else
+        pPool->iAgeTail = pPage->iAgePrev;
+    if (pPage->iAgePrev != NIL_PGMPOOL_IDX)
+        pPool->aPages[pPage->iAgePrev].iAgeNext = pPage->iAgeNext;
+    else
+        pPool->iAgeHead = pPage->iAgeNext;
+    pPage->iAgeNext = NIL_PGMPOOL_IDX;
+    pPage->iAgePrev = NIL_PGMPOOL_IDX;
+}
+
+
+/**
+ * Looks for pages sharing the monitor.
+ *
+ * @returns Pointer to the head page.
+ * @returns NULL if not found.
+ * @param   pPool       The Pool
+ * @param   pNewPage    The page which is going to be monitored.
+ */
+static PPGMPOOLPAGE pgmPoolMonitorGetPageByGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pNewPage)
+{
+    /*
+     * Look up the GCPhys in the hash.
+     */
+    RTGCPHYS GCPhys = pNewPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+    unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
+    if (i == NIL_PGMPOOL_IDX)
+        return NULL;
+    do
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[i];
+        if (    pPage->GCPhys - GCPhys < PAGE_SIZE
+            &&  pPage != pNewPage)
+        {
+            switch (pPage->enmKind)
+            {
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+                case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+                case PGMPOOLKIND_64BIT_PML4:
+                case PGMPOOLKIND_32BIT_PD:
+                case PGMPOOLKIND_PAE_PDPT:
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+                case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+                case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+                case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+#endif
+                {
+                    /* find the head */
+                    while (pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
+                    {
+                        Assert(pPage->iMonitoredPrev != pPage->idx);
+                        pPage = &pPool->aPages[pPage->iMonitoredPrev];
+                    }
+                    return pPage;
+                }
+
+                /* ignore, no monitoring. */
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                case PGMPOOLKIND_ROOT_NESTED:
+                case PGMPOOLKIND_PAE_PD_PHYS:
+                case PGMPOOLKIND_PAE_PDPT_PHYS:
+                case PGMPOOLKIND_32BIT_PD_PHYS:
+                case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+                case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
+                case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+#endif
+                    break;
+                default:
+                    AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
+            }
+        }
+
+        /* next */
+        i = pPage->iNext;
+    } while (i != NIL_PGMPOOL_IDX);
+    return NULL;
+}
+
+
+/**
+ * Enabled write monitoring of a guest page.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ */
+static int pgmPoolMonitorInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    LogFlow(("pgmPoolMonitorInsert %RGp\n", pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK));
+
+    /*
+     * Filter out the relevant kinds.
+     */
+    switch (pPage->enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_PAE_PDPT:
+            break;
+
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+        case PGMPOOLKIND_ROOT_NESTED:
+            /* Nothing to monitor here. */
+            return VINF_SUCCESS;
+
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+            /* Nothing to monitor here. */
+            return VINF_SUCCESS;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            break;
+
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            /* Nothing to monitor here. */
+            return VINF_SUCCESS;
+#endif
+
+        default:
+            AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
+    }
+
+    /*
+     * Install handler.
+     */
+    int rc;
+    PPGMPOOLPAGE pPageHead = pgmPoolMonitorGetPageByGCPhys(pPool, pPage);
+    if (pPageHead)
+    {
+        Assert(pPageHead != pPage); Assert(pPageHead->iMonitoredNext != pPage->idx);
+        Assert(pPageHead->iMonitoredPrev != pPage->idx);
+
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+        if (pPageHead->fDirty)
+            pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPageHead->idxDirtyEntry, false /* do not remove */);
+#endif
+
+        pPage->iMonitoredPrev = pPageHead->idx;
+        pPage->iMonitoredNext = pPageHead->iMonitoredNext;
+        if (pPageHead->iMonitoredNext != NIL_PGMPOOL_IDX)
+            pPool->aPages[pPageHead->iMonitoredNext].iMonitoredPrev = pPage->idx;
+        pPageHead->iMonitoredNext = pPage->idx;
+        rc = VINF_SUCCESS;
+        if (PGMPOOL_PAGE_IS_NESTED(pPage))
+            Log7Func(("Adding to monitoring list GCPhysPage=%RGp\n", pPage->GCPhys));
+    }
+    else
+    {
+        if (PGMPOOL_PAGE_IS_NESTED(pPage))
+            Log7Func(("Started monitoring GCPhysPage=%RGp HCPhys=%RHp enmKind=%s\n", pPage->GCPhys, pPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
+
+        Assert(pPage->iMonitoredNext == NIL_PGMPOOL_IDX); Assert(pPage->iMonitoredPrev == NIL_PGMPOOL_IDX);
+        PVMCC pVM = pPool->CTX_SUFF(pVM);
+        const RTGCPHYS GCPhysPage = pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+        rc = PGMHandlerPhysicalRegister(pVM, GCPhysPage, GCPhysPage + PAGE_OFFSET_MASK, pPool->hAccessHandlerType,
+                                        pPage - &pPool->aPages[0], NIL_RTR3PTR /*pszDesc*/);
+        /** @todo we should probably deal with out-of-memory conditions here, but for now increasing
+         * the heap size should suffice. */
+        AssertFatalMsgRC(rc, ("PGMHandlerPhysicalRegisterEx %RGp failed with %Rrc\n", GCPhysPage, rc));
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+        AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), ("fSyncFlags=%x syncff=%d\n", pVCpu->pgm.s.fSyncFlags, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3)));
+    }
+    pPage->fMonitored = true;
+    return rc;
+}
+
+
+/**
+ * Disables write monitoring of a guest page.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ */
+static int pgmPoolMonitorFlush(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    /*
+     * Filter out the relevant kinds.
+     */
+    switch (pPage->enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_PAE_PDPT:
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+            break;
+
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+        case PGMPOOLKIND_ROOT_NESTED:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            /* Nothing to monitor here. */
+            Assert(!pPage->fMonitored);
+            return VINF_SUCCESS;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            break;
+
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            /* Nothing to monitor here. */
+            Assert(!pPage->fMonitored);
+            return VINF_SUCCESS;
+#endif
+
+        default:
+            AssertFatalMsgFailed(("This can't happen! enmKind=%d\n", pPage->enmKind));
+    }
+    Assert(pPage->fMonitored);
+
+    /*
+     * Remove the page from the monitored list or uninstall it if last.
+     */
+    const PVMCC pVM = pPool->CTX_SUFF(pVM);
+    int rc;
+    if (    pPage->iMonitoredNext != NIL_PGMPOOL_IDX
+        ||  pPage->iMonitoredPrev != NIL_PGMPOOL_IDX)
+    {
+        if (pPage->iMonitoredPrev == NIL_PGMPOOL_IDX)
+        {
+            PPGMPOOLPAGE pNewHead = &pPool->aPages[pPage->iMonitoredNext];
+            pNewHead->iMonitoredPrev = NIL_PGMPOOL_IDX;
+            rc = PGMHandlerPhysicalChangeUserArg(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK, pPage->iMonitoredNext);
+
+            AssertFatalRCSuccess(rc);
+            pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
+        }
+        else
+        {
+            pPool->aPages[pPage->iMonitoredPrev].iMonitoredNext = pPage->iMonitoredNext;
+            if (pPage->iMonitoredNext != NIL_PGMPOOL_IDX)
+            {
+                pPool->aPages[pPage->iMonitoredNext].iMonitoredPrev = pPage->iMonitoredPrev;
+                pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
+            }
+            pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
+            rc = VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        rc = PGMHandlerPhysicalDeregister(pVM, pPage->GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK);
+        AssertFatalRC(rc);
+        PVMCPU pVCpu = VMMGetCpu(pVM);
+        AssertFatalMsg(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL) || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3),
+                  ("%#x %#x\n", pVCpu->pgm.s.fSyncFlags, pVM->fGlobalForcedActions));
+    }
+    pPage->fMonitored = false;
+
+    /*
+     * Remove it from the list of modified pages (if in it).
+     */
+    pgmPoolMonitorModifiedRemove(pPool, pPage);
+
+    if (PGMPOOL_PAGE_IS_NESTED(pPage))
+        Log7Func(("Stopped monitoring %RGp\n", pPage->GCPhys));
+
+    return rc;
+}
+
+
+/**
+ * Inserts the page into the list of modified pages.
+ *
+ * @param   pPool   The pool.
+ * @param   pPage   The page.
+ */
+void pgmPoolMonitorModifiedInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Log3(("pgmPoolMonitorModifiedInsert: idx=%d\n", pPage->idx));
+    AssertMsg(   pPage->iModifiedNext == NIL_PGMPOOL_IDX
+              && pPage->iModifiedPrev == NIL_PGMPOOL_IDX
+              && pPool->iModifiedHead != pPage->idx,
+              ("Next=%d Prev=%d idx=%d cModifications=%d Head=%d cModifiedPages=%d\n",
+               pPage->iModifiedNext, pPage->iModifiedPrev, pPage->idx, pPage->cModifications,
+               pPool->iModifiedHead, pPool->cModifiedPages));
+
+    pPage->iModifiedNext = pPool->iModifiedHead;
+    if (pPool->iModifiedHead != NIL_PGMPOOL_IDX)
+        pPool->aPages[pPool->iModifiedHead].iModifiedPrev = pPage->idx;
+    pPool->iModifiedHead = pPage->idx;
+    pPool->cModifiedPages++;
+#ifdef VBOX_WITH_STATISTICS
+    if (pPool->cModifiedPages > pPool->cModifiedPagesHigh)
+        pPool->cModifiedPagesHigh = pPool->cModifiedPages;
+#endif
+}
+
+
+/**
+ * Removes the page from the list of modified pages and resets the
+ * modification counter.
+ *
+ * @param   pPool   The pool.
+ * @param   pPage   The page which is believed to be in the list of modified pages.
+ */
+static void pgmPoolMonitorModifiedRemove(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    Log3(("pgmPoolMonitorModifiedRemove: idx=%d cModifications=%d\n", pPage->idx, pPage->cModifications));
+    if (pPool->iModifiedHead == pPage->idx)
+    {
+        Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
+        pPool->iModifiedHead = pPage->iModifiedNext;
+        if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
+        {
+            pPool->aPages[pPage->iModifiedNext].iModifiedPrev = NIL_PGMPOOL_IDX;
+            pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+        }
+        pPool->cModifiedPages--;
+    }
+    else if (pPage->iModifiedPrev != NIL_PGMPOOL_IDX)
+    {
+        pPool->aPages[pPage->iModifiedPrev].iModifiedNext = pPage->iModifiedNext;
+        if (pPage->iModifiedNext != NIL_PGMPOOL_IDX)
+        {
+            pPool->aPages[pPage->iModifiedNext].iModifiedPrev = pPage->iModifiedPrev;
+            pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+        }
+        pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+        pPool->cModifiedPages--;
+    }
+    else
+        Assert(pPage->iModifiedPrev == NIL_PGMPOOL_IDX);
+    pPage->cModifications = 0;
+}
+
+
+/**
+ * Zaps the list of modified pages, resetting their modification counters in the process.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+static void pgmPoolMonitorModifiedClearAll(PVMCC pVM)
+{
+    PGM_LOCK_VOID(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    LogFlow(("pgmPoolMonitorModifiedClearAll: cModifiedPages=%d\n", pPool->cModifiedPages));
+
+    unsigned cPages = 0; NOREF(cPages);
+
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    pgmPoolResetDirtyPages(pVM);
+#endif
+
+    uint16_t idx = pPool->iModifiedHead;
+    pPool->iModifiedHead = NIL_PGMPOOL_IDX;
+    while (idx != NIL_PGMPOOL_IDX)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[idx];
+        idx = pPage->iModifiedNext;
+        pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+        pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+        pPage->cModifications = 0;
+        Assert(++cPages);
+    }
+    AssertMsg(cPages == pPool->cModifiedPages, ("%d != %d\n", cPages, pPool->cModifiedPages));
+    pPool->cModifiedPages = 0;
+    PGM_UNLOCK(pVM);
+}
+
+
+/**
+ * Handle SyncCR3 pool tasks
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully added.
+ * @retval  VINF_PGM_SYNC_CR3 is it needs to be deferred to ring 3 (GC only)
+ * @param   pVCpu     The cross context virtual CPU structure.
+ * @remark  Should only be used when monitoring is available, thus placed in
+ *          the PGMPOOL_WITH_MONITORING \#ifdef.
+ */
+int pgmPoolSyncCR3(PVMCPUCC pVCpu)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    LogFlow(("pgmPoolSyncCR3 fSyncFlags=%x\n", pVCpu->pgm.s.fSyncFlags));
+
+    /*
+     * When monitoring shadowed pages, we reset the modification counters on CR3 sync.
+     * Occasionally we will have to clear all the shadow page tables because we wanted
+     * to monitor a page which was mapped by too many shadowed page tables. This operation
+     * sometimes referred to as a 'lightweight flush'.
+     */
+# ifdef IN_RING3 /* Don't flush in ring-0 or raw mode, it's taking too long. */
+    if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
+        pgmR3PoolClearAll(pVM, false /*fFlushRemTlb*/);
+# else  /* !IN_RING3 */
+    if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
+    {
+        Log(("SyncCR3: PGM_SYNC_CLEAR_PGM_POOL is set -> VINF_PGM_SYNC_CR3\n"));
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3); /** @todo no need to do global sync, right? */
+
+        /* Make sure all other VCPUs return to ring 3. */
+        if (pVM->cCpus > 1)
+        {
+            VM_FF_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING);
+            PGM_INVL_ALL_VCPU_TLBS(pVM);
+        }
+        return VINF_PGM_SYNC_CR3;
+    }
+# endif /* !IN_RING3 */
+    else
+    {
+        pgmPoolMonitorModifiedClearAll(pVM);
+
+        /* pgmPoolMonitorModifiedClearAll can cause a pgm pool flush (dirty page clearing), so make sure we handle this! */
+        if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
+        {
+            Log(("pgmPoolMonitorModifiedClearAll caused a pgm flush -> call pgmPoolSyncCR3 again!\n"));
+            return pgmPoolSyncCR3(pVCpu);
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Frees up at least one user entry.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully added.
+ *
+ * @param   pPool       The pool.
+ * @param   iUser       The user index.
+ */
+static int pgmPoolTrackFreeOneUser(PPGMPOOL pPool, uint16_t iUser)
+{
+    STAM_COUNTER_INC(&pPool->StatTrackFreeUpOneUser);
+    /*
+     * Just free cached pages in a braindead fashion.
+     */
+    /** @todo walk the age list backwards and free the first with usage. */
+    int rc = VINF_SUCCESS;
+    do
+    {
+        int rc2 = pgmPoolCacheFreeOne(pPool, iUser);
+        if (RT_FAILURE(rc2) && rc == VINF_SUCCESS)
+            rc = rc2;
+    } while (pPool->iUserFreeHead == NIL_PGMPOOL_USER_INDEX);
+    return rc;
+}
+
+
+/**
+ * Inserts a page into the cache.
+ *
+ * This will create user node for the page, insert it into the GCPhys
+ * hash, and insert it into the age list.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully added.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ * @param   GCPhys      The GC physical address of the page we're gonna shadow.
+ * @param   iUser       The user index.
+ * @param   iUserTable  The user table index.
+ */
+DECLINLINE(int) pgmPoolTrackInsert(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTGCPHYS GCPhys, uint16_t iUser, uint32_t iUserTable)
+{
+    int rc = VINF_SUCCESS;
+    PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
+
+    LogFlow(("pgmPoolTrackInsert GCPhys=%RGp iUser=%d iUserTable=%x\n", GCPhys, iUser, iUserTable)); RT_NOREF_PV(GCPhys);
+
+    if (iUser != NIL_PGMPOOL_IDX)
+    {
+#ifdef VBOX_STRICT
+        /*
+         * Check that the entry doesn't already exists.
+         */
+        if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
+        {
+            uint16_t i = pPage->iUserHead;
+            do
+            {
+                Assert(i < pPool->cMaxUsers);
+                AssertMsg(paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
+                i = paUsers[i].iNext;
+            } while (i != NIL_PGMPOOL_USER_INDEX);
+        }
+#endif
+
+        /*
+         * Find free a user node.
+         */
+        uint16_t i = pPool->iUserFreeHead;
+        if (i == NIL_PGMPOOL_USER_INDEX)
+        {
+            rc = pgmPoolTrackFreeOneUser(pPool, iUser);
+            if (RT_FAILURE(rc))
+                return rc;
+            i = pPool->iUserFreeHead;
+        }
+
+        /*
+         * Unlink the user node from the free list,
+         * initialize and insert it into the user list.
+         */
+        pPool->iUserFreeHead = paUsers[i].iNext;
+        paUsers[i].iNext = NIL_PGMPOOL_USER_INDEX;
+        paUsers[i].iUser = iUser;
+        paUsers[i].iUserTable = iUserTable;
+        pPage->iUserHead = i;
+    }
+    else
+        pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
+
+
+    /*
+     * Insert into cache and enable monitoring of the guest page if enabled.
+     *
+     * Until we implement caching of all levels, including the CR3 one, we'll
+     * have to make sure we don't try monitor & cache any recursive reuse of
+     * a monitored CR3 page. Because all windows versions are doing this we'll
+     * have to be able to do combined access monitoring, CR3 + PT and
+     * PD + PT (guest PAE).
+     *
+     * Update:
+     * We're now cooperating with the CR3 monitor if an uncachable page is found.
+     */
+    const bool fCanBeMonitored = true;
+    pgmPoolCacheInsert(pPool, pPage, fCanBeMonitored); /* This can be expanded. */
+    if (fCanBeMonitored)
+    {
+        rc = pgmPoolMonitorInsert(pPool, pPage);
+        AssertRC(rc);
+    }
+    return rc;
+}
+
+
+/**
+ * Adds a user reference to a page.
+ *
+ * This will move the page to the head of the
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if successfully added.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The cached page.
+ * @param   iUser       The user index.
+ * @param   iUserTable  The user table.
+ */
+static int pgmPoolTrackAddUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
+{
+    Log3(("pgmPoolTrackAddUser: GCPhys=%RGp iUser=%x iUserTable=%x\n", pPage->GCPhys, iUser, iUserTable));
+    PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
+    Assert(iUser != NIL_PGMPOOL_IDX);
+
+#  ifdef VBOX_STRICT
+    /*
+     * Check that the entry doesn't already exists. We only allow multiple
+     * users of top-level paging structures (SHW_POOL_ROOT_IDX).
+     */
+    if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
+    {
+        uint16_t i = pPage->iUserHead;
+        do
+        {
+            Assert(i < pPool->cMaxUsers);
+            /** @todo this assertion looks odd... Shouldn't it be && here? */
+            AssertMsg(paUsers[i].iUser != iUser || paUsers[i].iUserTable != iUserTable, ("%x %x vs new %x %x\n", paUsers[i].iUser, paUsers[i].iUserTable, iUser, iUserTable));
+            i = paUsers[i].iNext;
+        } while (i != NIL_PGMPOOL_USER_INDEX);
+    }
+#  endif
+
+    /*
+     * Allocate a user node.
+     */
+    uint16_t i = pPool->iUserFreeHead;
+    if (i == NIL_PGMPOOL_USER_INDEX)
+    {
+        int rc = pgmPoolTrackFreeOneUser(pPool, iUser);
+        if (RT_FAILURE(rc))
+            return rc;
+        i = pPool->iUserFreeHead;
+    }
+    pPool->iUserFreeHead = paUsers[i].iNext;
+
+    /*
+     * Initialize the user node and insert it.
+     */
+    paUsers[i].iNext = pPage->iUserHead;
+    paUsers[i].iUser = iUser;
+    paUsers[i].iUserTable = iUserTable;
+    pPage->iUserHead = i;
+
+#  ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    if (pPage->fDirty)
+        pgmPoolFlushDirtyPage(pPool->CTX_SUFF(pVM), pPool, pPage->idxDirtyEntry, false /* do not remove */);
+#  endif
+
+    /*
+     * Tell the cache to update its replacement stats for this page.
+     */
+    pgmPoolCacheUsed(pPool, pPage);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Frees a user record associated with a page.
+ *
+ * This does not clear the entry in the user table, it simply replaces the
+ * user record to the chain of free records.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The shadow page.
+ * @param   iUser       The shadow page pool index of the user table.
+ * @param   iUserTable  The index into the user table (shadowed).
+ *
+ * @remarks Don't call this for root pages.
+ */
+static void pgmPoolTrackFreeUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
+{
+    Log3(("pgmPoolTrackFreeUser %RGp %x %x\n", pPage->GCPhys, iUser, iUserTable));
+    PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
+    Assert(iUser != NIL_PGMPOOL_IDX);
+
+    /*
+     * Unlink and free the specified user entry.
+     */
+
+    /* Special: For PAE and 32-bit paging, there is usually no more than one user. */
+    uint16_t i = pPage->iUserHead;
+    if (    i != NIL_PGMPOOL_USER_INDEX
+        &&  paUsers[i].iUser == iUser
+        &&  paUsers[i].iUserTable == iUserTable)
+    {
+        pPage->iUserHead = paUsers[i].iNext;
+
+        paUsers[i].iUser = NIL_PGMPOOL_IDX;
+        paUsers[i].iNext = pPool->iUserFreeHead;
+        pPool->iUserFreeHead = i;
+        return;
+    }
+
+    /* General: Linear search. */
+    uint16_t iPrev = NIL_PGMPOOL_USER_INDEX;
+    while (i != NIL_PGMPOOL_USER_INDEX)
+    {
+        if (    paUsers[i].iUser == iUser
+            &&  paUsers[i].iUserTable == iUserTable)
+        {
+            if (iPrev != NIL_PGMPOOL_USER_INDEX)
+                paUsers[iPrev].iNext = paUsers[i].iNext;
+            else
+                pPage->iUserHead = paUsers[i].iNext;
+
+            paUsers[i].iUser = NIL_PGMPOOL_IDX;
+            paUsers[i].iNext = pPool->iUserFreeHead;
+            pPool->iUserFreeHead = i;
+            return;
+        }
+        iPrev = i;
+        i = paUsers[i].iNext;
+    }
+
+    /* Fatal: didn't find it */
+    AssertFatalMsgFailed(("Didn't find the user entry! iUser=%d iUserTable=%#x GCPhys=%RGp\n",
+                          iUser, iUserTable, pPage->GCPhys));
+}
+
+
+#if 0 /* unused */
+/**
+ * Gets the entry size of a shadow table.
+ *
+ * @param   enmKind     The kind of page.
+ *
+ * @returns The size of the entry in bytes. That is, 4 or 8.
+ * @returns If the kind is not for a table, an assertion is raised and 0 is
+ *          returned.
+ */
+DECLINLINE(unsigned) pgmPoolTrackGetShadowEntrySize(PGMPOOLKIND enmKind)
+{
+    switch (enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            return 4;
+
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_PAE_PDPT:
+        case PGMPOOLKIND_ROOT_NESTED:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+            return 8;
+
+        default:
+            AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
+    }
+}
+#endif /* unused */
+
+#if 0 /* unused */
+/**
+ * Gets the entry size of a guest table.
+ *
+ * @param   enmKind     The kind of page.
+ *
+ * @returns The size of the entry in bytes. That is, 0, 4 or 8.
+ * @returns If the kind is not for a table, an assertion is raised and 0 is
+ *          returned.
+ */
+DECLINLINE(unsigned) pgmPoolTrackGetGuestEntrySize(PGMPOOLKIND enmKind)
+{
+    switch (enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+            return 4;
+
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_PAE_PDPT:
+            return 8;
+
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+        case PGMPOOLKIND_ROOT_NESTED:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            /** @todo can we return 0? (nobody is calling this...) */
+            AssertFailed();
+            return 0;
+
+        default:
+            AssertFatalMsgFailed(("enmKind=%d\n", enmKind));
+    }
+}
+#endif /* unused */
+
+
+/**
+ * Checks one shadow page table entry for a mapping of a physical page.
+ *
+ * @returns true / false indicating removal of all relevant PTEs
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPhysPage   The guest page in question.
+ * @param   fFlushPTEs  Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
+ * @param   iShw        The shadow page table.
+ * @param   iPte        Page table entry or NIL_PGMPOOL_PHYSEXT_IDX_PTE if unknown
+ */
+static bool pgmPoolTrackFlushGCPhysPTInt(PVM pVM, PCPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw, uint16_t iPte)
+{
+    LogFlow(("pgmPoolTrackFlushGCPhysPTInt: pPhysPage=%RHp iShw=%d iPte=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw, iPte));
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    bool     fRet  = false;
+
+    /*
+     * Assert sanity.
+     */
+    Assert(iPte != NIL_PGMPOOL_PHYSEXT_IDX_PTE);
+    AssertFatalMsg(iShw < pPool->cCurPages && iShw != NIL_PGMPOOL_IDX, ("iShw=%d\n", iShw));
+    PPGMPOOLPAGE pPage = &pPool->aPages[iShw];
+
+    /*
+     * Then, clear the actual mappings to the page in the shadow PT.
+     */
+    switch (pPage->enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+        {
+            const uint32_t  u32 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
+            PX86PT          pPT = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+            uint32_t        u32AndMask = 0;
+            uint32_t        u32OrMask  = 0;
+
+            if (!fFlushPTEs)
+            {
+                /* Note! Disregarding the PGMPHYSHANDLER_F_NOT_IN_HM bit here. Should be harmless. */
+                switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
+                {
+                    case PGM_PAGE_HNDL_PHYS_STATE_NONE:         /* No handler installed. */
+                    case PGM_PAGE_HNDL_PHYS_STATE_DISABLED:     /* Monitoring is temporarily disabled. */
+                        u32OrMask = X86_PTE_RW;
+                        u32AndMask = UINT32_MAX;
+                        fRet = true;
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
+                        break;
+
+                    case PGM_PAGE_HNDL_PHYS_STATE_WRITE:        /* Write access is monitored. */
+                        u32OrMask = 0;
+                        u32AndMask = ~X86_PTE_RW;
+                        fRet = true;
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
+                        break;
+                    default:
+                        /* We will end up here when called with an "ALL" access handler. */
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+                        break;
+                }
+            }
+            else
+                STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+
+            /* Update the counter if we're removing references. */
+            if (!u32AndMask)
+            {
+                Assert(pPage->cPresent);
+                Assert(pPool->cPresent);
+                pPage->cPresent--;
+                pPool->cPresent--;
+            }
+
+            if ((pPT->a[iPte].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
+            {
+                Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX32\n", iPte, pPT->a[iPte]));
+                X86PTE Pte;
+                Pte.u = (pPT->a[iPte].u & u32AndMask) | u32OrMask;
+                if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
+                    Pte.u &= ~(X86PGUINT)X86_PTE_RW; /* need to disallow writes when dirty bit tracking is still active. */
+
+                ASMAtomicWriteU32(&pPT->a[iPte].u, Pte.u);
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
+                return fRet;
+            }
+#ifdef LOG_ENABLED
+            Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
+                if ((pPT->a[i].u & (X86_PTE_PG_MASK | X86_PTE_P)) == u32)
+                {
+                    Log(("i=%d cFound=%d\n", i, ++cFound));
+                }
+#endif
+            AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u32=%RX32 poolkind=%x\n", pPage->iFirstPresent, pPage->cPresent, u32, pPage->enmKind));
+            /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);*/
+            break;
+        }
+
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:   /* physical mask the same as PAE; RW bit as well; be careful! */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+#endif
+        {
+            const uint64_t  u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PTE_P;
+            PPGMSHWPTPAE    pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+            uint64_t        u64OrMask  = 0;
+            uint64_t        u64AndMask = 0;
+
+            if (!fFlushPTEs)
+            {
+                /* Note! Disregarding the PGMPHYSHANDLER_F_NOT_IN_HM bit here. Should be harmless. */
+                switch (PGM_PAGE_GET_HNDL_PHYS_STATE(pPhysPage))
+                {
+                    case PGM_PAGE_HNDL_PHYS_STATE_NONE:         /* No handler installed. */
+                    case PGM_PAGE_HNDL_PHYS_STATE_DISABLED:     /* Monitoring is temporarily disabled. */
+                        u64OrMask = X86_PTE_RW;
+                        u64AndMask = UINT64_MAX;
+                        fRet = true;
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
+                        break;
+
+                    case PGM_PAGE_HNDL_PHYS_STATE_WRITE:        /* Write access is monitored. */
+                        u64OrMask = 0;
+                        u64AndMask = ~(uint64_t)X86_PTE_RW;
+                        fRet = true;
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntryKeep);
+                        break;
+
+                    default:
+                        /* We will end up here when called with an "ALL" access handler. */
+                        STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+                        break;
+                }
+            }
+            else
+                STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+
+            /* Update the counter if we're removing references. */
+            if (!u64AndMask)
+            {
+                Assert(pPage->cPresent);
+                Assert(pPool->cPresent);
+                pPage->cPresent--;
+                pPool->cPresent--;
+            }
+
+            if ((PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK | X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == u64)
+            {
+                Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pte=%RX64\n", iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
+                X86PTEPAE Pte;
+                Pte.u = (PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & u64AndMask) | u64OrMask;
+                if (Pte.u & PGM_PTFLAGS_TRACK_DIRTY)
+                    Pte.u &= ~(X86PGPAEUINT)X86_PTE_RW;    /* need to disallow writes when dirty bit tracking is still active. */
+
+                PGMSHWPTEPAE_ATOMIC_SET(pPT->a[iPte], Pte.u);
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
+                return fRet;
+            }
+#ifdef LOG_ENABLED
+            Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            Log(("Found %RX64 expected %RX64\n", PGMSHWPTEPAE_GET_U(pPT->a[iPte]) & (X86_PTE_PAE_PG_MASK | X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX), u64));
+            for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPT->a); i++)
+                if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & (X86_PTE_PAE_PG_MASK | X86_PTE_P | X86_PTE_PAE_MBZ_MASK_NX)) == u64)
+                    Log(("i=%d cFound=%d\n", i, ++cFound));
+#endif
+            AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d u64=%RX64 poolkind=%x iPte=%d PT=%RX64\n", pPage->iFirstPresent, pPage->cPresent, u64, pPage->enmKind, iPte, PGMSHWPTEPAE_GET_LOG(pPT->a[iPte])));
+            /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);*/
+            break;
+        }
+
+#ifdef PGM_WITH_LARGE_PAGES
+        /* Large page case only. */
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:    /* X86_PDE4M_PS is same as leaf bit in EPT; be careful! */
+#endif
+        {
+            Assert(pVM->pgm.s.fNestedPaging);
+
+            const uint64_t  u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PDE4M_P | X86_PDE4M_PS;
+            PEPTPD          pPD = (PEPTPD)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+
+            if ((pPD->a[iPte].u & (EPT_PDE2M_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64)
+            {
+                Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
+                STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+                pPD->a[iPte].u = 0;
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
+
+                /* Update the counter as we're removing references. */
+                Assert(pPage->cPresent);
+                Assert(pPool->cPresent);
+                pPage->cPresent--;
+                pPool->cPresent--;
+
+                return fRet;
+            }
+# ifdef LOG_ENABLED
+            Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
+                if ((pPD->a[i].u & (EPT_PDE2M_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64)
+                    Log(("i=%d cFound=%d\n", i, ++cFound));
+# endif
+            AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);*/
+            break;
+        }
+
+        /* AMD-V nested paging */ /** @todo merge with EPT as we only check the parts that are identical. */
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        {
+            Assert(pVM->pgm.s.fNestedPaging);
+
+            const uint64_t  u64 = PGM_PAGE_GET_HCPHYS(pPhysPage) | X86_PDE4M_P | X86_PDE4M_PS;
+            PX86PDPAE       pPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+
+            if ((pPD->a[iPte].u & (X86_PDE2M_PAE_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64)
+            {
+                Log4(("pgmPoolTrackFlushGCPhysPTs: i=%d pde=%RX64\n", iPte, pPD->a[iPte]));
+                STAM_COUNTER_INC(&pPool->StatTrackFlushEntry);
+                pPD->a[iPte].u = 0;
+                PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);
+
+                /* Update the counter as we're removing references. */
+                Assert(pPage->cPresent);
+                Assert(pPool->cPresent);
+                pPage->cPresent--;
+                pPool->cPresent--;
+                return fRet;
+            }
+# ifdef LOG_ENABLED
+            Log(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            for (unsigned i = 0, cFound = 0; i < RT_ELEMENTS(pPD->a); i++)
+                if ((pPD->a[i].u & (X86_PDE2M_PAE_PG_MASK | X86_PDE4M_P | X86_PDE4M_PS)) == u64)
+                    Log(("i=%d cFound=%d\n", i, ++cFound));
+# endif
+            AssertFatalMsgFailed(("iFirstPresent=%d cPresent=%d\n", pPage->iFirstPresent, pPage->cPresent));
+            /*PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPD);*/
+            break;
+        }
+#endif /* PGM_WITH_LARGE_PAGES */
+
+        default:
+            AssertFatalMsgFailed(("enmKind=%d iShw=%d\n", pPage->enmKind, iShw));
+    }
+
+    /* not reached. */
+#ifndef _MSC_VER
+    return fRet;
+#endif
+}
+
+
+/**
+ * Scans one shadow page table for mappings of a physical page.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPhysPage   The guest page in question.
+ * @param   fFlushPTEs  Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
+ * @param   iShw        The shadow page table.
+ */
+static void pgmPoolTrackFlushGCPhysPT(PVM pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iShw)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool); NOREF(pPool);
+
+    /* We should only come here with when there's only one reference to this physical page. */
+    Assert(PGMPOOL_TD_GET_CREFS(PGM_PAGE_GET_TRACKING(pPhysPage)) == 1);
+
+    Log2(("pgmPoolTrackFlushGCPhysPT: pPhysPage=%RHp iShw=%d\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iShw));
+    STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPT, f);
+    bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, iShw, PGM_PAGE_GET_PTE_INDEX(pPhysPage));
+    if (!fKeptPTEs)
+        PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
+    STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPT, f);
+}
+
+
+/**
+ * Flushes a list of shadow page tables mapping the same physical page.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPhysPage   The guest page in question.
+ * @param   fFlushPTEs  Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
+ * @param   iPhysExt    The physical cross reference extent list to flush.
+ */
+static void pgmPoolTrackFlushGCPhysPTs(PVMCC pVM, PPGMPAGE pPhysPage, bool fFlushPTEs, uint16_t iPhysExt)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    bool     fKeepList = false;
+
+    STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTs, f);
+    Log2(("pgmPoolTrackFlushGCPhysPTs: pPhysPage=%RHp iPhysExt=%u\n", PGM_PAGE_GET_HCPHYS(pPhysPage), iPhysExt));
+
+    const uint16_t iPhysExtStart = iPhysExt;
+    PPGMPOOLPHYSEXT pPhysExt;
+    do
+    {
+        Assert(iPhysExt < pPool->cMaxPhysExts);
+        pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
+        for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
+        {
+            if (pPhysExt->aidx[i] != NIL_PGMPOOL_IDX)
+            {
+                bool fKeptPTEs = pgmPoolTrackFlushGCPhysPTInt(pVM, pPhysPage, fFlushPTEs, pPhysExt->aidx[i], pPhysExt->apte[i]);
+                if (!fKeptPTEs)
+                {
+                    pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
+                    pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+                }
+                else
+                    fKeepList = true;
+            }
+        }
+        /* next */
+        iPhysExt = pPhysExt->iNext;
+    } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
+
+    if (!fKeepList)
+    {
+        /* insert the list into the free list and clear the ram range entry. */
+        pPhysExt->iNext = pPool->iPhysExtFreeHead;
+        pPool->iPhysExtFreeHead = iPhysExtStart;
+        /* Invalidate the tracking data. */
+        PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
+    }
+
+    STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTs, f);
+}
+
+
+/**
+ * Flushes all shadow page table mappings of the given guest page.
+ *
+ * This is typically called when the host page backing the guest one has been
+ * replaced or when the page protection was changed due to a guest access
+ * caught by the monitoring.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if all references has been successfully cleared.
+ * @retval  VINF_PGM_SYNC_CR3 if we're better off with a CR3 sync and a page
+ *          pool cleaning. FF and sync flags are set.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhysPage  GC physical address of the page in question
+ * @param   pPhysPage   The guest page in question.
+ * @param   fFlushPTEs  Flush PTEs or allow them to be updated (e.g. in case of an RW bit change)
+ * @param   pfFlushTLBs This is set to @a true if the shadow TLBs should be
+ *                      flushed, it is NOT touched if this isn't necessary.
+ *                      The caller MUST initialized this to @a false.
+ */
+int pgmPoolTrackUpdateGCPhys(PVMCC pVM, RTGCPHYS GCPhysPage, PPGMPAGE pPhysPage, bool fFlushPTEs, bool *pfFlushTLBs)
+{
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    PGM_LOCK_VOID(pVM);
+    int rc = VINF_SUCCESS;
+
+#ifdef PGM_WITH_LARGE_PAGES
+    /* Is this page part of a large page? */
+    if (PGM_PAGE_GET_PDE_TYPE(pPhysPage) == PGM_PAGE_PDE_TYPE_PDE)
+    {
+        RTGCPHYS GCPhysBase = GCPhysPage & X86_PDE2M_PAE_PG_MASK;
+        GCPhysPage &= X86_PDE_PAE_PG_MASK;
+
+        /* Fetch the large page base. */
+        PPGMPAGE pLargePage;
+        if (GCPhysBase != GCPhysPage)
+        {
+            pLargePage = pgmPhysGetPage(pVM, GCPhysBase);
+            AssertFatal(pLargePage);
+        }
+        else
+            pLargePage = pPhysPage;
+
+        Log(("pgmPoolTrackUpdateGCPhys: update large page PDE for %RGp (%RGp)\n", GCPhysBase, GCPhysPage));
+
+        if (PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE)
+        {
+            /* Mark the large page as disabled as we need to break it up to change a single page in the 2 MB range. */
+            PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE_DISABLED);
+            pVM->pgm.s.cLargePagesDisabled++;
+
+            /* Update the base as that *only* that one has a reference and there's only one PDE to clear. */
+            rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhysBase, pLargePage, fFlushPTEs, pfFlushTLBs);
+
+            *pfFlushTLBs = true;
+            PGM_UNLOCK(pVM);
+            return rc;
+        }
+    }
+#else
+    NOREF(GCPhysPage);
+#endif /* PGM_WITH_LARGE_PAGES */
+
+    const uint16_t u16 = PGM_PAGE_GET_TRACKING(pPhysPage);
+    if (u16)
+    {
+        /*
+         * The zero page is currently screwing up the tracking and we'll
+         * have to flush the whole shebang. Unless VBOX_WITH_NEW_LAZY_PAGE_ALLOC
+         * is defined, zero pages won't normally be mapped. Some kind of solution
+         * will be needed for this problem of course, but it will have to wait...
+         */
+        if (    PGM_PAGE_IS_ZERO(pPhysPage)
+            ||  PGM_PAGE_IS_BALLOONED(pPhysPage))
+            rc = VINF_PGM_GCPHYS_ALIASED;
+        else
+        {
+            if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
+            {
+                Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
+                pgmPoolTrackFlushGCPhysPT(pVM,
+                                          pPhysPage,
+                                          fFlushPTEs,
+                                          PGMPOOL_TD_GET_IDX(u16));
+            }
+            else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
+                pgmPoolTrackFlushGCPhysPTs(pVM, pPhysPage, fFlushPTEs, PGMPOOL_TD_GET_IDX(u16));
+            else
+                rc = pgmPoolTrackFlushGCPhysPTsSlow(pVM, pPhysPage);
+            *pfFlushTLBs = true;
+        }
+    }
+
+    if (rc == VINF_PGM_GCPHYS_ALIASED)
+    {
+        pVCpu->pgm.s.fSyncFlags |= PGM_SYNC_CLEAR_PGM_POOL;
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        rc = VINF_PGM_SYNC_CR3;
+    }
+    PGM_UNLOCK(pVM);
+    return rc;
+}
+
+
+/**
+ * Scans all shadow page tables for mappings of a physical page.
+ *
+ * This may be slow, but it's most likely more efficient than cleaning
+ * out the entire page pool / cache.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if all references has been successfully cleared.
+ * @retval  VINF_PGM_GCPHYS_ALIASED if we're better off with a CR3 sync and
+ *          a page pool cleaning.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPhysPage   The guest page in question.
+ */
+int pgmPoolTrackFlushGCPhysPTsSlow(PVMCC pVM, PPGMPAGE pPhysPage)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    STAM_PROFILE_START(&pPool->StatTrackFlushGCPhysPTsSlow, s);
+    LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: cUsedPages=%d cPresent=%d pPhysPage=%R[pgmpage]\n",
+             pPool->cUsedPages, pPool->cPresent, pPhysPage));
+
+    /*
+     * There is a limit to what makes sense.
+     */
+    if (    pPool->cPresent > 1024
+        &&  pVM->cCpus == 1)
+    {
+        LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
+        STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
+        return VINF_PGM_GCPHYS_ALIASED;
+    }
+
+    /*
+     * Iterate all the pages until we've encountered all that in use.
+     * This is simple but not quite optimal solution.
+     */
+    const uint64_t  u64   = PGM_PAGE_GET_HCPHYS(pPhysPage);
+    unsigned        cLeft = pPool->cUsedPages;
+    unsigned        iPage = pPool->cCurPages;
+    while (--iPage >= PGMPOOL_IDX_FIRST)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[iPage];
+        if (    pPage->GCPhys != NIL_RTGCPHYS
+            &&  pPage->cPresent)
+        {
+            Assert(!PGMPOOL_PAGE_IS_NESTED(pPage)); /* see if it hits */
+            switch (pPage->enmKind)
+            {
+                /*
+                 * We only care about shadow page tables.
+                 */
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                {
+                    const uint32_t  u32      = (uint32_t)u64;
+                    unsigned        cPresent = pPage->cPresent;
+                    PX86PT          pPT      = (PX86PT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
+                    {
+                        const X86PGUINT uPte = pPT->a[i].u;
+                        if (uPte & X86_PTE_P)
+                        {
+                            if ((uPte & X86_PTE_PG_MASK) == u32)
+                            {
+                                //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX32\n", iPage, i, pPT->a[i]));
+                                ASMAtomicWriteU32(&pPT->a[i].u, 0);
+
+                                /* Update the counter as we're removing references. */
+                                Assert(pPage->cPresent);
+                                Assert(pPool->cPresent);
+                                pPage->cPresent--;
+                                pPool->cPresent--;
+                            }
+                            if (!--cPresent)
+                                break;
+                        }
+                    }
+                    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
+                    break;
+                }
+
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                {
+                    unsigned        cPresent = pPage->cPresent;
+                    PPGMSHWPTPAE    pPT = (PPGMSHWPTPAE)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
+                        if (PGMSHWPTEPAE_IS_P(pPT->a[i]))
+                        {
+                            if ((PGMSHWPTEPAE_GET_U(pPT->a[i]) & X86_PTE_PAE_PG_MASK) == u64)
+                            {
+                                //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
+                                PGMSHWPTEPAE_ATOMIC_SET(pPT->a[i], 0); /// @todo why not atomic?
+
+                                /* Update the counter as we're removing references. */
+                                Assert(pPage->cPresent);
+                                Assert(pPool->cPresent);
+                                pPage->cPresent--;
+                                pPool->cPresent--;
+                            }
+                            if (!--cPresent)
+                                break;
+                        }
+                    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
+                    break;
+                }
+
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                {
+                    unsigned  cPresent = pPage->cPresent;
+                    PEPTPT    pPT = (PEPTPT)PGMPOOL_PAGE_2_PTR(pVM, pPage);
+                    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pPT->a); i++)
+                    {
+                        X86PGPAEUINT const uPte = pPT->a[i].u;
+                        if (uPte & EPT_E_READ)
+                        {
+                            if ((uPte & EPT_PTE_PG_MASK) == u64)
+                            {
+                                //Log4(("pgmPoolTrackFlushGCPhysPTsSlow: idx=%d i=%d pte=%RX64\n", iPage, i, pPT->a[i]));
+                                ASMAtomicWriteU64(&pPT->a[i].u, 0);
+
+                                /* Update the counter as we're removing references. */
+                                Assert(pPage->cPresent);
+                                Assert(pPool->cPresent);
+                                pPage->cPresent--;
+                                pPool->cPresent--;
+                            }
+                            if (!--cPresent)
+                                break;
+                        }
+                    }
+                    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pPT);
+                    break;
+                }
+            }
+
+            if (!--cLeft)
+                break;
+        }
+    }
+
+    PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
+    STAM_PROFILE_STOP(&pPool->StatTrackFlushGCPhysPTsSlow, s);
+
+    /*
+     * There is a limit to what makes sense. The above search is very expensive, so force a pgm pool flush.
+     */
+    if (pPool->cPresent > 1024)
+    {
+        LogFlow(("pgmPoolTrackFlushGCPhysPTsSlow: giving up... (cPresent=%d)\n", pPool->cPresent));
+        return VINF_PGM_GCPHYS_ALIASED;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Clears the user entry in a user table.
+ *
+ * This is used to remove all references to a page when flushing it.
+ */
+static void pgmPoolTrackClearPageUser(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PCPGMPOOLUSER pUser)
+{
+    Assert(pUser->iUser != NIL_PGMPOOL_IDX);
+    Assert(pUser->iUser < pPool->cCurPages);
+    uint32_t iUserTable = pUser->iUserTable;
+
+    /*
+     * Map the user page.  Ignore references made by fictitious pages.
+     */
+    PPGMPOOLPAGE pUserPage = &pPool->aPages[pUser->iUser];
+    LogFlow(("pgmPoolTrackClearPageUser: clear %x in %s (%RGp) (flushing %s)\n", iUserTable, pgmPoolPoolKindToStr(pUserPage->enmKind), pUserPage->Core.Key, pgmPoolPoolKindToStr(pPage->enmKind)));
+    union
+    {
+        uint64_t       *pau64;
+        uint32_t       *pau32;
+    } u;
+    if (pUserPage->idx < PGMPOOL_IDX_FIRST)
+    {
+        Assert(!pUserPage->pvPageR3);
+        return;
+    }
+    u.pau64 = (uint64_t *)PGMPOOL_PAGE_2_PTR(pPool->CTX_SUFF(pVM), pUserPage);
+
+
+    /* Safety precaution in case we change the paging for other modes too in the future. */
+    Assert(!pgmPoolIsPageLocked(pPage)); RT_NOREF_PV(pPage);
+
+#ifdef VBOX_STRICT
+    /*
+     * Some sanity checks.
+     */
+    switch (pUserPage->enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            Assert(iUserTable < X86_PG_ENTRIES);
+            break;
+        case PGMPOOLKIND_PAE_PDPT:
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+            Assert(iUserTable < 4);
+            Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
+            break;
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            break;
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            break;
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
+            break;
+        case PGMPOOLKIND_64BIT_PML4:
+            Assert(!(u.pau64[iUserTable] & PGM_PLXFLAGS_PERMANENT));
+            /* GCPhys >> PAGE_SHIFT is the index here */
+            break;
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            break;
+
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            break;
+
+        case PGMPOOLKIND_ROOT_NESTED:
+            Assert(iUserTable < X86_PG_PAE_ENTRIES);
+            break;
+
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            Assert(iUserTable < EPT_PG_ENTRIES);
+            break;
+# endif
+
+        default:
+            AssertMsgFailed(("enmKind=%d GCPhys=%RGp\n", pUserPage->enmKind, pPage->GCPhys));
+            break;
+    }
+#endif /* VBOX_STRICT */
+
+    /*
+     * Clear the entry in the user page.
+     */
+    switch (pUserPage->enmKind)
+    {
+        /* 32-bit entries */
+        case PGMPOOLKIND_32BIT_PD:
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            ASMAtomicWriteU32(&u.pau32[iUserTable], 0);
+            break;
+
+        /* 64-bit entries */
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+        case PGMPOOLKIND_64BIT_PML4:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+        case PGMPOOLKIND_PAE_PDPT:
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+        case PGMPOOLKIND_ROOT_NESTED:
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+#endif
+            ASMAtomicWriteU64(&u.pau64[iUserTable], 0);
+            break;
+
+        default:
+            AssertFatalMsgFailed(("enmKind=%d iUser=%d iUserTable=%#x\n", pUserPage->enmKind, pUser->iUser, pUser->iUserTable));
+    }
+    PGM_DYNMAP_UNUSED_HINT_VM(pPool->CTX_SUFF(pVM), u.pau64);
+}
+
+
+/**
+ * Clears all users of a page.
+ */
+static void pgmPoolTrackClearPageUsers(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    /*
+     * Free all the user records.
+     */
+    LogFlow(("pgmPoolTrackClearPageUsers %RGp\n", pPage->GCPhys));
+
+    PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
+    uint16_t i = pPage->iUserHead;
+    while (i != NIL_PGMPOOL_USER_INDEX)
+    {
+        /* Clear enter in user table. */
+        pgmPoolTrackClearPageUser(pPool, pPage, &paUsers[i]);
+
+        /* Free it. */
+        const uint16_t iNext = paUsers[i].iNext;
+        paUsers[i].iUser = NIL_PGMPOOL_IDX;
+        paUsers[i].iNext = pPool->iUserFreeHead;
+        pPool->iUserFreeHead = i;
+
+        /* Next. */
+        i = iNext;
+    }
+    pPage->iUserHead = NIL_PGMPOOL_USER_INDEX;
+}
+
+
+/**
+ * Allocates a new physical cross reference extent.
+ *
+ * @returns Pointer to the allocated extent on success. NULL if we're out of them.
+ * @param   pVM         The cross context VM structure.
+ * @param   piPhysExt   Where to store the phys ext index.
+ */
+PPGMPOOLPHYSEXT pgmPoolTrackPhysExtAlloc(PVMCC pVM, uint16_t *piPhysExt)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    uint16_t iPhysExt = pPool->iPhysExtFreeHead;
+    if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
+    {
+        STAM_COUNTER_INC(&pPool->StamTrackPhysExtAllocFailures);
+        return NULL;
+    }
+    PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
+    pPool->iPhysExtFreeHead = pPhysExt->iNext;
+    pPhysExt->iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
+    *piPhysExt = iPhysExt;
+    return pPhysExt;
+}
+
+
+/**
+ * Frees a physical cross reference extent.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   iPhysExt    The extent to free.
+ */
+void pgmPoolTrackPhysExtFree(PVMCC pVM, uint16_t iPhysExt)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    Assert(iPhysExt < pPool->cMaxPhysExts);
+    PPGMPOOLPHYSEXT pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
+    for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
+    {
+        pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
+        pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+    }
+    pPhysExt->iNext = pPool->iPhysExtFreeHead;
+    pPool->iPhysExtFreeHead = iPhysExt;
+}
+
+
+/**
+ * Frees a physical cross reference extent.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   iPhysExt    The extent to free.
+ */
+void pgmPoolTrackPhysExtFreeList(PVMCC pVM, uint16_t iPhysExt)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    const uint16_t  iPhysExtStart = iPhysExt;
+    PPGMPOOLPHYSEXT pPhysExt;
+    do
+    {
+        Assert(iPhysExt < pPool->cMaxPhysExts);
+        pPhysExt = &pPool->CTX_SUFF(paPhysExts)[iPhysExt];
+        for (unsigned i = 0; i < RT_ELEMENTS(pPhysExt->aidx); i++)
+        {
+            pPhysExt->aidx[i] = NIL_PGMPOOL_IDX;
+            pPhysExt->apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        }
+
+        /* next */
+        iPhysExt = pPhysExt->iNext;
+    } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
+
+    pPhysExt->iNext = pPool->iPhysExtFreeHead;
+    pPool->iPhysExtFreeHead = iPhysExtStart;
+}
+
+
+/**
+ * Insert a reference into a list of physical cross reference extents.
+ *
+ * @returns The new tracking data for PGMPAGE.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   iPhysExt    The physical extent index of the list head.
+ * @param   iShwPT      The shadow page table index.
+ * @param   iPte        Page table entry
+ *
+ */
+static uint16_t pgmPoolTrackPhysExtInsert(PVMCC pVM, uint16_t iPhysExt, uint16_t iShwPT, uint16_t iPte)
+{
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    PPGMPOOL        pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
+
+    /*
+     * Special common cases.
+     */
+    if (paPhysExts[iPhysExt].aidx[1] == NIL_PGMPOOL_IDX)
+    {
+        paPhysExts[iPhysExt].aidx[1] = iShwPT;
+        paPhysExts[iPhysExt].apte[1] = iPte;
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
+        LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,%d pte %d,}\n", iPhysExt, iShwPT, iPte));
+        return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
+    }
+    if (paPhysExts[iPhysExt].aidx[2] == NIL_PGMPOOL_IDX)
+    {
+        paPhysExts[iPhysExt].aidx[2] = iShwPT;
+        paPhysExts[iPhysExt].apte[2] = iPte;
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
+        LogFlow(("pgmPoolTrackPhysExtInsert: %d:{,,%d pte %d}\n", iPhysExt, iShwPT, iPte));
+        return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
+    }
+    AssertCompile(RT_ELEMENTS(paPhysExts[iPhysExt].aidx) == 3);
+
+    /*
+     * General treatment.
+     */
+    const uint16_t iPhysExtStart = iPhysExt;
+    unsigned cMax = 15;
+    for (;;)
+    {
+        Assert(iPhysExt < pPool->cMaxPhysExts);
+        for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
+            if (paPhysExts[iPhysExt].aidx[i] == NIL_PGMPOOL_IDX)
+            {
+                paPhysExts[iPhysExt].aidx[i] = iShwPT;
+                paPhysExts[iPhysExt].apte[i] = iPte;
+                STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedMany);
+                LogFlow(("pgmPoolTrackPhysExtInsert: %d:{%d pte %d} i=%d cMax=%d\n", iPhysExt, iShwPT, iPte, i, cMax));
+                return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtStart);
+            }
+        if (!--cMax)
+        {
+            STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackOverflows);
+            pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
+            LogFlow(("pgmPoolTrackPhysExtInsert: overflow (1) iShwPT=%d\n", iShwPT));
+            return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
+        }
+
+        /* advance */
+        iPhysExt = paPhysExts[iPhysExt].iNext;
+        if (iPhysExt == NIL_PGMPOOL_PHYSEXT_INDEX)
+            break;
+    }
+
+    /*
+     * Add another extent to the list.
+     */
+    PPGMPOOLPHYSEXT pNew = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
+    if (!pNew)
+    {
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackNoExtentsLeft);
+        pgmPoolTrackPhysExtFreeList(pVM, iPhysExtStart);
+        LogFlow(("pgmPoolTrackPhysExtInsert: pgmPoolTrackPhysExtAlloc failed iShwPT=%d\n", iShwPT));
+        return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
+    }
+    pNew->iNext = iPhysExtStart;
+    pNew->aidx[0] = iShwPT;
+    pNew->apte[0] = iPte;
+    LogFlow(("pgmPoolTrackPhysExtInsert: added new extent %d:{%d pte %d}->%d\n", iPhysExt, iShwPT, iPte, iPhysExtStart));
+    return PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
+}
+
+
+/**
+ * Add a reference to guest physical page where extents are in use.
+ *
+ * @returns The new tracking data for PGMPAGE.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pPhysPage   Pointer to the aPages entry in the ram range.
+ * @param   u16         The ram range flags (top 16-bits).
+ * @param   iShwPT      The shadow page table index.
+ * @param   iPte        Page table entry
+ */
+uint16_t pgmPoolTrackPhysExtAddref(PVMCC pVM, PPGMPAGE pPhysPage, uint16_t u16, uint16_t iShwPT, uint16_t iPte)
+{
+    PGM_LOCK_VOID(pVM);
+    if (PGMPOOL_TD_GET_CREFS(u16) != PGMPOOL_TD_CREFS_PHYSEXT)
+    {
+        /*
+         * Convert to extent list.
+         */
+        Assert(PGMPOOL_TD_GET_CREFS(u16) == 1);
+        uint16_t iPhysExt;
+        PPGMPOOLPHYSEXT pPhysExt = pgmPoolTrackPhysExtAlloc(pVM, &iPhysExt);
+        if (pPhysExt)
+        {
+            LogFlow(("pgmPoolTrackPhysExtAddref: new extent: %d:{%d, %d}\n", iPhysExt, PGMPOOL_TD_GET_IDX(u16), iShwPT));
+            STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliased);
+            pPhysExt->aidx[0] = PGMPOOL_TD_GET_IDX(u16);
+            pPhysExt->apte[0] = PGM_PAGE_GET_PTE_INDEX(pPhysPage);
+            pPhysExt->aidx[1] = iShwPT;
+            pPhysExt->apte[1] = iPte;
+            u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExt);
+        }
+        else
+            u16 = PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED);
+    }
+    else if (u16 != PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, PGMPOOL_TD_IDX_OVERFLOWED))
+    {
+        /*
+         * Insert into the extent list.
+         */
+        u16 = pgmPoolTrackPhysExtInsert(pVM, PGMPOOL_TD_GET_IDX(u16), iShwPT, iPte);
+    }
+    else
+        STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatTrackAliasedLots);
+    PGM_UNLOCK(pVM);
+    return u16;
+}
+
+
+/**
+ * Clear references to guest physical memory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pPhysPage   Pointer to the aPages entry in the ram range.
+ * @param   iPte        Shadow PTE index
+ */
+void pgmPoolTrackPhysExtDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMPAGE pPhysPage, uint16_t iPte)
+{
+    PVMCC          pVM = pPool->CTX_SUFF(pVM);
+    const unsigned cRefs = PGM_PAGE_GET_TD_CREFS(pPhysPage);
+    AssertFatalMsg(cRefs == PGMPOOL_TD_CREFS_PHYSEXT, ("cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
+
+    uint16_t iPhysExt = PGM_PAGE_GET_TD_IDX(pPhysPage);
+    if (iPhysExt != PGMPOOL_TD_IDX_OVERFLOWED)
+    {
+        PGM_LOCK_VOID(pVM);
+
+        uint16_t        iPhysExtPrev = NIL_PGMPOOL_PHYSEXT_INDEX;
+        PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
+        do
+        {
+            Assert(iPhysExt < pPool->cMaxPhysExts);
+
+            /*
+             * Look for the shadow page and check if it's all freed.
+             */
+            for (unsigned i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
+            {
+                if (    paPhysExts[iPhysExt].aidx[i] == pPage->idx
+                    &&  paPhysExts[iPhysExt].apte[i] == iPte)
+                {
+                    paPhysExts[iPhysExt].aidx[i] = NIL_PGMPOOL_IDX;
+                    paPhysExts[iPhysExt].apte[i] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+
+                    for (i = 0; i < RT_ELEMENTS(paPhysExts[iPhysExt].aidx); i++)
+                        if (paPhysExts[iPhysExt].aidx[i] != NIL_PGMPOOL_IDX)
+                        {
+                            Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d\n", pPhysPage, pPage->idx));
+                            PGM_UNLOCK(pVM);
+                            return;
+                        }
+
+                    /* we can free the node. */
+                    const uint16_t iPhysExtNext = paPhysExts[iPhysExt].iNext;
+                    if (    iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX
+                        &&  iPhysExtNext == NIL_PGMPOOL_PHYSEXT_INDEX)
+                    {
+                        /* lonely node */
+                        pgmPoolTrackPhysExtFree(pVM, iPhysExt);
+                        Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d lonely\n", pPhysPage, pPage->idx));
+                        PGM_PAGE_SET_TRACKING(pVM, pPhysPage, 0);
+                    }
+                    else if (iPhysExtPrev == NIL_PGMPOOL_PHYSEXT_INDEX)
+                    {
+                        /* head */
+                        Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d head\n", pPhysPage, pPage->idx));
+                        PGM_PAGE_SET_TRACKING(pVM, pPhysPage, PGMPOOL_TD_MAKE(PGMPOOL_TD_CREFS_PHYSEXT, iPhysExtNext));
+                        pgmPoolTrackPhysExtFree(pVM, iPhysExt);
+                    }
+                    else
+                    {
+                        /* in list */
+                        Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage] idx=%d in list\n", pPhysPage, pPage->idx));
+                        paPhysExts[iPhysExtPrev].iNext = iPhysExtNext;
+                        pgmPoolTrackPhysExtFree(pVM, iPhysExt);
+                    }
+                    iPhysExt = iPhysExtNext;
+                    PGM_UNLOCK(pVM);
+                    return;
+                }
+            }
+
+            /* next */
+            iPhysExtPrev = iPhysExt;
+            iPhysExt = paPhysExts[iPhysExt].iNext;
+        } while (iPhysExt != NIL_PGMPOOL_PHYSEXT_INDEX);
+
+        PGM_UNLOCK(pVM);
+        AssertFatalMsgFailed(("not-found! cRefs=%d pPhysPage=%R[pgmpage] pPage=%p:{.idx=%d}\n", cRefs, pPhysPage, pPage, pPage->idx));
+    }
+    else /* nothing to do */
+        Log2(("pgmPoolTrackPhysExtDerefGCPhys: pPhysPage=%R[pgmpage]\n", pPhysPage));
+}
+
+/**
+ * Clear references to guest physical memory.
+ *
+ * This is the same as pgmPoolTracDerefGCPhysHint except that the guest
+ * physical address is assumed to be correct, so the linear search can be
+ * skipped and we can assert at an earlier point.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   HCPhys      The host physical address corresponding to the guest page.
+ * @param   GCPhys      The guest physical address corresponding to HCPhys.
+ * @param   iPte        Shadow PTE index
+ */
+static void pgmPoolTracDerefGCPhys(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhys, uint16_t iPte)
+{
+    /*
+     * Lookup the page and check if it checks out before derefing it.
+     */
+    PVMCC    pVM       = pPool->CTX_SUFF(pVM);
+    PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhys);
+    if (pPhysPage)
+    {
+        Assert(PGM_PAGE_GET_HCPHYS(pPhysPage));
+#ifdef LOG_ENABLED
+        RTHCPHYS HCPhysPage = PGM_PAGE_GET_HCPHYS(pPhysPage);
+        Log2(("pgmPoolTracDerefGCPhys %RHp vs %RHp\n", HCPhysPage, HCPhys));
+#endif
+        if (PGM_PAGE_GET_HCPHYS(pPhysPage) == HCPhys)
+        {
+            Assert(pPage->cPresent);
+            Assert(pPool->cPresent);
+            pPage->cPresent--;
+            pPool->cPresent--;
+            pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
+            return;
+        }
+
+        AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp; found page has HCPhys=%RHp iPte=%u fIsNested=%RTbool\n",
+                              HCPhys, GCPhys, PGM_PAGE_GET_HCPHYS(pPhysPage), iPte, PGMPOOL_PAGE_IS_NESTED(pPage)));
+    }
+    AssertFatalMsgFailed(("HCPhys=%RHp GCPhys=%RGp\n", HCPhys, GCPhys));
+}
+
+
+/**
+ * Clear references to guest physical memory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   HCPhys      The host physical address corresponding to the guest page.
+ * @param   GCPhysHint  The guest physical address which may corresponding to HCPhys.
+ * @param   iPte        Shadow pte index
+ */
+void pgmPoolTracDerefGCPhysHint(PPGMPOOL pPool, PPGMPOOLPAGE pPage, RTHCPHYS HCPhys, RTGCPHYS GCPhysHint, uint16_t iPte)
+{
+    Log4(("pgmPoolTracDerefGCPhysHint %RHp %RGp\n", HCPhys, GCPhysHint));
+
+    /*
+     * Try the hint first.
+     */
+    RTHCPHYS HCPhysHinted;
+    PVMCC    pVM       = pPool->CTX_SUFF(pVM);
+    PPGMPAGE pPhysPage = pgmPhysGetPage(pVM, GCPhysHint);
+    if (pPhysPage)
+    {
+        HCPhysHinted = PGM_PAGE_GET_HCPHYS(pPhysPage);
+        Assert(HCPhysHinted);
+        if (HCPhysHinted == HCPhys)
+        {
+            Assert(pPage->cPresent);
+            Assert(pPool->cPresent);
+            pPage->cPresent--;
+            pPool->cPresent--;
+            pgmTrackDerefGCPhys(pPool, pPage, pPhysPage, iPte);
+            return;
+        }
+    }
+    else
+        HCPhysHinted = UINT64_C(0xdeadbeefdeadbeef);
+
+    /*
+     * Damn, the hint didn't work.  We'll have to do an expensive linear search.
+     */
+    STAM_COUNTER_INC(&pPool->StatTrackLinearRamSearches);
+    PPGMRAMRANGE pRam = pPool->CTX_SUFF(pVM)->pgm.s.CTX_SUFF(pRamRangesX);
+    while (pRam)
+    {
+        unsigned iPage = pRam->cb >> PAGE_SHIFT;
+        while (iPage-- > 0)
+        {
+            if (PGM_PAGE_GET_HCPHYS(&pRam->aPages[iPage]) == HCPhys)
+            {
+                Log4(("pgmPoolTracDerefGCPhysHint: Linear HCPhys=%RHp GCPhysHint=%RGp GCPhysReal=%RGp\n",
+                      HCPhys, GCPhysHint, pRam->GCPhys + (iPage << PAGE_SHIFT)));
+                Assert(pPage->cPresent);
+                Assert(pPool->cPresent);
+                pPage->cPresent--;
+                pPool->cPresent--;
+                pgmTrackDerefGCPhys(pPool, pPage, &pRam->aPages[iPage], iPte);
+                return;
+            }
+        }
+        pRam = pRam->CTX_SUFF(pNext);
+    }
+
+    AssertFatalMsgFailed(("HCPhys=%RHp GCPhysHint=%RGp (Hinted page has HCPhys = %RHp)\n", HCPhys, GCPhysHint, HCPhysHinted));
+}
+
+
+/**
+ * Clear references to guest physical memory in a 32-bit / 32-bit page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ * @param   pGstPT      The guest page table.
+ */
+DECLINLINE(void) pgmPoolTrackDerefPT32Bit32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT, PCX86PT pGstPT)
+{
+    RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        const X86PGUINT uPte = pShwPT->a[i].u;
+        Assert(!(uPte & RT_BIT_32(10)));
+        if (uPte & X86_PTE_P)
+        {
+            Log4(("pgmPoolTrackDerefPT32Bit32Bit: i=%d pte=%RX32 hint=%RX32\n",
+                  i, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & X86_PTE_PG_MASK));
+            pgmPoolTracDerefGCPhysHint(pPool, pPage, uPte & X86_PTE_PG_MASK, pGstPT->a[i].u & fPgMask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to guest physical memory in a PAE / 32-bit page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ * @param   pGstPT      The guest page table (just a half one).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPTPae32Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PT pGstPT)
+{
+    RTGCPHYS32 const fPgMask = pPage->fA20Enabled ? X86_PTE_PG_MASK : X86_PTE_PG_MASK & ~RT_BIT_32(20);
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        Assert(   (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
+               || (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            Log4(("pgmPoolTrackDerefPTPae32Bit: i=%d pte=%RX64 hint=%RX32\n",
+                  i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PG_MASK));
+            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to guest physical memory in a PAE / PAE page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ * @param   pGstPT      The guest page table.
+ */
+DECLINLINE(void) pgmPoolTrackDerefPTPaePae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT, PCX86PTPAE pGstPT)
+{
+    RTGCPHYS const fPgMask = pPage->fA20Enabled ? X86_PTE_PAE_PG_MASK : X86_PTE_PAE_PG_MASK & ~RT_BIT_64(20);
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        Assert(   (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
+               || (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            Log4(("pgmPoolTrackDerefPTPaePae: i=%d pte=%RX32 hint=%RX32\n",
+                  i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & X86_PTE_PAE_PG_MASK));
+            pgmPoolTracDerefGCPhysHint(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), pGstPT->a[i].u & fPgMask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to guest physical memory in a 32-bit / 4MB page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPT32Bit4MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PT pShwPT)
+{
+    RTGCPHYS const  GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
+    RTGCPHYS        GCPhys        = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
+    {
+        const X86PGUINT uPte = pShwPT->a[i].u;
+        Assert(!(uPte & RT_BIT_32(10)));
+        if (uPte & X86_PTE_P)
+        {
+            Log4(("pgmPoolTrackDerefPT32Bit4MB: i=%d pte=%RX32 GCPhys=%RGp\n",
+                  i, uPte & X86_PTE_PG_MASK, GCPhys));
+            pgmPoolTracDerefGCPhys(pPool, pPage, uPte & X86_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to guest physical memory in a PAE / 2/4MB page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPTPaeBig(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PPGMSHWPTPAE pShwPT)
+{
+    RTGCPHYS const  GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
+    RTGCPHYS        GCPhys        = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
+    {
+        Assert(   (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == 0
+               || (PGMSHWPTEPAE_GET_U(pShwPT->a[i]) & UINT64_C(0x7ff0000000000400)) == UINT64_C(0x7ff0000000000000));
+        if (PGMSHWPTEPAE_IS_P(pShwPT->a[i]))
+        {
+            Log4(("pgmPoolTrackDerefPTPaeBig: i=%d pte=%RX64 hint=%RGp\n",
+                  i, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys));
+            pgmPoolTracDerefGCPhys(pPool, pPage, PGMSHWPTEPAE_GET_HCPHYS(pShwPT->a[i]), GCPhys & GCPhysA20Mask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in an EPT page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPT      The shadow page directory pointer table (mapping of the
+ *                      page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
+{
+    RTGCPHYS const  GCPhysA20Mask = pPage->fA20Enabled ? UINT64_MAX : ~RT_BIT_64(20);
+    RTGCPHYS        GCPhys        = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
+    {
+        X86PGPAEUINT const uPte = pShwPT->a[i].u;
+        Assert((uPte & UINT64_C(0xfff0000000000f80)) == 0);
+        if (uPte & EPT_E_READ)
+        {
+            Log4(("pgmPoolTrackDerefPTEPT: i=%d pte=%RX64 GCPhys=%RX64\n",
+                  i, uPte & EPT_PTE_PG_MASK, pPage->GCPhys));
+            pgmPoolTracDerefGCPhys(pPool, pPage, uPte & EPT_PTE_PG_MASK, GCPhys & GCPhysA20Mask, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+
+/**
+ * Clears references to shadowed pages in a SLAT EPT page table.
+ *
+ * @param   pPool   The pool.
+ * @param   pPage   The page.
+ * @param   pShwPT  The shadow page table (mapping of the page).
+ * @param   pGstPT  The guest page table.
+ */
+DECLINLINE(void) pgmPoolTrackDerefNestedPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT, PCEPTPT pGstPT)
+{
+    Assert(PGMPOOL_PAGE_IS_NESTED(pPage));
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++)
+    {
+        X86PGPAEUINT const uShwPte = pShwPT->a[i].u;
+        Assert((uShwPte & UINT64_C(0xfff0000000000f80)) == 0); /* Access, Dirty, UserX (not supported) and ignored bits 7, 11. */
+        if (uShwPte & EPT_PRESENT_MASK)
+        {
+            Log7Func(("Shw=%RX64 GstPte=%RX64\n", uShwPte, pGstPT->a[i].u));
+            pgmPoolTracDerefGCPhys(pPool, pPage, uShwPte & EPT_PTE_PG_MASK, pGstPT->a[i].u & EPT_PTE_PG_MASK, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to guest physical memory in a SLAT 2MB EPT page table.
+ *
+ * @param   pPool   The pool.
+ * @param   pPage   The page.
+ * @param   pShwPT  The shadow page table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefNestedPTEPT2MB(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPT pShwPT)
+{
+    Assert(pPage->fA20Enabled);
+    RTGCPHYS GCPhys = pPage->GCPhys + PAGE_SIZE * pPage->iFirstPresent;
+    for (unsigned i = pPage->iFirstPresent; i < RT_ELEMENTS(pShwPT->a); i++, GCPhys += PAGE_SIZE)
+    {
+        X86PGPAEUINT const uShwPte = pShwPT->a[i].u;
+        Assert((uShwPte & UINT64_C(0xfff0000000000f80)) == 0); /* Access, Dirty, UserX (not supported) and ignored bits 7, 11. */
+        if (uShwPte & EPT_PRESENT_MASK)
+        {
+            Log7Func(("Shw=%RX64 GstPte=%RX64\n", uShwPte, GCPhys));
+            pgmPoolTracDerefGCPhys(pPool, pPage, uShwPte & EPT_PTE_PG_MASK, GCPhys, i);
+            if (!pPage->cPresent)
+                break;
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in a SLAT EPT page directory.
+ *
+ * @param   pPool   The pool.
+ * @param   pPage   The page.
+ * @param   pShwPD  The shadow page directory (mapping of the page).
+ * @param   pGstPD  The guest page directory.
+ */
+DECLINLINE(void) pgmPoolTrackDerefNestedPDEpt(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD, PCEPTPD pGstPD)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+    {
+        X86PGPAEUINT const uPde = pShwPD->a[i].u;
+#ifdef PGM_WITH_LARGE_PAGES
+        AssertMsg((uPde & UINT64_C(0xfff0000000000f00)) == 0, ("uPde=%RX64\n", uPde));
+#else
+        AssertMsg((uPde & UINT64_C(0xfff0000000000f80)) == 0, ("uPde=%RX64\n", uPde));
+#endif
+        if (uPde & EPT_PRESENT_MASK)
+        {
+#ifdef PGM_WITH_LARGE_PAGES
+            if (uPde & EPT_E_LEAF)
+            {
+                Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n", i, uPde & EPT_PDE2M_PG_MASK, pPage->GCPhys));
+                pgmPoolTracDerefGCPhys(pPool, pPage, uPde & EPT_PDE2M_PG_MASK, pGstPD->a[i].u & EPT_PDE2M_PG_MASK, i);
+            }
+            else
+#endif
+            {
+                PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & EPT_PDE_PG_MASK);
+                if (pSubPage)
+                    pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+                else
+                    AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
+            }
+        }
+    }
+}
+
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+
+/**
+ * Clear references to shadowed pages in a 32 bits page directory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPD      The shadow page directory (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPD(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PD pShwPD)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+    {
+        X86PGUINT const uPde = pShwPD->a[i].u;
+        if (uPde & X86_PDE_P)
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, pShwPD->a[i].u & X86_PDE_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%x\n", pShwPD->a[i].u & X86_PDE_PG_MASK));
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in a PAE (legacy or 64 bits) page directory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPD      The shadow page directory (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPDPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPAE pShwPD)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+    {
+        X86PGPAEUINT const uPde = pShwPD->a[i].u;
+        if (uPde & X86_PDE_P)
+        {
+#ifdef PGM_WITH_LARGE_PAGES
+            if (uPde & X86_PDE_PS)
+            {
+                Log4(("pgmPoolTrackDerefPDPae: i=%d pde=%RX64 GCPhys=%RX64\n",
+                      i, uPde & X86_PDE2M_PAE_PG_MASK, pPage->GCPhys));
+                pgmPoolTracDerefGCPhys(pPool, pPage, uPde & X86_PDE2M_PAE_PG_MASK,
+                                       pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
+                                       i);
+            }
+            else
+#endif
+            {
+                Assert((uPde & (X86_PDE_PAE_MBZ_MASK_NX | UINT64_C(0x7ff0000000000000))) == 0);
+                PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & X86_PDE_PAE_PG_MASK);
+                if (pSubPage)
+                    pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+                else
+                    AssertFatalMsgFailed(("%RX64\n", uPde & X86_PDE_PAE_PG_MASK));
+                /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
+            }
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in a PAE page directory pointer table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPDPT   The shadow page directory pointer table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPDPTPae(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
+{
+    for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
+    {
+        X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
+        Assert((uPdpe & (X86_PDPE_PAE_MBZ_MASK | UINT64_C(0x7ff0000000000200))) == 0);
+        if (uPdpe & X86_PDPE_P)
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK));
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in a 64-bit page directory pointer table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPDPT   The shadow page directory pointer table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPDPT64Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PDPT pShwPDPT)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
+    {
+        X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
+        Assert((uPdpe & (X86_PDPE_LM_MBZ_MASK_NX | UINT64_C(0x7ff0000000000200))) == 0);
+        if (uPdpe & X86_PDPE_P)
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & X86_PDPE_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%RX64\n", uPdpe & X86_PDPE_PG_MASK));
+            /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in a 64-bit level 4 page table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPML4    The shadow page directory pointer table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPML464Bit(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PX86PML4 pShwPML4)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPML4->a); i++)
+    {
+        X86PGPAEUINT const uPml4e = pShwPML4->a[i].u;
+        Assert((uPml4e & (X86_PML4E_MBZ_MASK_NX | UINT64_C(0x7ff0000000000200))) == 0);
+        if (uPml4e & X86_PML4E_P)
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPml4e & X86_PDPE_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%RX64\n", uPml4e & X86_PML4E_PG_MASK));
+            /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in an EPT page directory.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPD      The shadow page directory (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPDEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPD pShwPD)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPD->a); i++)
+    {
+        X86PGPAEUINT const uPde = pShwPD->a[i].u;
+#ifdef PGM_WITH_LARGE_PAGES
+        AssertMsg((uPde & UINT64_C(0xfff0000000000f00)) == 0, ("uPde=%RX64\n", uPde));
+#else
+        AssertMsg((uPde & UINT64_C(0xfff0000000000f80)) == 0, ("uPde=%RX64\n", uPde));
+#endif
+        if (uPde & EPT_E_READ)
+        {
+#ifdef PGM_WITH_LARGE_PAGES
+            if (uPde & EPT_E_LEAF)
+            {
+                Log4(("pgmPoolTrackDerefPDEPT: i=%d pde=%RX64 GCPhys=%RX64\n",
+                      i, uPde & EPT_PDE2M_PG_MASK, pPage->GCPhys));
+                pgmPoolTracDerefGCPhys(pPool, pPage, uPde & EPT_PDE2M_PG_MASK,
+                                       pPage->GCPhys + i * 2 * _1M /* pPage->GCPhys = base address of the memory described by the PD */,
+                                       i);
+            }
+            else
+#endif
+            {
+                PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPde & EPT_PDE_PG_MASK);
+                if (pSubPage)
+                    pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+                else
+                    AssertFatalMsgFailed(("%RX64\n", pShwPD->a[i].u & EPT_PDE_PG_MASK));
+            }
+            /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
+        }
+    }
+}
+
+
+/**
+ * Clear references to shadowed pages in an EPT page directory pointer table.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ * @param   pShwPDPT   The shadow page directory pointer table (mapping of the page).
+ */
+DECLINLINE(void) pgmPoolTrackDerefPDPTEPT(PPGMPOOL pPool, PPGMPOOLPAGE pPage, PEPTPDPT pShwPDPT)
+{
+    for (unsigned i = 0; i < RT_ELEMENTS(pShwPDPT->a); i++)
+    {
+        X86PGPAEUINT const uPdpe = pShwPDPT->a[i].u;
+        Assert((uPdpe & UINT64_C(0xfff0000000000f80)) == 0);
+        if (uPdpe & EPT_E_READ)
+        {
+            PPGMPOOLPAGE pSubPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, uPdpe & EPT_PDPTE_PG_MASK);
+            if (pSubPage)
+                pgmPoolTrackFreeUser(pPool, pSubPage, pPage->idx, i);
+            else
+                AssertFatalMsgFailed(("%RX64\n", uPdpe & EPT_PDPTE_PG_MASK));
+            /** @todo 64-bit guests: have to ensure that we're not exhausting the dynamic mappings! */
+        }
+    }
+}
+
+
+/**
+ * Clears all references made by this page.
+ *
+ * This includes other shadow pages and GC physical addresses.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The page.
+ */
+static void pgmPoolTrackDeref(PPGMPOOL pPool, PPGMPOOLPAGE pPage)
+{
+    /*
+     * Map the shadow page and take action according to the page kind.
+     */
+    PVMCC pVM   = pPool->CTX_SUFF(pVM);
+    void *pvShw = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+    switch (pPage->enmKind)
+    {
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            void *pvGst;
+            int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+            pgmPoolTrackDerefPT32Bit32Bit(pPool, pPage, (PX86PT)pvShw, (PCX86PT)pvGst);
+            PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            void *pvGst;
+            int rc = PGM_GCPHYS_2_PTR_EX(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+            pgmPoolTrackDerefPTPae32Bit(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PT)pvGst);
+            PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            void *pvGst;
+            int rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+            pgmPoolTrackDerefPTPaePae(pPool, pPage, (PPGMSHWPTPAE)pvShw, (PCX86PTPAE)pvGst);
+            PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvGst);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS: /* treat it like a 4 MB page */
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            pgmPoolTrackDerefPT32Bit4MB(pPool, pPage, (PX86PT)pvShw);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:   /* treat it like a 2 MB page */
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+        {
+            STAM_PROFILE_START(&pPool->StatTrackDerefGCPhys, g);
+            pgmPoolTrackDerefPTPaeBig(pPool, pPage, (PPGMSHWPTPAE)pvShw);
+            STAM_PROFILE_STOP(&pPool->StatTrackDerefGCPhys, g);
+            break;
+        }
+
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+        case PGMPOOLKIND_PAE_PD_PHYS:
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+            pgmPoolTrackDerefPDPae(pPool, pPage, (PX86PDPAE)pvShw);
+            break;
+
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+        case PGMPOOLKIND_32BIT_PD:
+            pgmPoolTrackDerefPD(pPool, pPage, (PX86PD)pvShw);
+            break;
+
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+        case PGMPOOLKIND_PAE_PDPT:
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+            pgmPoolTrackDerefPDPTPae(pPool, pPage, (PX86PDPT)pvShw);
+            break;
+
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+            pgmPoolTrackDerefPDPT64Bit(pPool, pPage, (PX86PDPT)pvShw);
+            break;
+
+        case PGMPOOLKIND_64BIT_PML4:
+            pgmPoolTrackDerefPML464Bit(pPool, pPage, (PX86PML4)pvShw);
+            break;
+
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+            pgmPoolTrackDerefPTEPT(pPool, pPage, (PEPTPT)pvShw);
+            break;
+
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+            pgmPoolTrackDerefPDEPT(pPool, pPage, (PEPTPD)pvShw);
+            break;
+
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+            pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
+            break;
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+        {
+            void *pvGst;
+            int const rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+            pgmPoolTrackDerefNestedPTEPT(pPool, pPage, (PEPTPT)pvShw, (PCEPTPT)pvGst);
+            break;
+        }
+
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
+            pgmPoolTrackDerefNestedPTEPT2MB(pPool, pPage, (PEPTPT)pvShw);
+            break;
+
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+        {
+            void *pvGst;
+            int const rc = PGM_GCPHYS_2_PTR(pVM, pPage->GCPhys, &pvGst); AssertReleaseRC(rc);
+            pgmPoolTrackDerefNestedPDEpt(pPool, pPage, (PEPTPD)pvShw, (PCEPTPD)pvGst);
+            break;
+        }
+
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            pgmPoolTrackDerefPDPTEPT(pPool, pPage, (PEPTPDPT)pvShw);
+            break;
+#endif
+
+        default:
+            AssertFatalMsgFailed(("enmKind=%d GCPhys=%RGp\n", pPage->enmKind, pPage->GCPhys));
+    }
+
+    /* paranoia, clear the shadow page. Remove this laser (i.e. let Alloc and ClearAll do it). */
+    STAM_PROFILE_START(&pPool->StatZeroPage, z);
+    ASMMemZeroPage(pvShw);
+    STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
+    pPage->fZeroed = true;
+    Assert(!pPage->cPresent);
+    PGM_DYNMAP_UNUSED_HINT_VM(pVM, pvShw);
+}
+
+
+/**
+ * Flushes a pool page.
+ *
+ * This moves the page to the free list after removing all user references to it.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ * @param   pPool       The pool.
+ * @param   pPage       The shadow page.
+ * @param   fFlush      Flush the TLBS when required (should only be false in very specific use cases!!)
+ */
+int pgmPoolFlushPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, bool fFlush)
+{
+    PVMCC   pVM = pPool->CTX_SUFF(pVM);
+    bool    fFlushRequired = false;
+
+    int rc = VINF_SUCCESS;
+    STAM_PROFILE_START(&pPool->StatFlushPage, f);
+    LogFlow(("pgmPoolFlushPage: pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
+             pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
+
+    if (PGMPOOL_PAGE_IS_NESTED(pPage))
+        Log7Func(("pPage=%p:{.Key=%RHp, .idx=%d, .enmKind=%s, .GCPhys=%RGp}\n",
+                  pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), pPage->GCPhys));
+
+    /*
+     * Reject any attempts at flushing any of the special root pages (shall
+     * not happen).
+     */
+    AssertMsgReturn(pPage->idx >= PGMPOOL_IDX_FIRST,
+                    ("pgmPoolFlushPage: special root page, rejected. enmKind=%s idx=%d\n",
+                     pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx),
+                    VINF_SUCCESS);
+
+    PGM_LOCK_VOID(pVM);
+
+    /*
+     * Quietly reject any attempts at flushing the currently active shadow CR3 mapping
+     */
+    if (pgmPoolIsPageLocked(pPage))
+    {
+        AssertMsg(   pPage->enmKind == PGMPOOLKIND_64BIT_PML4
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PDPT
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PDPT_FOR_32BIT
+                  || pPage->enmKind == PGMPOOLKIND_32BIT_PD
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PD_FOR_PAE_PD
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD
+                  || pPage->enmKind == PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD
+                  || pPage->enmKind == PGMPOOLKIND_ROOT_NESTED,
+                  ("Can't free the shadow CR3! (%RHp vs %RHp kind=%d\n", PGMGetHyperCR3(VMMGetCpu(pVM)), pPage->Core.Key, pPage->enmKind));
+        Log(("pgmPoolFlushPage: current active shadow CR3, rejected. enmKind=%s idx=%d\n", pgmPoolPoolKindToStr(pPage->enmKind), pPage->idx));
+        PGM_UNLOCK(pVM);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Mark the page as being in need of an ASMMemZeroPage().
+     */
+    pPage->fZeroed = false;
+
+#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    if (pPage->fDirty)
+        pgmPoolFlushDirtyPage(pVM, pPool, pPage->idxDirtyEntry, false /* do not remove */);
+#endif
+
+    /* If there are any users of this table, then we *must* issue a tlb flush on all VCPUs. */
+    if (pPage->iUserHead != NIL_PGMPOOL_USER_INDEX)
+        fFlushRequired = true;
+
+    /*
+     * Clear the page.
+     */
+    pgmPoolTrackClearPageUsers(pPool, pPage);
+    STAM_PROFILE_START(&pPool->StatTrackDeref,a);
+    pgmPoolTrackDeref(pPool, pPage);
+    STAM_PROFILE_STOP(&pPool->StatTrackDeref,a);
+
+    /*
+     * Flush it from the cache.
+     */
+    pgmPoolCacheFlushPage(pPool, pPage);
+
+    /*
+     * Deregistering the monitoring.
+     */
+    if (pPage->fMonitored)
+        rc = pgmPoolMonitorFlush(pPool, pPage);
+
+    /*
+     * Free the page.
+     */
+    Assert(pPage->iNext == NIL_PGMPOOL_IDX);
+    pPage->iNext = pPool->iFreeHead;
+    pPool->iFreeHead = pPage->idx;
+    pPage->enmKind = PGMPOOLKIND_FREE;
+    pPage->enmAccess = PGMPOOLACCESS_DONTCARE;
+    pPage->GCPhys = NIL_RTGCPHYS;
+    pPage->fReusedFlushPending = false;
+
+    pPool->cUsedPages--;
+
+    /* Flush the TLBs of all VCPUs if required. */
+    if (    fFlushRequired
+        &&  fFlush)
+    {
+        PGM_INVL_ALL_VCPU_TLBS(pVM);
+    }
+
+    PGM_UNLOCK(pVM);
+    STAM_PROFILE_STOP(&pPool->StatFlushPage, f);
+    return rc;
+}
+
+
+/**
+ * Frees a usage of a pool page.
+ *
+ * The caller is responsible to updating the user table so that it no longer
+ * references the shadow page.
+ *
+ * @param   pPool       The pool.
+ * @param   pPage       The shadow page.
+ * @param   iUser       The shadow page pool index of the user table.
+ *                      NIL_PGMPOOL_IDX for root pages.
+ * @param   iUserTable  The index into the user table (shadowed). Ignored if
+ *                      root page.
+ */
+void pgmPoolFreeByPage(PPGMPOOL pPool, PPGMPOOLPAGE pPage, uint16_t iUser, uint32_t iUserTable)
+{
+    PVMCC pVM = pPool->CTX_SUFF(pVM);
+
+    STAM_PROFILE_START(&pPool->StatFree, a);
+    LogFlow(("pgmPoolFreeByPage: pPage=%p:{.Key=%RHp, .idx=%d, enmKind=%s} iUser=%d iUserTable=%#x\n",
+             pPage, pPage->Core.Key, pPage->idx, pgmPoolPoolKindToStr(pPage->enmKind), iUser, iUserTable));
+    AssertReturnVoid(pPage->idx >= PGMPOOL_IDX_FIRST); /* paranoia (#6349) */
+
+    PGM_LOCK_VOID(pVM);
+    if (iUser != NIL_PGMPOOL_IDX)
+        pgmPoolTrackFreeUser(pPool, pPage, iUser, iUserTable);
+    if (!pPage->fCached)
+        pgmPoolFlushPage(pPool, pPage);
+    PGM_UNLOCK(pVM);
+    STAM_PROFILE_STOP(&pPool->StatFree, a);
+}
+
+
+/**
+ * Makes one or more free page free.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS on success.
+ *
+ * @param   pPool       The pool.
+ * @param   enmKind     Page table kind
+ * @param   iUser       The user of the page.
+ */
+static int pgmPoolMakeMoreFreePages(PPGMPOOL pPool, PGMPOOLKIND enmKind, uint16_t iUser)
+{
+    PVMCC pVM = pPool->CTX_SUFF(pVM);
+    LogFlow(("pgmPoolMakeMoreFreePages: enmKind=%d iUser=%d\n", enmKind, iUser));
+    NOREF(enmKind);
+
+    /*
+     * If the pool isn't full grown yet, expand it.
+     */
+    if (pPool->cCurPages < pPool->cMaxPages)
+    {
+        STAM_PROFILE_ADV_SUSPEND(&pPool->StatAlloc, a);
+#ifdef IN_RING3
+        int rc = PGMR3PoolGrow(pVM, VMMGetCpu(pVM));
+#else
+        int rc = PGMR0PoolGrow(pVM, VMMGetCpuId(pVM));
+#endif
+        if (RT_FAILURE(rc))
+            return rc;
+        STAM_PROFILE_ADV_RESUME(&pPool->StatAlloc, a);
+        if (pPool->iFreeHead != NIL_PGMPOOL_IDX)
+            return VINF_SUCCESS;
+    }
+
+    /*
+     * Free one cached page.
+     */
+    return pgmPoolCacheFreeOne(pPool, iUser);
+}
+
+
+/**
+ * Allocates a page from the pool.
+ *
+ * This page may actually be a cached page and not in need of any processing
+ * on the callers part.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if a NEW page was allocated.
+ * @retval  VINF_PGM_CACHED_PAGE if a CACHED page was returned.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   GCPhys      The GC physical address of the page we're gonna shadow.
+ *                      For 4MB and 2MB PD entries, it's the first address the
+ *                      shadow PT is covering.
+ * @param   enmKind     The kind of mapping.
+ * @param   enmAccess   Access type for the mapping (only relevant for big pages)
+ * @param   fA20Enabled Whether the A20 gate is enabled or not.
+ * @param   iUser       The shadow page pool index of the user table.  Root
+ *                      pages should pass NIL_PGMPOOL_IDX.
+ * @param   iUserTable  The index into the user table (shadowed).  Ignored for
+ *                      root pages (iUser == NIL_PGMPOOL_IDX).
+ * @param   fLockPage   Lock the page
+ * @param   ppPage      Where to store the pointer to the page. NULL is stored here on failure.
+ */
+int pgmPoolAlloc(PVMCC pVM, RTGCPHYS GCPhys, PGMPOOLKIND enmKind, PGMPOOLACCESS enmAccess, bool fA20Enabled,
+                 uint16_t iUser, uint32_t iUserTable, bool fLockPage, PPPGMPOOLPAGE ppPage)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    STAM_PROFILE_ADV_START(&pPool->StatAlloc, a);
+    LogFlow(("pgmPoolAlloc: GCPhys=%RGp enmKind=%s iUser=%d iUserTable=%#x\n", GCPhys, pgmPoolPoolKindToStr(enmKind), iUser, iUserTable));
+    *ppPage = NULL;
+    /** @todo CSAM/PGMPrefetchPage messes up here during CSAMR3CheckGates
+     *  (TRPMR3SyncIDT) because of FF priority. Try fix that?
+     *  Assert(!(pVM->pgm.s.fGlobalSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)); */
+
+#if defined(VBOX_STRICT) && defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT)
+    PVMCPUCC pVCpu = VMMGetCpu(pVM);
+    Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT || PGMPOOL_PAGE_IS_KIND_NESTED(enmKind));
+#endif
+
+    PGM_LOCK_VOID(pVM);
+
+    if (pPool->fCacheEnabled)
+    {
+        int rc2 = pgmPoolCacheAlloc(pPool, GCPhys, enmKind, enmAccess, fA20Enabled, iUser, iUserTable, ppPage);
+        if (RT_SUCCESS(rc2))
+        {
+            if (fLockPage)
+                pgmPoolLockPage(pPool, *ppPage);
+            PGM_UNLOCK(pVM);
+            STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
+            LogFlow(("pgmPoolAlloc: cached returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d}\n", rc2, *ppPage, (*ppPage)->Core.Key, (*ppPage)->idx));
+            return rc2;
+        }
+    }
+
+    /*
+     * Allocate a new one.
+     */
+    int         rc = VINF_SUCCESS;
+    uint16_t    iNew = pPool->iFreeHead;
+    if (iNew == NIL_PGMPOOL_IDX)
+    {
+        rc = pgmPoolMakeMoreFreePages(pPool, enmKind, iUser);
+        if (RT_FAILURE(rc))
+        {
+            PGM_UNLOCK(pVM);
+            Log(("pgmPoolAlloc: returns %Rrc (Free)\n", rc));
+            STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
+            return rc;
+        }
+        iNew = pPool->iFreeHead;
+        AssertReleaseMsgReturn(iNew != NIL_PGMPOOL_IDX, ("iNew=%#x\n", iNew), VERR_PGM_POOL_IPE);
+    }
+
+    /* unlink the free head */
+    PPGMPOOLPAGE pPage = &pPool->aPages[iNew];
+    pPool->iFreeHead = pPage->iNext;
+    pPage->iNext = NIL_PGMPOOL_IDX;
+
+    /*
+     * Initialize it.
+     */
+    pPool->cUsedPages++;                /* physical handler registration / pgmPoolTrackFlushGCPhysPTsSlow requirement. */
+    pPage->enmKind = enmKind;
+    pPage->enmAccess = enmAccess;
+    pPage->GCPhys = GCPhys;
+    pPage->fA20Enabled = fA20Enabled;
+    pPage->fSeenNonGlobal = false;      /* Set this to 'true' to disable this feature. */
+    pPage->fMonitored = false;
+    pPage->fCached = false;
+    pPage->fDirty = false;
+    pPage->fReusedFlushPending = false;
+    pPage->cModifications = 0;
+    pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+    pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+    pPage->cPresent = 0;
+    pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+    pPage->idxDirtyEntry = 0;
+    pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
+    pPage->GCPtrLastAccessHandlerRip   = NIL_RTGCPTR;
+    pPage->cLastAccessHandler = 0;
+    pPage->cLocked = 0;
+# ifdef VBOX_STRICT
+    pPage->GCPtrDirtyFault = NIL_RTGCPTR;
+# endif
+
+    /*
+     * Insert into the tracking and cache. If this fails, free the page.
+     */
+    int rc3 = pgmPoolTrackInsert(pPool, pPage, GCPhys, iUser, iUserTable);
+    if (RT_FAILURE(rc3))
+    {
+        pPool->cUsedPages--;
+        pPage->enmKind      = PGMPOOLKIND_FREE;
+        pPage->enmAccess    = PGMPOOLACCESS_DONTCARE;
+        pPage->GCPhys       = NIL_RTGCPHYS;
+        pPage->iNext        = pPool->iFreeHead;
+        pPool->iFreeHead    = pPage->idx;
+        PGM_UNLOCK(pVM);
+        STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
+        Log(("pgmPoolAlloc: returns %Rrc (Insert)\n", rc3));
+        return rc3;
+    }
+
+    /*
+     * Commit the allocation, clear the page and return.
+     */
+#ifdef VBOX_WITH_STATISTICS
+    if (pPool->cUsedPages > pPool->cUsedPagesHigh)
+        pPool->cUsedPagesHigh = pPool->cUsedPages;
+#endif
+
+    if (!pPage->fZeroed)
+    {
+        STAM_PROFILE_START(&pPool->StatZeroPage, z);
+        void *pv = PGMPOOL_PAGE_2_PTR(pVM, pPage);
+        ASMMemZeroPage(pv);
+        STAM_PROFILE_STOP(&pPool->StatZeroPage, z);
+    }
+
+    *ppPage = pPage;
+    if (fLockPage)
+        pgmPoolLockPage(pPool, pPage);
+    PGM_UNLOCK(pVM);
+    LogFlow(("pgmPoolAlloc: returns %Rrc *ppPage=%p:{.Key=%RHp, .idx=%d, .fCached=%RTbool, .fMonitored=%RTbool}\n",
+             rc, pPage, pPage->Core.Key, pPage->idx, pPage->fCached, pPage->fMonitored));
+    STAM_PROFILE_ADV_STOP(&pPool->StatAlloc, a);
+    return rc;
+}
+
+
+/**
+ * Frees a usage of a pool page.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   HCPhys      The HC physical address of the shadow page.
+ * @param   iUser       The shadow page pool index of the user table.
+ *                      NIL_PGMPOOL_IDX if root page.
+ * @param   iUserTable  The index into the user table (shadowed).  Ignored if
+ *                      root page.
+ */
+void pgmPoolFree(PVM pVM, RTHCPHYS HCPhys, uint16_t iUser, uint32_t iUserTable)
+{
+    LogFlow(("pgmPoolFree: HCPhys=%RHp iUser=%d iUserTable=%#x\n", HCPhys, iUser, iUserTable));
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+    pgmPoolFreeByPage(pPool, pgmPoolGetPage(pPool, HCPhys), iUser, iUserTable);
+}
+
+
+/**
+ * Internal worker for finding a 'in-use' shadow page give by it's physical address.
+ *
+ * @returns Pointer to the shadow page structure.
+ * @param   pPool       The pool.
+ * @param   HCPhys      The HC physical address of the shadow page.
+ */
+PPGMPOOLPAGE pgmPoolGetPage(PPGMPOOL pPool, RTHCPHYS HCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
+
+    /*
+     * Look up the page.
+     */
+    PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
+
+    AssertFatalMsg(pPage && pPage->enmKind != PGMPOOLKIND_FREE, ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0));
+    return pPage;
+}
+
+
+/**
+ * Internal worker for finding a page for debugging purposes, no assertions.
+ *
+ * @returns Pointer to the shadow page structure.  NULL on if not found.
+ * @param   pPool       The pool.
+ * @param   HCPhys      The HC physical address of the shadow page.
+ */
+PPGMPOOLPAGE pgmPoolQueryPageForDbg(PPGMPOOL pPool, RTHCPHYS HCPhys)
+{
+    PGM_LOCK_ASSERT_OWNER(pPool->CTX_SUFF(pVM));
+    return (PPGMPOOLPAGE)RTAvloHCPhysGet(&pPool->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
+}
+
+
+/**
+ * Internal worker for PGM_HCPHYS_2_PTR.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   HCPhys      The HC physical address of the shadow page.
+ * @param   ppv         Where to return the address.
+ */
+int pgmPoolHCPhys2Ptr(PVM pVM, RTHCPHYS HCPhys, void **ppv)
+{
+    PPGMPOOLPAGE pPage = (PPGMPOOLPAGE)RTAvloHCPhysGet(&pVM->pgm.s.CTX_SUFF(pPool)->HCPhysTree, HCPhys & X86_PTE_PAE_PG_MASK);
+    AssertMsgReturn(pPage && pPage->enmKind != PGMPOOLKIND_FREE,
+                    ("HCPhys=%RHp pPage=%p idx=%d\n", HCPhys, pPage, (pPage) ? pPage->idx : 0),
+                    VERR_PGM_POOL_GET_PAGE_FAILED);
+    *ppv = (uint8_t *)pPage->CTX_SUFF(pvPage) + (HCPhys & PAGE_OFFSET_MASK);
+    return VINF_SUCCESS;
+}
+
+#ifdef IN_RING3 /* currently only used in ring 3; save some space in the R0 & GC modules (left it here as we might need it elsewhere later on) */
+
+/**
+ * Flush the specified page if present
+ *
+ * @param   pVM     The cross context VM structure.
+ * @param   GCPhys  Guest physical address of the page to flush
+ */
+void pgmPoolFlushPageByGCPhys(PVM pVM, RTGCPHYS GCPhys)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    VM_ASSERT_EMT(pVM);
+
+    /*
+     * Look up the GCPhys in the hash.
+     */
+    GCPhys = GCPhys & ~(RTGCPHYS)PAGE_OFFSET_MASK;
+    unsigned i = pPool->aiHash[PGMPOOL_HASH(GCPhys)];
+    if (i == NIL_PGMPOOL_IDX)
+        return;
+
+    do
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[i];
+        if (pPage->GCPhys - GCPhys < PAGE_SIZE)
+        {
+            Assert(!PGMPOOL_PAGE_IS_NESTED(pPage));  /* Temporary to see if it hits. Remove later. */
+            switch (pPage->enmKind)
+            {
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+                case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+                case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+                case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+                case PGMPOOLKIND_64BIT_PML4:
+                case PGMPOOLKIND_32BIT_PD:
+                case PGMPOOLKIND_PAE_PDPT:
+                {
+                    Log(("PGMPoolFlushPage: found pgm pool pages for %RGp\n", GCPhys));
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+                    if (pPage->fDirty)
+                        STAM_COUNTER_INC(&pPool->StatForceFlushDirtyPage);
+                    else
+# endif
+                        STAM_COUNTER_INC(&pPool->StatForceFlushPage);
+                    Assert(!pgmPoolIsPageLocked(pPage));
+                    pgmPoolMonitorChainFlush(pPool, pPage);
+                    return;
+                }
+
+                /* ignore, no monitoring. */
+                case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+                case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+                case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+                case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+                case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+                case PGMPOOLKIND_ROOT_NESTED:
+                case PGMPOOLKIND_PAE_PD_PHYS:
+                case PGMPOOLKIND_PAE_PDPT_PHYS:
+                case PGMPOOLKIND_32BIT_PD_PHYS:
+                case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+                    break;
+
+                default:
+                    AssertFatalMsgFailed(("enmKind=%d idx=%d\n", pPage->enmKind, pPage->idx));
+            }
+        }
+
+        /* next */
+        i = pPage->iNext;
+    } while (i != NIL_PGMPOOL_IDX);
+    return;
+}
+
+
+/**
+ * Reset CPU on hot plugging.
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu              The cross context virtual CPU structure.
+ */
+void pgmR3PoolResetUnpluggedCpu(PVM pVM, PVMCPU pVCpu)
+{
+    pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
+
+    pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+}
+
+
+/**
+ * Flushes the entire cache.
+ *
+ * It will assert a global CR3 flush (FF) and assumes the caller is aware of
+ * this and execute this CR3 flush.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+void pgmR3PoolReset(PVM pVM)
+{
+    PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+    STAM_PROFILE_START(&pPool->StatR3Reset, a);
+    LogFlow(("pgmR3PoolReset:\n"));
+
+    /*
+     * If there are no pages in the pool, there is nothing to do.
+     */
+    if (pPool->cCurPages <= PGMPOOL_IDX_FIRST)
+    {
+        STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
+        return;
+    }
+
+    /*
+     * Exit the shadow mode since we're going to clear everything,
+     * including the root page.
+     */
+    VMCC_FOR_EACH_VMCPU(pVM)
+        pgmR3ExitShadowModeBeforePoolFlush(pVCpu);
+    VMCC_FOR_EACH_VMCPU_END(pVM);
+
+
+    /*
+     * Nuke the free list and reinsert all pages into it.
+     */
+    for (unsigned i = pPool->cCurPages - 1; i >= PGMPOOL_IDX_FIRST; i--)
+    {
+        PPGMPOOLPAGE pPage = &pPool->aPages[i];
+
+        if (pPage->fMonitored)
+            pgmPoolMonitorFlush(pPool, pPage);
+        pPage->iModifiedNext = NIL_PGMPOOL_IDX;
+        pPage->iModifiedPrev = NIL_PGMPOOL_IDX;
+        pPage->iMonitoredNext = NIL_PGMPOOL_IDX;
+        pPage->iMonitoredPrev = NIL_PGMPOOL_IDX;
+        pPage->GCPhys     = NIL_RTGCPHYS;
+        pPage->enmKind    = PGMPOOLKIND_FREE;
+        pPage->enmAccess  = PGMPOOLACCESS_DONTCARE;
+        Assert(pPage->idx == i);
+        pPage->iNext      = i + 1;
+        pPage->fA20Enabled = true;
+        pPage->fZeroed    = false;       /* This could probably be optimized, but better safe than sorry. */
+        pPage->fSeenNonGlobal = false;
+        pPage->fMonitored = false;
+        pPage->fDirty     = false;
+        pPage->fCached    = false;
+        pPage->fReusedFlushPending = false;
+        pPage->iUserHead  = NIL_PGMPOOL_USER_INDEX;
+        pPage->cPresent = 0;
+        pPage->iFirstPresent = NIL_PGMPOOL_PRESENT_INDEX;
+        pPage->cModifications = 0;
+        pPage->iAgeNext   = NIL_PGMPOOL_IDX;
+        pPage->iAgePrev   = NIL_PGMPOOL_IDX;
+        pPage->idxDirtyEntry = 0;
+        pPage->GCPtrLastAccessHandlerRip = NIL_RTGCPTR;
+        pPage->GCPtrLastAccessHandlerFault = NIL_RTGCPTR;
+        pPage->cLastAccessHandler = 0;
+        pPage->cLocked    = 0;
+# ifdef VBOX_STRICT
+        pPage->GCPtrDirtyFault = NIL_RTGCPTR;
+# endif
+    }
+    pPool->aPages[pPool->cCurPages - 1].iNext = NIL_PGMPOOL_IDX;
+    pPool->iFreeHead = PGMPOOL_IDX_FIRST;
+    pPool->cUsedPages = 0;
+
+    /*
+     * Zap and reinitialize the user records.
+     */
+    pPool->cPresent = 0;
+    pPool->iUserFreeHead = 0;
+    PPGMPOOLUSER paUsers = pPool->CTX_SUFF(paUsers);
+    const unsigned cMaxUsers = pPool->cMaxUsers;
+    for (unsigned i = 0; i < cMaxUsers; i++)
+    {
+        paUsers[i].iNext = i + 1;
+        paUsers[i].iUser = NIL_PGMPOOL_IDX;
+        paUsers[i].iUserTable = 0xfffffffe;
+    }
+    paUsers[cMaxUsers - 1].iNext = NIL_PGMPOOL_USER_INDEX;
+
+    /*
+     * Clear all the GCPhys links and rebuild the phys ext free list.
+     */
+    for (PPGMRAMRANGE pRam = pVM->pgm.s.CTX_SUFF(pRamRangesX);
+         pRam;
+         pRam = pRam->CTX_SUFF(pNext))
+    {
+        unsigned iPage = pRam->cb >> PAGE_SHIFT;
+        while (iPage-- > 0)
+            PGM_PAGE_SET_TRACKING(pVM, &pRam->aPages[iPage], 0);
+    }
+
+    pPool->iPhysExtFreeHead = 0;
+    PPGMPOOLPHYSEXT paPhysExts = pPool->CTX_SUFF(paPhysExts);
+    const unsigned cMaxPhysExts = pPool->cMaxPhysExts;
+    for (unsigned i = 0; i < cMaxPhysExts; i++)
+    {
+        paPhysExts[i].iNext = i + 1;
+        paPhysExts[i].aidx[0] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[0] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[1] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[1] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+        paPhysExts[i].aidx[2] = NIL_PGMPOOL_IDX;
+        paPhysExts[i].apte[2] = NIL_PGMPOOL_PHYSEXT_IDX_PTE;
+    }
+    paPhysExts[cMaxPhysExts - 1].iNext = NIL_PGMPOOL_PHYSEXT_INDEX;
+
+    /*
+     * Just zap the modified list.
+     */
+    pPool->cModifiedPages = 0;
+    pPool->iModifiedHead = NIL_PGMPOOL_IDX;
+
+    /*
+     * Clear the GCPhys hash and the age list.
+     */
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aiHash); i++)
+        pPool->aiHash[i] = NIL_PGMPOOL_IDX;
+    pPool->iAgeHead = NIL_PGMPOOL_IDX;
+    pPool->iAgeTail = NIL_PGMPOOL_IDX;
+
+# ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
+    /* Clear all dirty pages. */
+    pPool->idxFreeDirtyPage = 0;
+    pPool->cDirtyPages      = 0;
+    for (unsigned i = 0; i < RT_ELEMENTS(pPool->aidxDirtyPages); i++)
+        pPool->aidxDirtyPages[i] = NIL_PGMPOOL_IDX;
+# endif
+
+    /*
+     * Reinsert active pages into the hash and ensure monitoring chains are correct.
+     */
+    VMCC_FOR_EACH_VMCPU(pVM)
+    {
+        /*
+         * Re-enter the shadowing mode and assert Sync CR3 FF.
+         */
+        pgmR3ReEnterShadowModeAfterPoolFlush(pVM, pVCpu);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+        VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
+    }
+    VMCC_FOR_EACH_VMCPU_END(pVM);
+
+    STAM_PROFILE_STOP(&pPool->StatR3Reset, a);
+}
+
+#endif /* IN_RING3 */
+
+#if defined(LOG_ENABLED) || defined(VBOX_STRICT)
+/**
+ * Stringifies a PGMPOOLKIND value.
+ */
+static const char *pgmPoolPoolKindToStr(uint8_t enmKind)
+{
+    switch ((PGMPOOLKIND)enmKind)
+    {
+        case PGMPOOLKIND_INVALID:
+            return "PGMPOOLKIND_INVALID";
+        case PGMPOOLKIND_FREE:
+            return "PGMPOOLKIND_FREE";
+        case PGMPOOLKIND_32BIT_PT_FOR_PHYS:
+            return "PGMPOOLKIND_32BIT_PT_FOR_PHYS";
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT:
+            return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT";
+        case PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB:
+            return "PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB";
+        case PGMPOOLKIND_PAE_PT_FOR_PHYS:
+            return "PGMPOOLKIND_PAE_PT_FOR_PHYS";
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_PT:
+            return "PGMPOOLKIND_PAE_PT_FOR_32BIT_PT";
+        case PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB:
+            return "PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB";
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_PT:
+            return "PGMPOOLKIND_PAE_PT_FOR_PAE_PT";
+        case PGMPOOLKIND_PAE_PT_FOR_PAE_2MB:
+            return "PGMPOOLKIND_PAE_PT_FOR_PAE_2MB";
+        case PGMPOOLKIND_32BIT_PD:
+            return "PGMPOOLKIND_32BIT_PD";
+        case PGMPOOLKIND_32BIT_PD_PHYS:
+            return "PGMPOOLKIND_32BIT_PD_PHYS";
+        case PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD:
+            return "PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD";
+        case PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD:
+            return "PGMPOOLKIND_PAE_PD1_FOR_32BIT_PD";
+        case PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD:
+            return "PGMPOOLKIND_PAE_PD2_FOR_32BIT_PD";
+        case PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD:
+            return "PGMPOOLKIND_PAE_PD3_FOR_32BIT_PD";
+        case PGMPOOLKIND_PAE_PD_FOR_PAE_PD:
+            return "PGMPOOLKIND_PAE_PD_FOR_PAE_PD";
+        case PGMPOOLKIND_PAE_PD_PHYS:
+            return "PGMPOOLKIND_PAE_PD_PHYS";
+        case PGMPOOLKIND_PAE_PDPT_FOR_32BIT:
+            return "PGMPOOLKIND_PAE_PDPT_FOR_32BIT";
+        case PGMPOOLKIND_PAE_PDPT:
+            return "PGMPOOLKIND_PAE_PDPT";
+        case PGMPOOLKIND_PAE_PDPT_PHYS:
+            return "PGMPOOLKIND_PAE_PDPT_PHYS";
+        case PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT:
+            return "PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT";
+        case PGMPOOLKIND_64BIT_PDPT_FOR_PHYS:
+            return "PGMPOOLKIND_64BIT_PDPT_FOR_PHYS";
+        case PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD:
+            return "PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD";
+        case PGMPOOLKIND_64BIT_PD_FOR_PHYS:
+            return "PGMPOOLKIND_64BIT_PD_FOR_PHYS";
+        case PGMPOOLKIND_64BIT_PML4:
+            return "PGMPOOLKIND_64BIT_PML4";
+        case PGMPOOLKIND_EPT_PDPT_FOR_PHYS:
+            return "PGMPOOLKIND_EPT_PDPT_FOR_PHYS";
+        case PGMPOOLKIND_EPT_PD_FOR_PHYS:
+            return "PGMPOOLKIND_EPT_PD_FOR_PHYS";
+        case PGMPOOLKIND_EPT_PT_FOR_PHYS:
+            return "PGMPOOLKIND_EPT_PT_FOR_PHYS";
+        case PGMPOOLKIND_ROOT_NESTED:
+            return "PGMPOOLKIND_ROOT_NESTED";
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_PT:
+            return "PGMPOOLKIND_EPT_PT_FOR_EPT_PT";
+        case PGMPOOLKIND_EPT_PT_FOR_EPT_2MB:
+            return "PGMPOOLKIND_EPT_PT_FOR_EPT_2MB";
+        case PGMPOOLKIND_EPT_PD_FOR_EPT_PD:
+            return "PGMPOOLKIND_EPT_PD_FOR_EPT_PD";
+        case PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT:
+            return "PGMPOOLKIND_EPT_PDPT_FOR_EPT_PDPT";
+        case PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4:
+            return "PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4";
+    }
+    return "Unknown kind!";
+}
+#endif /* LOG_ENABLED || VBOX_STRICT */
+
diff --git a/src/VBox/VMM/VMMAll/PGMAllShw.h b/src/VBox/VMM/VMMAll/PGMAllShw.h
new file mode 100644
index 00000000..53c09f80
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/PGMAllShw.h
@@ -0,0 +1,647 @@
+/* $Id: PGMAllShw.h $ */
+/** @file
+ * VBox - Page Manager, Shadow Paging Template - All context code.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#undef SHWUINT
+#undef SHWPT
+#undef PSHWPT
+#undef SHWPTE
+#undef PSHWPTE
+#undef SHWPD
+#undef PSHWPD
+#undef SHWPDE
+#undef PSHWPDE
+#undef SHW_PDE_PG_MASK
+#undef SHW_PD_SHIFT
+#undef SHW_PD_MASK
+#undef SHW_PDE_ATOMIC_SET
+#undef SHW_PDE_ATOMIC_SET2
+#undef SHW_PDE_IS_P
+#undef SHW_PDE_IS_A
+#undef SHW_PDE_IS_BIG
+#undef SHW_PTE_PG_MASK
+#undef SHW_PTE_IS_P
+#undef SHW_PTE_IS_RW
+#undef SHW_PTE_IS_US
+#undef SHW_PTE_IS_A
+#undef SHW_PTE_IS_D
+#undef SHW_PTE_IS_P_RW
+#undef SHW_PTE_IS_TRACK_DIRTY
+#undef SHW_PTE_GET_HCPHYS
+#undef SHW_PTE_GET_U
+#undef SHW_PTE_LOG64
+#undef SHW_PTE_SET
+#undef SHW_PTE_ATOMIC_SET
+#undef SHW_PTE_ATOMIC_SET2
+#undef SHW_PTE_SET_RO
+#undef SHW_PTE_SET_RW
+#undef SHW_PT_SHIFT
+#undef SHW_PT_MASK
+#undef SHW_TOTAL_PD_ENTRIES
+#undef SHW_PDPT_SHIFT
+#undef SHW_PDPT_MASK
+#undef SHW_PDPE_PG_MASK
+
+#if PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_NESTED_32BIT
+# define SHWUINT                        uint32_t
+# define SHWPT                          X86PT
+# define PSHWPT                         PX86PT
+# define SHWPTE                         X86PTE
+# define PSHWPTE                        PX86PTE
+# define SHWPD                          X86PD
+# define PSHWPD                         PX86PD
+# define SHWPDE                         X86PDE
+# define PSHWPDE                        PX86PDE
+# define SHW_PDE_PG_MASK                X86_PDE_PG_MASK
+# define SHW_PD_SHIFT                   X86_PD_SHIFT
+# define SHW_PD_MASK                    X86_PD_MASK
+# define SHW_TOTAL_PD_ENTRIES           X86_PG_ENTRIES
+# define SHW_PDE_IS_P(Pde)              ( (Pde).u & X86_PDE_P )
+# define SHW_PDE_IS_A(Pde)              ( (Pde).u & X86_PDE_A )
+# define SHW_PDE_IS_BIG(Pde)            ( (Pde).u & X86_PDE_PS  )
+# define SHW_PDE_ATOMIC_SET(Pde, uNew)  do { ASMAtomicWriteU32(&(Pde).u, (uNew)); } while (0)
+# define SHW_PDE_ATOMIC_SET2(Pde, Pde2) do { ASMAtomicWriteU32(&(Pde).u, (Pde2).u); } while (0)
+# define SHW_PTE_PG_MASK                X86_PTE_PG_MASK
+# define SHW_PTE_IS_P(Pte)              ( (Pte).u & X86_PTE_P )
+# define SHW_PTE_IS_RW(Pte)             ( (Pte).u & X86_PTE_RW )
+# define SHW_PTE_IS_US(Pte)             ( (Pte).u & X86_PTE_US )
+# define SHW_PTE_IS_A(Pte)              ( (Pte).u & X86_PTE_A )
+# define SHW_PTE_IS_D(Pte)              ( (Pte).u & X86_PTE_D )
+# define SHW_PTE_IS_P_RW(Pte)           ( ((Pte).u & (X86_PTE_P | X86_PTE_RW)) == (X86_PTE_P | X86_PTE_RW) )
+# define SHW_PTE_IS_TRACK_DIRTY(Pte)    ( !!((Pte).u & PGM_PTFLAGS_TRACK_DIRTY) )
+# define SHW_PTE_GET_HCPHYS(Pte)        ( (Pte).u & X86_PTE_PG_MASK )
+# define SHW_PTE_LOG64(Pte)             ( (uint64_t)(Pte).u )
+# define SHW_PTE_GET_U(Pte)             ( (Pte).u )             /**< Use with care. */
+# define SHW_PTE_SET(Pte, uNew)         do { (Pte).u = (uNew); } while (0)
+# define SHW_PTE_ATOMIC_SET(Pte, uNew)  do { ASMAtomicWriteU32(&(Pte).u, (uNew)); } while (0)
+# define SHW_PTE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU32(&(Pte).u, (Pte2).u); } while (0)
+# define SHW_PTE_SET_RO(Pte)            do { (Pte).u &= ~(X86PGUINT)X86_PTE_RW; } while (0)
+# define SHW_PTE_SET_RW(Pte)            do { (Pte).u |= X86_PTE_RW; } while (0)
+# define SHW_PT_SHIFT                   X86_PT_SHIFT
+# define SHW_PT_MASK                    X86_PT_MASK
+
+#elif PGM_SHW_TYPE == PGM_TYPE_EPT
+# define SHWUINT                        uint64_t
+# define SHWPT                          EPTPT
+# define PSHWPT                         PEPTPT
+# define SHWPTE                         EPTPTE
+# define PSHWPTE                        PEPTPTE
+# define SHWPD                          EPTPD
+# define PSHWPD                         PEPTPD
+# define SHWPDE                         EPTPDE
+# define PSHWPDE                        PEPTPDE
+# define SHW_PDE_PG_MASK                EPT_PDE_PG_MASK
+# define SHW_PD_SHIFT                   EPT_PD_SHIFT
+# define SHW_PD_MASK                    EPT_PD_MASK
+# define SHW_PDE_IS_P(Pde)              ( (Pde).u & EPT_E_READ /* always set*/ )
+# define SHW_PDE_IS_A(Pde)              ( 1 ) /* We don't use EPT_E_ACCESSED, use with care! */
+# define SHW_PDE_IS_BIG(Pde)            ( (Pde).u & EPT_E_LEAF )
+# define SHW_PDE_ATOMIC_SET(Pde, uNew)  do { ASMAtomicWriteU64(&(Pde).u, (uNew)); } while (0)
+# define SHW_PDE_ATOMIC_SET2(Pde, Pde2) do { ASMAtomicWriteU64(&(Pde).u, (Pde2).u); } while (0)
+# define SHW_PTE_PG_MASK                EPT_PTE_PG_MASK
+# define SHW_PTE_IS_P(Pte)              ( (Pte).u & EPT_E_READ ) /* Approximation, works for us. */
+# define SHW_PTE_IS_RW(Pte)             ( (Pte).u & EPT_E_WRITE )
+# define SHW_PTE_IS_US(Pte)             ( true )
+# define SHW_PTE_IS_A(Pte)              ( true )
+# define SHW_PTE_IS_D(Pte)              ( true )
+# define SHW_PTE_IS_P_RW(Pte)           ( ((Pte).u & (EPT_E_READ | EPT_E_WRITE)) == (EPT_E_READ | EPT_E_WRITE) )
+# define SHW_PTE_IS_TRACK_DIRTY(Pte)    ( false )
+# define SHW_PTE_GET_HCPHYS(Pte)        ( (Pte).u & EPT_PTE_PG_MASK )
+# define SHW_PTE_LOG64(Pte)             ( (Pte).u )
+# define SHW_PTE_GET_U(Pte)             ( (Pte).u )             /**< Use with care. */
+# define SHW_PTE_SET(Pte, uNew)         do { (Pte).u = (uNew); } while (0)
+# define SHW_PTE_ATOMIC_SET(Pte, uNew)  do { ASMAtomicWriteU64(&(Pte).u, (uNew)); } while (0)
+# define SHW_PTE_ATOMIC_SET2(Pte, Pte2) do { ASMAtomicWriteU64(&(Pte).u, (Pte2).u); } while (0)
+# define SHW_PTE_SET_RO(Pte)            do { (Pte).u &= ~(uint64_t)EPT_E_WRITE; } while (0)
+# define SHW_PTE_SET_RW(Pte)            do { (Pte).u |= EPT_E_WRITE; } while (0)
+# define SHW_PT_SHIFT                   EPT_PT_SHIFT
+# define SHW_PT_MASK                    EPT_PT_MASK
+# define SHW_PDPT_SHIFT                 EPT_PDPT_SHIFT
+# define SHW_PDPT_MASK                  EPT_PDPT_MASK
+# define SHW_PDPE_PG_MASK               EPT_PDPE_PG_MASK
+# define SHW_TOTAL_PD_ENTRIES           (EPT_PG_AMD64_ENTRIES * EPT_PG_AMD64_PDPE_ENTRIES)
+
+#else
+# define SHWUINT                        uint64_t
+# define SHWPT                          PGMSHWPTPAE
+# define PSHWPT                         PPGMSHWPTPAE
+# define SHWPTE                         PGMSHWPTEPAE
+# define PSHWPTE                        PPGMSHWPTEPAE
+# define SHWPD                          X86PDPAE
+# define PSHWPD                         PX86PDPAE
+# define SHWPDE                         X86PDEPAE
+# define PSHWPDE                        PX86PDEPAE
+# define SHW_PDE_PG_MASK                X86_PDE_PAE_PG_MASK
+# define SHW_PD_SHIFT                   X86_PD_PAE_SHIFT
+# define SHW_PD_MASK                    X86_PD_PAE_MASK
+# define SHW_PDE_IS_P(Pde)              ( (Pde).u & X86_PDE_P )
+# define SHW_PDE_IS_A(Pde)              ( (Pde).u & X86_PDE_A )
+# define SHW_PDE_IS_BIG(Pde)            ( (Pde).u & X86_PDE_PS )
+# define SHW_PDE_ATOMIC_SET(Pde, uNew)  do { ASMAtomicWriteU64(&(Pde).u, (uNew)); } while (0)
+# define SHW_PDE_ATOMIC_SET2(Pde, Pde2) do { ASMAtomicWriteU64(&(Pde).u, (Pde2).u); } while (0)
+# define SHW_PTE_PG_MASK                X86_PTE_PAE_PG_MASK
+# define SHW_PTE_IS_P(Pte)              PGMSHWPTEPAE_IS_P(Pte)
+# define SHW_PTE_IS_RW(Pte)             PGMSHWPTEPAE_IS_RW(Pte)
+# define SHW_PTE_IS_US(Pte)             PGMSHWPTEPAE_IS_US(Pte)
+# define SHW_PTE_IS_A(Pte)              PGMSHWPTEPAE_IS_A(Pte)
+# define SHW_PTE_IS_D(Pte)              PGMSHWPTEPAE_IS_D(Pte)
+# define SHW_PTE_IS_P_RW(Pte)           PGMSHWPTEPAE_IS_P_RW(Pte)
+# define SHW_PTE_IS_TRACK_DIRTY(Pte)    PGMSHWPTEPAE_IS_TRACK_DIRTY(Pte)
+# define SHW_PTE_GET_HCPHYS(Pte)        PGMSHWPTEPAE_GET_HCPHYS(Pte)
+# define SHW_PTE_LOG64(Pte)             PGMSHWPTEPAE_GET_LOG(Pte)
+# define SHW_PTE_GET_U(Pte)             PGMSHWPTEPAE_GET_U(Pte)     /**< Use with care. */
+# define SHW_PTE_SET(Pte, uNew)         PGMSHWPTEPAE_SET(Pte, uNew)
+# define SHW_PTE_ATOMIC_SET(Pte, uNew)  PGMSHWPTEPAE_ATOMIC_SET(Pte, uNew)
+# define SHW_PTE_ATOMIC_SET2(Pte, Pte2) PGMSHWPTEPAE_ATOMIC_SET2(Pte, Pte2)
+# define SHW_PTE_SET_RO(Pte)            PGMSHWPTEPAE_SET_RO(Pte)
+# define SHW_PTE_SET_RW(Pte)            PGMSHWPTEPAE_SET_RW(Pte)
+# define SHW_PT_SHIFT                   X86_PT_PAE_SHIFT
+# define SHW_PT_MASK                    X86_PT_PAE_MASK
+
+# if PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64 || /* whatever: */ PGM_SHW_TYPE == PGM_TYPE_NONE
+#  define SHW_PDPT_SHIFT                X86_PDPT_SHIFT
+#  define SHW_PDPT_MASK                 X86_PDPT_MASK_AMD64
+#  define SHW_PDPE_PG_MASK              X86_PDPE_PG_MASK
+#  define SHW_TOTAL_PD_ENTRIES          (X86_PG_AMD64_ENTRIES * X86_PG_AMD64_PDPE_ENTRIES)
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE
+#  define SHW_PDPT_SHIFT                X86_PDPT_SHIFT
+#  define SHW_PDPT_MASK                 X86_PDPT_MASK_PAE
+#  define SHW_PDPE_PG_MASK              X86_PDPE_PG_MASK
+#  define SHW_TOTAL_PD_ENTRIES          (X86_PG_PAE_ENTRIES * X86_PG_PAE_PDPE_ENTRIES)
+
+# else
+#  error "Misconfigured PGM_SHW_TYPE or something..."
+# endif
+#endif
+
+#if PGM_SHW_TYPE == PGM_TYPE_NONE && PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
+# error "PGM_TYPE_IS_NESTED_OR_EPT is true for PGM_TYPE_NONE!"
+#endif
+
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+RT_C_DECLS_BEGIN
+PGM_SHW_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys);
+PGM_SHW_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, size_t cbPages, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags);
+PGM_SHW_DECL(int, Exit)(PVMCPUCC pVCpu);
+#ifdef IN_RING3
+PGM_SHW_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta);
+#endif
+RT_C_DECLS_END
+
+
+/**
+ * Enters the shadow mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+PGM_SHW_DECL(int, Enter)(PVMCPUCC pVCpu)
+{
+#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
+
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    RTGCPHYS    GCPhysCR3;
+    PGMPOOLKIND enmKind;
+    if (pVCpu->pgm.s.enmGuestSlatMode != PGMSLAT_EPT)
+    {
+        GCPhysCR3 = RT_BIT_64(63);
+        enmKind   = PGMPOOLKIND_ROOT_NESTED;
+    }
+    else
+    {
+        GCPhysCR3 = pVCpu->pgm.s.uEptPtr & EPT_EPTP_PG_MASK;
+        enmKind   = PGMPOOLKIND_EPT_PML4_FOR_EPT_PML4;
+    }
+# else
+    RTGCPHYS    const GCPhysCR3 = RT_BIT_64(63);
+    PGMPOOLKIND const enmKind   = PGMPOOLKIND_ROOT_NESTED;
+# endif
+    PVMCC       const pVM       = pVCpu->CTX_SUFF(pVM);
+
+    Assert(HMIsNestedPagingActive(pVM));
+    Assert(pVM->pgm.s.fNestedPaging);
+    Assert(!pVCpu->pgm.s.pShwPageCR3R3);
+
+    PGM_LOCK_VOID(pVM);
+
+    PPGMPOOLPAGE pNewShwPageCR3;
+    int rc = pgmPoolAlloc(pVM, GCPhysCR3, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
+                          NIL_PGMPOOL_IDX, UINT32_MAX, true /*fLockPage*/,
+                          &pNewShwPageCR3);
+    AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
+
+    pVCpu->pgm.s.pShwPageCR3R3 = pgmPoolConvertPageToR3(pVM->pgm.s.CTX_SUFF(pPool), pNewShwPageCR3);
+    pVCpu->pgm.s.pShwPageCR3R0 = pgmPoolConvertPageToR0(pVM->pgm.s.CTX_SUFF(pPool), pNewShwPageCR3);
+
+    PGM_UNLOCK(pVM);
+
+    Log(("Enter nested shadow paging mode: root %RHv phys %RHp\n", pVCpu->pgm.s.pShwPageCR3R3, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->Core.Key));
+#else
+    NOREF(pVCpu);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exits the shadow mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+PGM_SHW_DECL(int, Exit)(PVMCPUCC pVCpu)
+{
+#if PGM_TYPE_IS_NESTED_OR_EPT(PGM_SHW_TYPE)
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (pVCpu->pgm.s.CTX_SUFF(pShwPageCR3))
+    {
+        PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
+
+        PGM_LOCK_VOID(pVM);
+
+# if defined(VBOX_WITH_NESTED_HWVIRT_VMX_EPT) && PGM_SHW_TYPE == PGM_TYPE_EPT
+        if (pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_EPT)
+            pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
+# endif
+
+        /* Do *not* unlock this page as we have two of them floating around in the 32-bit host & 64-bit guest case.
+         * We currently assert when you try to free one of them; don't bother to really allow this.
+         *
+         * Note that this is two nested paging root pages max. This isn't a leak. They are reused.
+         */
+        /* pgmPoolUnlockPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)); */
+
+        pgmPoolFreeByPage(pPool, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3), NIL_PGMPOOL_IDX, UINT32_MAX);
+        pVCpu->pgm.s.pShwPageCR3R3 = 0;
+        pVCpu->pgm.s.pShwPageCR3R0 = 0;
+
+        PGM_UNLOCK(pVM);
+
+        Log(("Leave nested shadow paging mode\n"));
+    }
+#else
+    RT_NOREF_PV(pVCpu);
+#endif
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets effective page information (from the VMM page directory).
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Guest Context virtual address of the page.
+ * @param   pfFlags     Where to store the flags. These are X86_PTE_*.
+ * @param   pHCPhys     Where to store the HC physical address of the page.
+ *                      This is page aligned.
+ * @remark  You should use PGMMapGetPage() for pages in a mapping.
+ */
+PGM_SHW_DECL(int, GetPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
+{
+#if PGM_SHW_TYPE == PGM_TYPE_NONE
+    RT_NOREF(pVCpu, GCPtr);
+    AssertFailed();
+    *pfFlags = 0;
+    *pHCPhys = NIL_RTHCPHYS;
+    return VERR_PGM_SHW_NONE_IPE;
+
+#else  /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Get the PDE.
+     */
+# if PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64
+    X86PDEPAE Pde;
+
+    /* PML4 */
+    X86PML4E        Pml4e = pgmShwGetLongModePML4E(pVCpu, GCPtr);
+    if (!(Pml4e.u & X86_PML4E_P))
+        return VERR_PAGE_TABLE_NOT_PRESENT;
+
+    /* PDPT */
+    PX86PDPT        pPDPT;
+    int rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pml4e.u & X86_PML4E_PG_MASK, &pPDPT);
+    if (RT_FAILURE(rc))
+        return rc;
+    const unsigned  iPDPT = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
+    X86PDPE         Pdpe = pPDPT->a[iPDPT];
+    if (!(Pdpe.u & X86_PDPE_P))
+        return VERR_PAGE_TABLE_NOT_PRESENT;
+
+    /* PD */
+    PX86PDPAE       pPd;
+    rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pdpe.u & X86_PDPE_PG_MASK, &pPd);
+    if (RT_FAILURE(rc))
+        return rc;
+    const unsigned  iPd = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
+    Pde = pPd->a[iPd];
+
+    /* Merge accessed, write, user and no-execute bits into the PDE. */
+    AssertCompile(X86_PML4E_A  == X86_PDPE_A  && X86_PML4E_A  == X86_PDE_A);
+    AssertCompile(X86_PML4E_RW == X86_PDPE_RW && X86_PML4E_RW == X86_PDE_RW);
+    AssertCompile(X86_PML4E_US == X86_PDPE_US && X86_PML4E_US == X86_PDE_US);
+    AssertCompile(X86_PML4E_NX == X86_PDPE_LM_NX && X86_PML4E_NX == X86_PDE_PAE_NX);
+    Pde.u &= (Pml4e.u & Pdpe.u) | ~(X86PGPAEUINT)(X86_PML4E_A | X86_PML4E_RW | X86_PML4E_US);
+    Pde.u |= (Pml4e.u | Pdpe.u) & X86_PML4E_NX;
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE
+    X86PDEPAE       Pde = pgmShwGetPaePDE(pVCpu, GCPtr);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_EPT
+    Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT);
+    PEPTPD          pPDDst;
+    int rc = pgmShwGetEPTPDPtr(pVCpu, GCPtr, NULL, &pPDDst);
+    if (rc == VINF_SUCCESS) /** @todo this function isn't expected to return informational status codes. Check callers / fix. */
+    { /* likely */ }
+    else
+    {
+        AssertRC(rc);
+        return rc;
+    }
+    Assert(pPDDst);
+
+    const unsigned  iPd = ((GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK);
+    EPTPDE Pde = pPDDst->a[iPd];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_NESTED_32BIT
+    X86PDE          Pde = pgmShwGet32BitPDE(pVCpu, GCPtr);
+
+# else
+#  error "Misconfigured PGM_SHW_TYPE or something..."
+# endif
+    if (!SHW_PDE_IS_P(Pde))
+        return VERR_PAGE_TABLE_NOT_PRESENT;
+
+    /* Deal with large pages. */
+    if (SHW_PDE_IS_BIG(Pde))
+    {
+        /*
+         * Store the results.
+         * RW and US flags depend on the entire page translation hierarchy - except for
+         * legacy PAE which has a simplified PDPE.
+         */
+        if (pfFlags)
+        {
+            *pfFlags = (Pde.u & ~SHW_PDE_PG_MASK);
+# if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64
+            if (   (Pde.u & X86_PTE_PAE_NX)
+#  if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE)
+                && CPUMIsGuestNXEnabled(pVCpu) /** @todo why do we have to check the guest state here? */
+#  endif
+               )
+                *pfFlags |= X86_PTE_PAE_NX;
+# endif
+        }
+
+        if (pHCPhys)
+            *pHCPhys = (Pde.u & SHW_PDE_PG_MASK) + (GCPtr & (RT_BIT(SHW_PD_SHIFT) - 1) & X86_PAGE_4K_BASE_MASK);
+
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Get PT entry.
+     */
+    PSHWPT          pPT;
+    int rc2 = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pde.u & SHW_PDE_PG_MASK, &pPT);
+    if (RT_FAILURE(rc2))
+        return rc2;
+    const unsigned  iPt = (GCPtr >> SHW_PT_SHIFT) & SHW_PT_MASK;
+    SHWPTE          Pte = pPT->a[iPt];
+    if (!SHW_PTE_IS_P(Pte))
+        return VERR_PAGE_NOT_PRESENT;
+
+    /*
+     * Store the results.
+     * RW and US flags depend on the entire page translation hierarchy - except for
+     * legacy PAE which has a simplified PDPE.
+     */
+    if (pfFlags)
+    {
+        *pfFlags = (SHW_PTE_GET_U(Pte) & ~SHW_PTE_PG_MASK)
+                 & ((Pde.u & (X86_PTE_RW | X86_PTE_US)) | ~(uint64_t)(X86_PTE_RW | X86_PTE_US));
+
+# if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE) || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64
+        /* The NX bit is determined by a bitwise OR between the PT and PD */
+        if (   ((SHW_PTE_GET_U(Pte) | Pde.u) & X86_PTE_PAE_NX)
+#  if PGM_WITH_NX(PGM_SHW_TYPE, PGM_SHW_TYPE)
+            && CPUMIsGuestNXEnabled(pVCpu) /** @todo why do we have to check the guest state here? */
+#  endif
+           )
+            *pfFlags |= X86_PTE_PAE_NX;
+# endif
+    }
+
+    if (pHCPhys)
+        *pHCPhys = SHW_PTE_GET_HCPHYS(Pte);
+
+    return VINF_SUCCESS;
+#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+}
+
+
+/**
+ * Modify page flags for a range of pages in the shadow context.
+ *
+ * The existing flags are ANDed with the fMask and ORed with the fFlags.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   GCPtr       Virtual address of the first page in the range. Page aligned!
+ * @param   cb          Size (in bytes) of the range to apply the modification to. Page aligned!
+ * @param   fFlags      The OR  mask - page flags X86_PTE_*, excluding the page mask of course.
+ * @param   fMask       The AND mask - page flags X86_PTE_*.
+ *                      Be extremely CAREFUL with ~'ing values because they can be 32-bit!
+ * @param   fOpFlags    A combination of the PGM_MK_PK_XXX flags.
+ * @remark  You must use PGMMapModifyPage() for pages in a mapping.
+ */
+PGM_SHW_DECL(int, ModifyPage)(PVMCPUCC pVCpu, RTGCUINTPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
+{
+#if PGM_SHW_TYPE == PGM_TYPE_NONE
+    RT_NOREF(pVCpu, GCPtr, cb, fFlags, fMask, fOpFlags);
+    AssertFailed();
+    return VERR_PGM_SHW_NONE_IPE;
+
+#else  /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PGM_LOCK_ASSERT_OWNER(pVM);
+
+    /*
+     * Walk page tables and pages till we're done.
+     */
+    int rc;
+    for (;;)
+    {
+        /*
+         * Get the PDE.
+         */
+# if PGM_SHW_TYPE == PGM_TYPE_AMD64 || PGM_SHW_TYPE == PGM_TYPE_NESTED_AMD64
+        X86PDEPAE       Pde;
+        /* PML4 */
+        X86PML4E        Pml4e = pgmShwGetLongModePML4E(pVCpu, GCPtr);
+        if (!(Pml4e.u & X86_PML4E_P))
+            return VERR_PAGE_TABLE_NOT_PRESENT;
+
+        /* PDPT */
+        PX86PDPT        pPDPT;
+        rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pml4e.u & X86_PML4E_PG_MASK, &pPDPT);
+        if (RT_FAILURE(rc))
+            return rc;
+        const unsigned  iPDPT = (GCPtr >> SHW_PDPT_SHIFT) & SHW_PDPT_MASK;
+        X86PDPE         Pdpe = pPDPT->a[iPDPT];
+        if (!(Pdpe.u & X86_PDPE_P))
+            return VERR_PAGE_TABLE_NOT_PRESENT;
+
+        /* PD */
+        PX86PDPAE       pPd;
+        rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pdpe.u & X86_PDPE_PG_MASK, &pPd);
+        if (RT_FAILURE(rc))
+            return rc;
+        const unsigned iPd = (GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK;
+        Pde = pPd->a[iPd];
+
+# elif PGM_SHW_TYPE == PGM_TYPE_PAE || PGM_SHW_TYPE == PGM_TYPE_NESTED_PAE
+        X86PDEPAE       Pde = pgmShwGetPaePDE(pVCpu, GCPtr);
+
+# elif PGM_SHW_TYPE == PGM_TYPE_EPT
+        Assert(pVCpu->pgm.s.enmGuestSlatMode == PGMSLAT_DIRECT);
+        const unsigned  iPd = ((GCPtr >> SHW_PD_SHIFT) & SHW_PD_MASK);
+        PEPTPD          pPDDst;
+        EPTPDE          Pde;
+
+        rc = pgmShwGetEPTPDPtr(pVCpu, GCPtr, NULL, &pPDDst);
+        if (rc != VINF_SUCCESS)
+        {
+            AssertRC(rc);
+            return rc;
+        }
+        Assert(pPDDst);
+        Pde = pPDDst->a[iPd];
+
+# else /* PGM_TYPE_32BIT || PGM_SHW_TYPE == PGM_TYPE_NESTED_32BIT */
+        X86PDE          Pde = pgmShwGet32BitPDE(pVCpu, GCPtr);
+# endif
+        if (!SHW_PDE_IS_P(Pde))
+            return VERR_PAGE_TABLE_NOT_PRESENT;
+
+        AssertFatalMsg(!SHW_PDE_IS_BIG(Pde), ("Pde=%#RX64\n", (uint64_t)Pde.u));
+
+        /*
+         * Map the page table.
+         */
+        PSHWPT          pPT;
+        rc = PGM_HCPHYS_2_PTR(pVM, pVCpu, Pde.u & SHW_PDE_PG_MASK, &pPT);
+        if (RT_FAILURE(rc))
+            return rc;
+
+        unsigned        iPTE = (GCPtr >> SHW_PT_SHIFT) & SHW_PT_MASK;
+        while (iPTE < RT_ELEMENTS(pPT->a))
+        {
+            if (SHW_PTE_IS_P(pPT->a[iPTE]))
+            {
+                SHWPTE const    OrgPte = pPT->a[iPTE];
+                SHWPTE          NewPte;
+
+                SHW_PTE_SET(NewPte, (SHW_PTE_GET_U(OrgPte) & (fMask | SHW_PTE_PG_MASK)) | (fFlags & ~SHW_PTE_PG_MASK));
+                if (!SHW_PTE_IS_P(NewPte))
+                {
+                    /** @todo Some CSAM code path might end up here and upset
+                     *  the page pool. */
+                    AssertMsgFailed(("NewPte=%#RX64 OrgPte=%#RX64 GCPtr=%#RGv\n", SHW_PTE_LOG64(NewPte), SHW_PTE_LOG64(OrgPte), GCPtr));
+                }
+                else if (   SHW_PTE_IS_RW(NewPte)
+                         && !SHW_PTE_IS_RW(OrgPte)
+                         && !(fOpFlags & PGM_MK_PG_IS_MMIO2) )
+                {
+                    /** @todo Optimize \#PF handling by caching data.  We can
+                     *        then use this when PGM_MK_PG_IS_WRITE_FAULT is
+                     *        set instead of resolving the guest physical
+                     *        address yet again. */
+                    PGMPTWALK GstWalk;
+                    rc = PGMGstGetPage(pVCpu, GCPtr, &GstWalk);
+                    AssertRC(rc);
+                    if (RT_SUCCESS(rc))
+                    {
+                        Assert((GstWalk.fEffective & X86_PTE_RW) || !(CPUMGetGuestCR0(pVCpu) & X86_CR0_WP /* allow netware hack */));
+                        PPGMPAGE pPage = pgmPhysGetPage(pVM, GstWalk.GCPhys);
+                        Assert(pPage);
+                        if (pPage)
+                        {
+                            rc = pgmPhysPageMakeWritable(pVM, pPage, GstWalk.GCPhys);
+                            AssertRCReturn(rc, rc);
+                            Log(("%s: pgmPhysPageMakeWritable on %RGv / %RGp %R[pgmpage]\n", __PRETTY_FUNCTION__, GCPtr, GstWalk.GCPhys, pPage));
+                        }
+                    }
+                }
+
+                SHW_PTE_ATOMIC_SET2(pPT->a[iPTE], NewPte);
+# if PGM_SHW_TYPE == PGM_TYPE_EPT
+                HMInvalidatePhysPage(pVM, (RTGCPHYS)GCPtr);
+# else
+                PGM_INVL_PG_ALL_VCPU(pVM, GCPtr);
+# endif
+            }
+
+            /* next page */
+            cb -= HOST_PAGE_SIZE;
+            if (!cb)
+                return VINF_SUCCESS;
+            GCPtr += HOST_PAGE_SIZE;
+            iPTE++;
+        }
+    }
+#endif /* PGM_SHW_TYPE != PGM_TYPE_NONE */
+}
+
+
+#ifdef IN_RING3
+/**
+ * Relocate any GC pointers related to shadow mode paging.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   offDelta    The relocation offset.
+ */
+PGM_SHW_DECL(int, Relocate)(PVMCPUCC pVCpu, RTGCPTR offDelta)
+{
+    RT_NOREF(pVCpu, offDelta);
+    return VINF_SUCCESS;
+}
+#endif
+
diff --git a/src/VBox/VMM/VMMAll/SELMAll.cpp b/src/VBox/VMM/VMMAll/SELMAll.cpp
new file mode 100644
index 00000000..c4320022
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/SELMAll.cpp
@@ -0,0 +1,392 @@
+/* $Id: SELMAll.cpp $ */
+/** @file
+ * SELM All contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_SELM
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/em.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/hm.h>
+#include "SELMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/param.h>
+#include <iprt/assert.h>
+#include <VBox/vmm/vmm.h>
+#include <iprt/x86.h>
+#include <iprt/string.h>
+
+
+
+/**
+ * Converts a GC selector based address to a flat address.
+ *
+ * No limit checks are done. Use the SELMToFlat*() or SELMValidate*() functions
+ * for that.
+ *
+ * @returns Flat address.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idxSeg      The selector register to use (X86_SREG_XXX).
+ * @param   pCtx        Pointer to the register context for the CPU.
+ * @param   Addr        Address part.
+ */
+VMMDECL(RTGCPTR) SELMToFlat(PVMCPUCC pVCpu, unsigned idxSeg, PCPUMCTX pCtx, RTGCPTR Addr)
+{
+    Assert(idxSeg < RT_ELEMENTS(pCtx->aSRegs));
+    PCPUMSELREG pSReg = &pCtx->aSRegs[idxSeg];
+
+    /*
+     * Deal with real & v86 mode first.
+     */
+    if (    pCtx->eflags.Bits.u1VM
+        ||  CPUMIsGuestInRealMode(pVCpu))
+    {
+        uint32_t uFlat = (uint32_t)Addr & 0xffff;
+        if (CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg))
+            uFlat += (uint32_t)pSReg->u64Base;
+        else
+            uFlat += (uint32_t)pSReg->Sel << 4;
+        return (RTGCPTR)uFlat;
+    }
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->cs));
+
+    /* 64 bits mode: CS, DS, ES and SS are treated as if each segment base is 0
+       (Intel(r) 64 and IA-32 Architectures Software Developer's Manual: 3.4.2.1). */
+    if (    pCtx->cs.Attr.n.u1Long
+        &&  CPUMIsGuestInLongMode(pVCpu))
+    {
+        switch (idxSeg)
+        {
+            case X86_SREG_FS:
+            case X86_SREG_GS:
+                return (RTGCPTR)(pSReg->u64Base + Addr);
+
+            default:
+                return Addr;    /* base 0 */
+        }
+    }
+
+    /* AMD64 manual: compatibility mode ignores the high 32 bits when calculating an effective address. */
+    Assert(pSReg->u64Base <= 0xffffffff);
+    return (uint32_t)pSReg->u64Base + (uint32_t)Addr;
+}
+
+
+/**
+ * Converts a GC selector based address to a flat address.
+ *
+ * Some basic checking is done, but not all kinds yet.
+ *
+ * @returns VBox status
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   idxSeg      The selector register to use (X86_SREG_XXX).
+ * @param   pCtx        Pointer to the register context for the CPU.
+ * @param   Addr        Address part.
+ * @param   fFlags      SELMTOFLAT_FLAGS_*
+ *                      GDT entires are valid.
+ * @param   ppvGC       Where to store the GC flat address.
+ */
+VMMDECL(int) SELMToFlatEx(PVMCPU pVCpu, unsigned idxSeg, PCPUMCTX pCtx, RTGCPTR Addr, uint32_t fFlags, PRTGCPTR ppvGC)
+{
+    AssertReturn(idxSeg < RT_ELEMENTS(pCtx->aSRegs), VERR_INVALID_PARAMETER);
+    PCPUMSELREG pSReg = &pCtx->aSRegs[idxSeg];
+
+    /*
+     * Deal with real & v86 mode first.
+     */
+    if (    pCtx->eflags.Bits.u1VM
+        ||  CPUMIsGuestInRealMode(pVCpu))
+    {
+        if (ppvGC)
+        {
+            uint32_t uFlat = (uint32_t)Addr & 0xffff;
+            if (CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg))
+                *ppvGC = (uint32_t)pSReg->u64Base + uFlat;
+            else
+                *ppvGC = ((uint32_t)pSReg->Sel << 4) + uFlat;
+        }
+        return VINF_SUCCESS;
+    }
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg));
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &pCtx->cs));
+
+    /* 64 bits mode: CS, DS, ES and SS are treated as if each segment base is 0
+       (Intel(r) 64 and IA-32 Architectures Software Developer's Manual: 3.4.2.1). */
+    RTGCPTR  pvFlat;
+    bool     fCheckLimit   = true;
+    if (    pCtx->cs.Attr.n.u1Long
+        &&  CPUMIsGuestInLongMode(pVCpu))
+    {
+        fCheckLimit = false;
+        switch (idxSeg)
+        {
+            case X86_SREG_FS:
+            case X86_SREG_GS:
+                pvFlat = pSReg->u64Base + Addr;
+                break;
+
+            default:
+                pvFlat = Addr;
+                break;
+        }
+    }
+    else
+    {
+        /* AMD64 manual: compatibility mode ignores the high 32 bits when calculating an effective address. */
+        Assert(pSReg->u64Base <= UINT32_C(0xffffffff));
+        pvFlat  = (uint32_t)pSReg->u64Base + (uint32_t)Addr;
+        Assert(pvFlat <= UINT32_MAX);
+    }
+
+    /*
+     * Check type if present.
+     */
+    if (pSReg->Attr.n.u1Present)
+    {
+        switch (pSReg->Attr.n.u4Type)
+        {
+            /* Read only selector type. */
+            case X86_SEL_TYPE_RO:
+            case X86_SEL_TYPE_RO_ACC:
+            case X86_SEL_TYPE_RW:
+            case X86_SEL_TYPE_RW_ACC:
+            case X86_SEL_TYPE_EO:
+            case X86_SEL_TYPE_EO_ACC:
+            case X86_SEL_TYPE_ER:
+            case X86_SEL_TYPE_ER_ACC:
+                if (!(fFlags & SELMTOFLAT_FLAGS_NO_PL))
+                {
+                    /** @todo fix this mess */
+                }
+                /* check limit. */
+                if (fCheckLimit && Addr > pSReg->u32Limit)
+                    return VERR_OUT_OF_SELECTOR_BOUNDS;
+                /* ok */
+                if (ppvGC)
+                    *ppvGC = pvFlat;
+                return VINF_SUCCESS;
+
+            case X86_SEL_TYPE_EO_CONF:
+            case X86_SEL_TYPE_EO_CONF_ACC:
+            case X86_SEL_TYPE_ER_CONF:
+            case X86_SEL_TYPE_ER_CONF_ACC:
+                if (!(fFlags & SELMTOFLAT_FLAGS_NO_PL))
+                {
+                    /** @todo fix this mess */
+                }
+                /* check limit. */
+                if (fCheckLimit && Addr > pSReg->u32Limit)
+                    return VERR_OUT_OF_SELECTOR_BOUNDS;
+                /* ok */
+                if (ppvGC)
+                    *ppvGC = pvFlat;
+                return VINF_SUCCESS;
+
+            case X86_SEL_TYPE_RO_DOWN:
+            case X86_SEL_TYPE_RO_DOWN_ACC:
+            case X86_SEL_TYPE_RW_DOWN:
+            case X86_SEL_TYPE_RW_DOWN_ACC:
+                if (!(fFlags & SELMTOFLAT_FLAGS_NO_PL))
+                {
+                    /** @todo fix this mess */
+                }
+                /* check limit. */
+                if (fCheckLimit)
+                {
+                    if (!pSReg->Attr.n.u1Granularity && Addr > UINT32_C(0xffff))
+                        return VERR_OUT_OF_SELECTOR_BOUNDS;
+                    if (Addr <= pSReg->u32Limit)
+                        return VERR_OUT_OF_SELECTOR_BOUNDS;
+                }
+                /* ok */
+                if (ppvGC)
+                    *ppvGC = pvFlat;
+                return VINF_SUCCESS;
+
+            default:
+                return VERR_INVALID_SELECTOR;
+
+        }
+    }
+    return VERR_SELECTOR_NOT_PRESENT;
+}
+
+
+
+/**
+ * Validates and converts a GC selector based code address to a flat
+ * address when in real or v8086 mode.
+ *
+ * @returns VINF_SUCCESS.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ * @param   SelCS   Selector part.
+ * @param   pSReg   The hidden CS register part. Optional.
+ * @param   Addr    Address part.
+ * @param   ppvFlat Where to store the flat address.
+ */
+DECLINLINE(int) selmValidateAndConvertCSAddrRealMode(PVMCPU pVCpu, RTSEL SelCS, PCCPUMSELREGHID pSReg, RTGCPTR Addr,
+                                                     PRTGCPTR ppvFlat)
+{
+    NOREF(pVCpu);
+    uint32_t uFlat = Addr & 0xffff;
+    if (!pSReg || !CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSReg))
+        uFlat += (uint32_t)SelCS << 4;
+    else
+        uFlat += (uint32_t)pSReg->u64Base;
+    *ppvFlat = uFlat;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Validates and converts a GC selector based code address to a flat address
+ * when in protected/long mode using the standard hidden selector registers
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   SelCPL      Current privilege level.  Get this from SS - CS might be
+ *                      conforming!  A full selector can be passed, we'll only
+ *                      use the RPL part.
+ * @param   SelCS       Selector part.
+ * @param   pSRegCS     The full CS selector register.
+ * @param   Addr        The address (think IP/EIP/RIP).
+ * @param   ppvFlat     Where to store the flat address upon successful return.
+ */
+DECLINLINE(int) selmValidateAndConvertCSAddrHidden(PVMCPU pVCpu, RTSEL SelCPL, RTSEL SelCS, PCCPUMSELREGHID pSRegCS,
+                                                   RTGCPTR Addr, PRTGCPTR ppvFlat)
+{
+    NOREF(SelCPL); NOREF(SelCS);
+
+    /*
+     * Check if present.
+     */
+    if (pSRegCS->Attr.n.u1Present)
+    {
+        /*
+         * Type check.
+         */
+        if (     pSRegCS->Attr.n.u1DescType == 1
+            &&  (pSRegCS->Attr.n.u4Type & X86_SEL_TYPE_CODE))
+        {
+            /* 64 bits mode: CS, DS, ES and SS are treated as if each segment base is 0
+               (Intel(r) 64 and IA-32 Architectures Software Developer's Manual: 3.4.2.1). */
+            if (    pSRegCS->Attr.n.u1Long
+                &&  CPUMIsGuestInLongMode(pVCpu))
+            {
+                *ppvFlat = Addr;
+                return VINF_SUCCESS;
+            }
+
+            /*
+             * Limit check. Note that the limit in the hidden register is the
+             * final value. The granularity bit was included in its calculation.
+             */
+            uint32_t u32Limit = pSRegCS->u32Limit;
+            if ((uint32_t)Addr <= u32Limit)
+            {
+                *ppvFlat = (uint32_t)Addr + (uint32_t)pSRegCS->u64Base;
+                return VINF_SUCCESS;
+            }
+
+            return VERR_OUT_OF_SELECTOR_BOUNDS;
+        }
+        return VERR_NOT_CODE_SELECTOR;
+    }
+    return VERR_SELECTOR_NOT_PRESENT;
+}
+
+
+/**
+ * Validates and converts a GC selector based code address to a flat address.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   fEFlags     Current EFLAGS.
+ * @param   SelCPL      Current privilege level.  Get this from SS - CS might be
+ *                      conforming!  A full selector can be passed, we'll only
+ *                      use the RPL part.
+ * @param   SelCS       Selector part.
+ * @param   pSRegCS     The full CS selector register.
+ * @param   Addr        The address (think IP/EIP/RIP).
+ * @param   ppvFlat     Where to store the flat address upon successful return.
+ */
+VMMDECL(int) SELMValidateAndConvertCSAddr(PVMCPU pVCpu, uint32_t fEFlags, RTSEL SelCPL, RTSEL SelCS, PCPUMSELREG pSRegCS,
+                                          RTGCPTR Addr, PRTGCPTR ppvFlat)
+{
+    if (   (fEFlags & X86_EFL_VM)
+        || CPUMIsGuestInRealMode(pVCpu))
+        return selmValidateAndConvertCSAddrRealMode(pVCpu, SelCS, pSRegCS, Addr, ppvFlat);
+
+    Assert(CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, pSRegCS));
+    Assert(pSRegCS->Sel == SelCS);
+
+    return selmValidateAndConvertCSAddrHidden(pVCpu, SelCPL, SelCS, pSRegCS, Addr, ppvFlat);
+}
+
+
+/**
+ * Gets info about the current TSS.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if we've got a TSS loaded.
+ * @retval  VERR_SELM_NO_TSS if we haven't got a TSS (rather unlikely).
+ *
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pGCPtrTss           Where to store the TSS address.
+ * @param   pcbTss              Where to store the TSS size limit.
+ * @param   pfCanHaveIOBitmap   Where to store the can-have-I/O-bitmap indicator. (optional)
+ */
+VMMDECL(int) SELMGetTSSInfo(PVM pVM, PVMCPU pVCpu, PRTGCUINTPTR pGCPtrTss, PRTGCUINTPTR pcbTss, bool *pfCanHaveIOBitmap)
+{
+    NOREF(pVM);
+
+    /*
+     * The TR hidden register is always valid.
+     */
+    CPUMSELREGHID trHid;
+    RTSEL tr = CPUMGetGuestTR(pVCpu, &trHid);
+    if (!(tr & X86_SEL_MASK_OFF_RPL))
+        return VERR_SELM_NO_TSS;
+
+    *pGCPtrTss = trHid.u64Base;
+    *pcbTss    = trHid.u32Limit + (trHid.u32Limit != UINT32_MAX); /* be careful. */
+    if (pfCanHaveIOBitmap)
+        *pfCanHaveIOBitmap = trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
+                          || trHid.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_BUSY;
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMAll/TMAll.cpp b/src/VBox/VMM/VMMAll/TMAll.cpp
new file mode 100644
index 00000000..677decdd
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/TMAll.cpp
@@ -0,0 +1,2850 @@
+/* $Id: TMAll.cpp $ */
+/** @file
+ * TM - Timeout Manager, all contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TM
+#ifdef DEBUG_bird
+# define DBGFTRACE_DISABLED /* annoying */
+#endif
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/dbgftrace.h>
+#ifdef IN_RING3
+#endif
+#include <VBox/vmm/pdmdev.h> /* (for TMTIMER_GET_CRITSECT implementation) */
+#include "TMInternal.h"
+#include <VBox/vmm/vmcc.h>
+
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/sup.h>
+#include <iprt/time.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/asm-math.h>
+#include <iprt/string.h>
+#ifdef IN_RING3
+# include <iprt/thread.h>
+#endif
+
+#include "TMInline.h"
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#ifdef VBOX_STRICT
+/** @def TMTIMER_GET_CRITSECT
+ * Helper for safely resolving the critical section for a timer belonging to a
+ * device instance.
+ * @todo needs reworking later as it uses PDMDEVINSR0::pDevInsR0RemoveMe.  */
+# ifdef IN_RING3
+#  define TMTIMER_GET_CRITSECT(a_pVM, a_pTimer) ((a_pTimer)->pCritSect)
+# else
+#  define TMTIMER_GET_CRITSECT(a_pVM, a_pTimer) tmRZTimerGetCritSect(a_pVM, a_pTimer)
+# endif
+#endif
+
+/** @def TMTIMER_ASSERT_CRITSECT
+ * Checks that the caller owns the critical section if one is associated with
+ * the timer. */
+#ifdef VBOX_STRICT
+# define TMTIMER_ASSERT_CRITSECT(a_pVM, a_pTimer) \
+    do { \
+        if ((a_pTimer)->pCritSect) \
+        { \
+            VMSTATE      enmState; \
+            PPDMCRITSECT pCritSect = TMTIMER_GET_CRITSECT(a_pVM, a_pTimer); \
+            AssertMsg(   pCritSect \
+                      && (   PDMCritSectIsOwner((a_pVM), pCritSect) \
+                          || (enmState = (a_pVM)->enmVMState) == VMSTATE_CREATING \
+                          || enmState == VMSTATE_RESETTING \
+                          || enmState == VMSTATE_RESETTING_LS ),\
+                      ("pTimer=%p (%s) pCritSect=%p (%s)\n", a_pTimer, (a_pTimer)->szName, \
+                       (a_pTimer)->pCritSect, R3STRING(PDMR3CritSectName((a_pTimer)->pCritSect)) )); \
+        } \
+    } while (0)
+#else
+# define TMTIMER_ASSERT_CRITSECT(pVM, pTimer) do { } while (0)
+#endif
+
+/** @def TMTIMER_ASSERT_SYNC_CRITSECT_ORDER
+ * Checks for lock order trouble between the timer critsect and the critical
+ * section critsect.  The virtual sync critsect must always be entered before
+ * the one associated with the timer (see TMR3TimerQueuesDo).  It is OK if there
+ * isn't any critical section associated with the timer or if the calling thread
+ * doesn't own it, ASSUMING of course that the thread using this macro is going
+ * to enter the virtual sync critical section anyway.
+ *
+ * @remarks This is a sligtly relaxed timer locking attitude compared to
+ *          TMTIMER_ASSERT_CRITSECT, however, the calling device/whatever code
+ *          should know what it's doing if it's stopping or starting a timer
+ *          without taking the device lock.
+ */
+#ifdef VBOX_STRICT
+# define TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer) \
+    do { \
+        if ((pTimer)->pCritSect) \
+        { \
+            VMSTATE      enmState; \
+            PPDMCRITSECT pCritSect = TMTIMER_GET_CRITSECT(pVM, pTimer); \
+            AssertMsg(   pCritSect \
+                      && (   !PDMCritSectIsOwner((pVM), pCritSect) \
+                          || PDMCritSectIsOwner((pVM), &(pVM)->tm.s.VirtualSyncLock) \
+                          || (enmState = (pVM)->enmVMState) == VMSTATE_CREATING \
+                          || enmState == VMSTATE_RESETTING \
+                          || enmState == VMSTATE_RESETTING_LS ),\
+                      ("pTimer=%p (%s) pCritSect=%p (%s)\n", pTimer, pTimer->szName, \
+                       (pTimer)->pCritSect, R3STRING(PDMR3CritSectName((pTimer)->pCritSect)) )); \
+        } \
+    } while (0)
+#else
+# define TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer) do { } while (0)
+#endif
+
+
+#if defined(VBOX_STRICT) && defined(IN_RING0)
+/**
+ * Helper for  TMTIMER_GET_CRITSECT
+ * @todo This needs a redo!
+ */
+DECLINLINE(PPDMCRITSECT) tmRZTimerGetCritSect(PVMCC pVM, PTMTIMER pTimer)
+{
+    if (pTimer->enmType == TMTIMERTYPE_DEV)
+    {
+        RTCCUINTREG  fSavedFlags = ASMAddFlags(X86_EFL_AC); /** @todo fix ring-3 pointer use */
+        PPDMDEVINSR0 pDevInsR0   = ((struct PDMDEVINSR3 *)pTimer->u.Dev.pDevIns)->pDevInsR0RemoveMe; /* !ring-3 read! */
+        ASMSetFlags(fSavedFlags);
+        struct PDMDEVINSR3 *pDevInsR3 = pDevInsR0->pDevInsForR3R0;
+        if (pTimer->pCritSect == pDevInsR3->pCritSectRoR3)
+            return pDevInsR0->pCritSectRoR0;
+        uintptr_t offCritSect = (uintptr_t)pTimer->pCritSect - (uintptr_t)pDevInsR3->pvInstanceDataR3;
+        if (offCritSect < pDevInsR0->pReg->cbInstanceShared)
+            return (PPDMCRITSECT)((uintptr_t)pDevInsR0->pvInstanceDataR0 + offCritSect);
+    }
+    RT_NOREF(pVM);
+    Assert(pTimer->pCritSect == NULL);
+    return NULL;
+}
+#endif /* VBOX_STRICT && IN_RING0 */
+
+
+/**
+ * Notification that execution is about to start.
+ *
+ * This call must always be paired with a TMNotifyEndOfExecution call.
+ *
+ * The function may, depending on the configuration, resume the TSC and future
+ * clocks that only ticks when we're executing guest code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(void) TMNotifyStartOfExecution(PVMCC pVM, PVMCPUCC pVCpu)
+{
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    pVCpu->tm.s.uTscStartExecuting = SUPReadTsc();
+    pVCpu->tm.s.fExecuting         = true;
+#endif
+    if (pVM->tm.s.fTSCTiedToExecution)
+        tmCpuTickResume(pVM, pVCpu);
+}
+
+
+/**
+ * Notification that execution has ended.
+ *
+ * This call must always be paired with a TMNotifyStartOfExecution call.
+ *
+ * The function may, depending on the configuration, suspend the TSC and future
+ * clocks that only ticks when we're executing guest code.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   uTsc        TSC value when exiting guest context.
+ */
+VMMDECL(void) TMNotifyEndOfExecution(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uTsc)
+{
+    if (pVM->tm.s.fTSCTiedToExecution)
+        tmCpuTickPause(pVCpu); /** @todo use uTsc here if we can. */
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    /*
+     * Calculate the elapsed tick count and convert it to nanoseconds.
+     */
+# ifdef IN_RING3
+    PSUPGLOBALINFOPAGE const pGip = g_pSUPGlobalInfoPage;
+    uint64_t       cTicks = uTsc - pVCpu->tm.s.uTscStartExecuting - SUPGetTscDelta(pGip);
+    uint64_t const uCpuHz = pGip ? SUPGetCpuHzFromGip(pGip) : pVM->tm.s.cTSCTicksPerSecondHost;
+# else
+    uint64_t       cTicks = uTsc - pVCpu->tm.s.uTscStartExecuting - SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet);
+    uint64_t const uCpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
+# endif
+    AssertStmt(cTicks <= uCpuHz << 2, cTicks = uCpuHz << 2); /* max 4 sec */
+
+    uint64_t cNsExecutingDelta;
+    if (uCpuHz < _4G)
+        cNsExecutingDelta = ASMMultU64ByU32DivByU32(cTicks, RT_NS_1SEC, uCpuHz);
+    else if (uCpuHz < 16*_1G64)
+        cNsExecutingDelta = ASMMultU64ByU32DivByU32(cTicks >> 2, RT_NS_1SEC, uCpuHz >> 2);
+    else
+    {
+        Assert(uCpuHz < 64 * _1G64);
+        cNsExecutingDelta = ASMMultU64ByU32DivByU32(cTicks >> 4, RT_NS_1SEC, uCpuHz >> 4);
+    }
+
+    /*
+     * Update the data.
+     *
+     * Note! We're not using strict memory ordering here to speed things us.
+     *       The data is in a single cache line and this thread is the only
+     *       one writing to that line, so I cannot quite imagine why we would
+     *       need any strict ordering here.
+     */
+    uint64_t const cNsExecutingNew = pVCpu->tm.s.cNsExecuting + cNsExecutingDelta;
+    uint32_t uGen = ASMAtomicUoIncU32(&pVCpu->tm.s.uTimesGen); Assert(uGen & 1);
+    ASMCompilerBarrier();
+    pVCpu->tm.s.fExecuting   = false;
+    pVCpu->tm.s.cNsExecuting = cNsExecutingNew;
+    pVCpu->tm.s.cPeriodsExecuting++;
+    ASMCompilerBarrier();
+    ASMAtomicUoWriteU32(&pVCpu->tm.s.uTimesGen, (uGen | 1) + 1);
+
+    /*
+     * Update stats.
+     */
+# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
+    STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecuting, cNsExecutingDelta);
+    if (cNsExecutingDelta < 5000)
+        STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecTiny, cNsExecutingDelta);
+    else if (cNsExecutingDelta < 50000)
+        STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecShort, cNsExecutingDelta);
+    else
+        STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsExecLong, cNsExecutingDelta);
+# endif
+
+    /* The timer triggers occational updating of the others and total stats: */
+    if (RT_LIKELY(!pVCpu->tm.s.fUpdateStats))
+    { /*likely*/ }
+    else
+    {
+        pVCpu->tm.s.fUpdateStats = false;
+
+        uint64_t const cNsTotalNew = RTTimeNanoTS() - pVCpu->tm.s.nsStartTotal;
+        uint64_t const cNsOtherNew = cNsTotalNew - cNsExecutingNew - pVCpu->tm.s.cNsHalted;
+
+# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
+        STAM_REL_COUNTER_ADD(&pVCpu->tm.s.StatNsTotal, cNsTotalNew - pVCpu->tm.s.cNsTotalStat);
+        int64_t const cNsOtherNewDelta = cNsOtherNew - pVCpu->tm.s.cNsOtherStat;
+        if (cNsOtherNewDelta > 0)
+            STAM_REL_COUNTER_ADD(&pVCpu->tm.s.StatNsOther, (uint64_t)cNsOtherNewDelta);
+# endif
+
+        pVCpu->tm.s.cNsTotalStat = cNsTotalNew;
+        pVCpu->tm.s.cNsOtherStat = cNsOtherNew;
+    }
+
+#endif
+}
+
+
+/**
+ * Notification that the cpu is entering the halt state
+ *
+ * This call must always be paired with a TMNotifyEndOfExecution call.
+ *
+ * The function may, depending on the configuration, resume the TSC and future
+ * clocks that only ticks when we're halted.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(void) TMNotifyStartOfHalt(PVMCPUCC pVCpu)
+{
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    pVCpu->tm.s.nsStartHalting = RTTimeNanoTS();
+    pVCpu->tm.s.fHalting       = true;
+#endif
+
+    if (    pVM->tm.s.fTSCTiedToExecution
+        &&  !pVM->tm.s.fTSCNotTiedToHalt)
+        tmCpuTickResume(pVM, pVCpu);
+}
+
+
+/**
+ * Notification that the cpu is leaving the halt state
+ *
+ * This call must always be paired with a TMNotifyStartOfHalt call.
+ *
+ * The function may, depending on the configuration, suspend the TSC and future
+ * clocks that only ticks when we're halted.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(void) TMNotifyEndOfHalt(PVMCPUCC pVCpu)
+{
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+
+    if (    pVM->tm.s.fTSCTiedToExecution
+        &&  !pVM->tm.s.fTSCNotTiedToHalt)
+        tmCpuTickPause(pVCpu);
+
+#ifndef VBOX_WITHOUT_NS_ACCOUNTING
+    uint64_t const u64NsTs        = RTTimeNanoTS();
+    uint64_t const cNsTotalNew    = u64NsTs - pVCpu->tm.s.nsStartTotal;
+    uint64_t const cNsHaltedDelta = u64NsTs - pVCpu->tm.s.nsStartHalting;
+    uint64_t const cNsHaltedNew   = pVCpu->tm.s.cNsHalted + cNsHaltedDelta;
+    uint64_t const cNsOtherNew    = cNsTotalNew - pVCpu->tm.s.cNsExecuting - cNsHaltedNew;
+
+    uint32_t uGen = ASMAtomicUoIncU32(&pVCpu->tm.s.uTimesGen); Assert(uGen & 1);
+    ASMCompilerBarrier();
+    pVCpu->tm.s.fHalting     = false;
+    pVCpu->tm.s.fUpdateStats = false;
+    pVCpu->tm.s.cNsHalted    = cNsHaltedNew;
+    pVCpu->tm.s.cPeriodsHalted++;
+    ASMCompilerBarrier();
+    ASMAtomicUoWriteU32(&pVCpu->tm.s.uTimesGen, (uGen | 1) + 1);
+
+# if defined(VBOX_WITH_STATISTICS) || defined(VBOX_WITH_NS_ACCOUNTING_STATS)
+    STAM_REL_PROFILE_ADD_PERIOD(&pVCpu->tm.s.StatNsHalted, cNsHaltedDelta);
+    STAM_REL_COUNTER_ADD(&pVCpu->tm.s.StatNsTotal, cNsTotalNew - pVCpu->tm.s.cNsTotalStat);
+    int64_t const cNsOtherNewDelta = cNsOtherNew - pVCpu->tm.s.cNsOtherStat;
+    if (cNsOtherNewDelta > 0)
+        STAM_REL_COUNTER_ADD(&pVCpu->tm.s.StatNsOther, (uint64_t)cNsOtherNewDelta);
+# endif
+    pVCpu->tm.s.cNsTotalStat = cNsTotalNew;
+    pVCpu->tm.s.cNsOtherStat = cNsOtherNew;
+#endif
+}
+
+
+/**
+ * Raise the timer force action flag and notify the dedicated timer EMT.
+ *
+ * @param   pVM         The cross context VM structure.
+ */
+DECLINLINE(void) tmScheduleNotify(PVMCC pVM)
+{
+    VMCPUID idCpu = pVM->tm.s.idTimerCpu;
+    AssertReturnVoid(idCpu < pVM->cCpus);
+    PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, idCpu);
+
+    if (!VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+    {
+        Log5(("TMAll(%u): FF: 0 -> 1\n", __LINE__));
+        VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+#ifdef IN_RING3
+        VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+        STAM_COUNTER_INC(&pVM->tm.s.StatScheduleSetFF);
+    }
+}
+
+
+/**
+ * Schedule the queue which was changed.
+ */
+DECLINLINE(void) tmSchedule(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer)
+{
+    int rc = PDMCritSectTryEnter(pVM, &pQueue->TimerLock);
+    if (RT_SUCCESS_NP(rc))
+    {
+        STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a);
+        Log3(("tmSchedule: tmTimerQueueSchedule\n"));
+        tmTimerQueueSchedule(pVM, pQueueCC, pQueue);
+#ifdef VBOX_STRICT
+        tmTimerQueuesSanityChecks(pVM, "tmSchedule");
+#endif
+        STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatScheduleOne), a);
+        PDMCritSectLeave(pVM, &pQueue->TimerLock);
+        return;
+    }
+
+    TMTIMERSTATE enmState = pTimer->enmState;
+    if (TMTIMERSTATE_IS_PENDING_SCHEDULING(enmState))
+        tmScheduleNotify(pVM);
+}
+
+
+/**
+ * Try change the state to enmStateNew from enmStateOld
+ * and link the timer into the scheduling queue.
+ *
+ * @returns Success indicator.
+ * @param   pTimer          Timer in question.
+ * @param   enmStateNew     The new timer state.
+ * @param   enmStateOld     The old timer state.
+ */
+DECLINLINE(bool) tmTimerTry(PTMTIMER pTimer, TMTIMERSTATE enmStateNew, TMTIMERSTATE enmStateOld)
+{
+    /*
+     * Attempt state change.
+     */
+    bool fRc;
+    TM_TRY_SET_STATE(pTimer, enmStateNew, enmStateOld, fRc);
+    return fRc;
+}
+
+
+/**
+ * Links the timer onto the scheduling queue.
+ *
+ * @param   pQueueCC    The current context queue (same as @a pQueue for
+ *                      ring-3).
+ * @param   pQueue      The shared queue data.
+ * @param   pTimer      The timer.
+ *
+ * @todo    FIXME: Look into potential race with the thread running the queues
+ *          and stuff.
+ */
+DECLINLINE(void) tmTimerLinkSchedule(PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer)
+{
+    Assert(pTimer->idxScheduleNext == UINT32_MAX);
+    const uint32_t idxHeadNew = pTimer - &pQueueCC->paTimers[0];
+    AssertReturnVoid(idxHeadNew < pQueueCC->cTimersAlloc);
+
+    uint32_t idxHead;
+    do
+    {
+        idxHead = pQueue->idxSchedule;
+        Assert(idxHead == UINT32_MAX || idxHead < pQueueCC->cTimersAlloc);
+        pTimer->idxScheduleNext = idxHead;
+    } while (!ASMAtomicCmpXchgU32(&pQueue->idxSchedule, idxHeadNew, idxHead));
+}
+
+
+/**
+ * Try change the state to enmStateNew from enmStateOld
+ * and link the timer into the scheduling queue.
+ *
+ * @returns Success indicator.
+ * @param   pQueueCC    The current context queue (same as @a pQueue for
+ *                      ring-3).
+ * @param   pQueue      The shared queue data.
+ * @param   pTimer      Timer in question.
+ * @param   enmStateNew The new timer state.
+ * @param   enmStateOld The old timer state.
+ */
+DECLINLINE(bool) tmTimerTryWithLink(PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer,
+                                    TMTIMERSTATE enmStateNew, TMTIMERSTATE enmStateOld)
+{
+    if (tmTimerTry(pTimer, enmStateNew, enmStateOld))
+    {
+        tmTimerLinkSchedule(pQueueCC, pQueue, pTimer);
+        return true;
+    }
+    return false;
+}
+
+
+/**
+ * Links a timer into the active list of a timer queue.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pQueueCC        The current context queue (same as @a pQueue for
+ *                          ring-3).
+ * @param   pQueue          The shared queue data.
+ * @param   pTimer          The timer.
+ * @param   u64Expire       The timer expiration time.
+ *
+ * @remarks Called while owning the relevant queue lock.
+ */
+DECL_FORCE_INLINE(void) tmTimerQueueLinkActive(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue,
+                                               PTMTIMER pTimer, uint64_t u64Expire)
+{
+    Assert(pTimer->idxNext == UINT32_MAX);
+    Assert(pTimer->idxPrev == UINT32_MAX);
+    Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE || pQueue->enmClock != TMCLOCK_VIRTUAL_SYNC); /* (active is not a stable state) */
+    RT_NOREF(pVM);
+
+    PTMTIMER pCur = tmTimerQueueGetHead(pQueueCC, pQueue);
+    if (pCur)
+    {
+        for (;; pCur = tmTimerGetNext(pQueueCC, pCur))
+        {
+            if (pCur->u64Expire > u64Expire)
+            {
+                const PTMTIMER pPrev = tmTimerGetPrev(pQueueCC, pCur);
+                tmTimerSetNext(pQueueCC, pTimer, pCur);
+                tmTimerSetPrev(pQueueCC, pTimer, pPrev);
+                if (pPrev)
+                    tmTimerSetNext(pQueueCC, pPrev, pTimer);
+                else
+                {
+                    tmTimerQueueSetHead(pQueueCC, pQueue, pTimer);
+                    ASMAtomicWriteU64(&pQueue->u64Expire, u64Expire);
+                    DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerQueueLinkActive head", pTimer->szName);
+                }
+                tmTimerSetPrev(pQueueCC, pCur, pTimer);
+                return;
+            }
+            if (pCur->idxNext == UINT32_MAX)
+            {
+                tmTimerSetNext(pQueueCC, pCur, pTimer);
+                tmTimerSetPrev(pQueueCC, pTimer, pCur);
+                DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerQueueLinkActive tail", pTimer->szName);
+                return;
+            }
+        }
+    }
+    else
+    {
+        tmTimerQueueSetHead(pQueueCC, pQueue, pTimer);
+        ASMAtomicWriteU64(&pQueue->u64Expire, u64Expire);
+        DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerQueueLinkActive empty", pTimer->szName);
+    }
+}
+
+
+
+/**
+ * Schedules the given timer on the given queue.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pQueueCC    The current context queue (same as @a pQueue for
+ *                      ring-3).
+ * @param   pQueue      The shared queue data.
+ * @param   pTimer      The timer that needs scheduling.
+ *
+ * @remarks Called while owning the lock.
+ */
+DECLINLINE(void) tmTimerQueueScheduleOne(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue, PTMTIMER pTimer)
+{
+    Assert(pQueue->enmClock != TMCLOCK_VIRTUAL_SYNC);
+    RT_NOREF(pVM);
+
+    /*
+     * Processing.
+     */
+    unsigned cRetries = 2;
+    do
+    {
+        TMTIMERSTATE enmState = pTimer->enmState;
+        switch (enmState)
+        {
+            /*
+             * Reschedule timer (in the active list).
+             */
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+                if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_PENDING_SCHEDULE, TMTIMERSTATE_PENDING_RESCHEDULE)))
+                    break; /* retry */
+                tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer);
+                RT_FALL_THRU();
+
+            /*
+             * Schedule timer (insert into the active list).
+             */
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+                Assert(pTimer->idxNext == UINT32_MAX); Assert(pTimer->idxPrev == UINT32_MAX);
+                if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_ACTIVE, TMTIMERSTATE_PENDING_SCHEDULE)))
+                    break; /* retry */
+                tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, pTimer->u64Expire);
+                return;
+
+            /*
+             * Stop the timer in active list.
+             */
+            case TMTIMERSTATE_PENDING_STOP:
+                if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_PENDING_STOP_SCHEDULE, TMTIMERSTATE_PENDING_STOP)))
+                    break; /* retry */
+                tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer);
+                RT_FALL_THRU();
+
+            /*
+             * Stop the timer (not on the active list).
+             */
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                Assert(pTimer->idxNext == UINT32_MAX); Assert(pTimer->idxPrev == UINT32_MAX);
+                if (RT_UNLIKELY(!tmTimerTry(pTimer, TMTIMERSTATE_STOPPED, TMTIMERSTATE_PENDING_STOP_SCHEDULE)))
+                    break;
+                return;
+
+            /*
+             * The timer is pending destruction by TMR3TimerDestroy, our caller.
+             * Nothing to do here.
+             */
+            case TMTIMERSTATE_DESTROY:
+                break;
+
+            /*
+             * Postpone these until they get into the right state.
+             */
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+                tmTimerLinkSchedule(pQueueCC, pQueue, pTimer);
+                STAM_COUNTER_INC(&pVM->tm.s.CTX_SUFF_Z(StatPostponed));
+                return;
+
+            /*
+             * None of these can be in the schedule.
+             */
+            case TMTIMERSTATE_FREE:
+            case TMTIMERSTATE_STOPPED:
+            case TMTIMERSTATE_ACTIVE:
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+            default:
+                AssertMsgFailed(("Timer (%p) in the scheduling list has an invalid state %s (%d)!",
+                                 pTimer, tmTimerState(pTimer->enmState), pTimer->enmState));
+                return;
+        }
+    } while (cRetries-- > 0);
+}
+
+
+/**
+ * Schedules the specified timer queue.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pQueueCC        The current context queue (same as @a pQueue for
+ *                          ring-3) data of the queue to schedule.
+ * @param   pQueue          The shared queue data of the queue to schedule.
+ *
+ * @remarks Called while owning the lock.
+ */
+void tmTimerQueueSchedule(PVMCC pVM, PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue)
+{
+    Assert(PDMCritSectIsOwner(pVM, &pQueue->TimerLock));
+
+    /*
+     * Dequeue the scheduling list and iterate it.
+     */
+    uint32_t idxNext = ASMAtomicXchgU32(&pQueue->idxSchedule, UINT32_MAX);
+    Log2(("tmTimerQueueSchedule: pQueue=%p:{.enmClock=%d, idxNext=%RI32, .u64Expired=%'RU64}\n", pQueue, pQueue->enmClock, idxNext, pQueue->u64Expire));
+    while (idxNext != UINT32_MAX)
+    {
+        AssertBreak(idxNext < pQueueCC->cTimersAlloc);
+
+        /*
+         * Unlink the head timer and take down the index of the next one.
+         */
+        PTMTIMER pTimer = &pQueueCC->paTimers[idxNext];
+        idxNext = pTimer->idxScheduleNext;
+        pTimer->idxScheduleNext = UINT32_MAX;
+
+        /*
+         * Do the scheduling.
+         */
+        Log2(("tmTimerQueueSchedule: %p:{.enmState=%s, .enmClock=%d, .enmType=%d, .szName=%s}\n",
+              pTimer, tmTimerState(pTimer->enmState), pQueue->enmClock, pTimer->enmType, pTimer->szName));
+        tmTimerQueueScheduleOne(pVM, pQueueCC, pQueue, pTimer);
+        Log2(("tmTimerQueueSchedule: %p: new %s\n", pTimer, tmTimerState(pTimer->enmState)));
+    }
+    Log2(("tmTimerQueueSchedule: u64Expired=%'RU64\n", pQueue->u64Expire));
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * Checks that the timer queues are sane.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pszWhere    Caller location clue.
+ */
+void tmTimerQueuesSanityChecks(PVMCC pVM, const char *pszWhere)
+{
+    for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++)
+    {
+        PTMTIMERQUEUE const   pQueue   = &pVM->tm.s.aTimerQueues[idxQueue];
+        PTMTIMERQUEUECC const pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, idxQueue, pQueue);
+        Assert(pQueue->enmClock == (TMCLOCK)idxQueue);
+
+        int rc = PDMCritSectTryEnter(pVM, &pQueue->TimerLock);
+        if (RT_SUCCESS(rc))
+        {
+            if (   pQueue->enmClock != TMCLOCK_VIRTUAL_SYNC
+                || PDMCritSectTryEnter(pVM, &pVM->tm.s.VirtualSyncLock) == VINF_SUCCESS)
+            {
+                /* Check the linking of the active lists. */
+                PTMTIMER pPrev = NULL;
+                for (PTMTIMER pCur = tmTimerQueueGetHead(pQueueCC, pQueue);
+                     pCur;
+                     pPrev = pCur, pCur = tmTimerGetNext(pQueueCC, pCur))
+                {
+                    AssertMsg(tmTimerGetPrev(pQueueCC, pCur) == pPrev, ("%s: %p != %p\n", pszWhere, tmTimerGetPrev(pQueueCC, pCur), pPrev));
+                    TMTIMERSTATE enmState = pCur->enmState;
+                    switch (enmState)
+                    {
+                        case TMTIMERSTATE_ACTIVE:
+                            AssertMsg(   pCur->idxScheduleNext == UINT32_MAX
+                                      || pCur->enmState != TMTIMERSTATE_ACTIVE,
+                                      ("%s: %RI32\n", pszWhere, pCur->idxScheduleNext));
+                            break;
+                        case TMTIMERSTATE_PENDING_STOP:
+                        case TMTIMERSTATE_PENDING_RESCHEDULE:
+                        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+                            break;
+                        default:
+                            AssertMsgFailed(("%s: Invalid state enmState=%d %s\n", pszWhere, enmState, tmTimerState(enmState)));
+                            break;
+                    }
+                }
+
+# ifdef IN_RING3
+                /* Go thru all the timers and check that the active ones all are in the active lists. */
+                uint32_t idxTimer = pQueue->cTimersAlloc;
+                uint32_t cFree    = 0;
+                while (idxTimer-- > 0)
+                {
+                    PTMTIMER const     pTimer   = &pQueue->paTimers[idxTimer];
+                    TMTIMERSTATE const enmState = pTimer->enmState;
+                    switch (enmState)
+                    {
+                        case TMTIMERSTATE_FREE:
+                            cFree++;
+                            break;
+
+                        case TMTIMERSTATE_ACTIVE:
+                        case TMTIMERSTATE_PENDING_STOP:
+                        case TMTIMERSTATE_PENDING_RESCHEDULE:
+                        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+                        {
+                            PTMTIMERR3 pCurAct = tmTimerQueueGetHead(pQueueCC, pQueue);
+                            Assert(pTimer->idxPrev != UINT32_MAX || pTimer == pCurAct);
+                            while (pCurAct && pCurAct != pTimer)
+                                pCurAct = tmTimerGetNext(pQueueCC, pCurAct);
+                            Assert(pCurAct == pTimer);
+                            break;
+                        }
+
+                        case TMTIMERSTATE_PENDING_SCHEDULE:
+                        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                        case TMTIMERSTATE_STOPPED:
+                        case TMTIMERSTATE_EXPIRED_DELIVER:
+                        {
+                            Assert(pTimer->idxNext == UINT32_MAX);
+                            Assert(pTimer->idxPrev == UINT32_MAX);
+                            for (PTMTIMERR3 pCurAct = tmTimerQueueGetHead(pQueueCC, pQueue);
+                                 pCurAct;
+                                 pCurAct = tmTimerGetNext(pQueueCC, pCurAct))
+                            {
+                                Assert(pCurAct != pTimer);
+                                Assert(tmTimerGetNext(pQueueCC, pCurAct) != pTimer);
+                                Assert(tmTimerGetPrev(pQueueCC, pCurAct) != pTimer);
+                            }
+                            break;
+                        }
+
+                        /* ignore */
+                        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+                            break;
+
+                        case TMTIMERSTATE_INVALID:
+                            Assert(idxTimer == 0);
+                            break;
+
+                        /* shouldn't get here! */
+                        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+                        case TMTIMERSTATE_DESTROY:
+                        default:
+                            AssertMsgFailed(("Invalid state enmState=%d %s\n", enmState, tmTimerState(enmState)));
+                            break;
+                    }
+
+                    /* Check the handle value. */
+                    if (enmState > TMTIMERSTATE_INVALID && enmState < TMTIMERSTATE_DESTROY)
+                    {
+                        Assert((pTimer->hSelf & TMTIMERHANDLE_TIMER_IDX_MASK) == idxTimer);
+                        Assert(((pTimer->hSelf >> TMTIMERHANDLE_QUEUE_IDX_SHIFT) & TMTIMERHANDLE_QUEUE_IDX_SMASK) == idxQueue);
+                    }
+                }
+                Assert(cFree == pQueue->cTimersFree);
+# endif /* IN_RING3 */
+
+                if (pQueue->enmClock == TMCLOCK_VIRTUAL_SYNC)
+                    PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock);
+            }
+            PDMCritSectLeave(pVM, &pQueue->TimerLock);
+        }
+    }
+}
+#endif /* !VBOX_STRICT */
+
+#ifdef VBOX_HIGH_RES_TIMERS_HACK
+
+/**
+ * Worker for tmTimerPollInternal that handles misses when the dedicated timer
+ * EMT is polling.
+ *
+ * @returns See tmTimerPollInternal.
+ * @param   pVM                 The cross context VM structure.
+ * @param   u64Now              Current virtual clock timestamp.
+ * @param   u64Delta            The delta to the next even in ticks of the
+ *                              virtual clock.
+ * @param   pu64Delta           Where to return the delta.
+ */
+DECLINLINE(uint64_t) tmTimerPollReturnMiss(PVM pVM, uint64_t u64Now, uint64_t u64Delta, uint64_t *pu64Delta)
+{
+    Assert(!(u64Delta & RT_BIT_64(63)));
+
+    if (!pVM->tm.s.fVirtualWarpDrive)
+    {
+        *pu64Delta = u64Delta;
+        return u64Delta + u64Now + pVM->tm.s.u64VirtualOffset;
+    }
+
+    /*
+     * Warp drive adjustments - this is the reverse of what tmVirtualGetRaw is doing.
+     */
+    uint64_t const u64Start = pVM->tm.s.u64VirtualWarpDriveStart;
+    uint32_t const u32Pct   = pVM->tm.s.u32VirtualWarpDrivePercentage;
+
+    uint64_t u64GipTime = u64Delta + u64Now + pVM->tm.s.u64VirtualOffset;
+    u64GipTime -= u64Start; /* the start is GIP time. */
+    if (u64GipTime >= u64Delta)
+    {
+        ASMMultU64ByU32DivByU32(u64GipTime, 100, u32Pct);
+        ASMMultU64ByU32DivByU32(u64Delta, 100, u32Pct);
+    }
+    else
+    {
+        u64Delta -= u64GipTime;
+        ASMMultU64ByU32DivByU32(u64GipTime, 100, u32Pct);
+        u64Delta += u64GipTime;
+    }
+    *pu64Delta = u64Delta;
+    u64GipTime += u64Start;
+    return u64GipTime;
+}
+
+
+/**
+ * Worker for tmTimerPollInternal dealing with returns on virtual CPUs other
+ * than the one dedicated to timer work.
+ *
+ * @returns See tmTimerPollInternal.
+ * @param   pVM                 The cross context VM structure.
+ * @param   u64Now              Current virtual clock timestamp.
+ * @param   pu64Delta           Where to return the delta.
+ */
+DECL_FORCE_INLINE(uint64_t) tmTimerPollReturnOtherCpu(PVM pVM, uint64_t u64Now, uint64_t *pu64Delta)
+{
+    static const uint64_t s_u64OtherRet = 500000000; /* 500 ms for non-timer EMTs. */
+    *pu64Delta = s_u64OtherRet;
+    return u64Now + pVM->tm.s.u64VirtualOffset + s_u64OtherRet;
+}
+
+
+/**
+ * Worker for tmTimerPollInternal.
+ *
+ * @returns See tmTimerPollInternal.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pVCpuDst    The cross context virtual CPU structure of the dedicated
+ *                      timer EMT.
+ * @param   u64Now      Current virtual clock timestamp.
+ * @param   pu64Delta   Where to return the delta.
+ * @param   pCounter    The statistics counter to update.
+ */
+DECL_FORCE_INLINE(uint64_t) tmTimerPollReturnHit(PVM pVM, PVMCPU pVCpu, PVMCPU pVCpuDst, uint64_t u64Now,
+                                                 uint64_t *pu64Delta, PSTAMCOUNTER pCounter)
+{
+    STAM_COUNTER_INC(pCounter); NOREF(pCounter);
+    if (pVCpuDst != pVCpu)
+        return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta);
+    *pu64Delta = 0;
+    return 0;
+}
+
+
+/**
+ * Common worker for TMTimerPollGIP and TMTimerPoll.
+ *
+ * This function is called before FFs are checked in the inner execution EM loops.
+ *
+ * @returns The GIP timestamp of the next event.
+ *          0 if the next event has already expired.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pu64Delta   Where to store the delta.
+ *
+ * @thread  The emulation thread.
+ *
+ * @remarks GIP uses ns ticks.
+ */
+DECL_FORCE_INLINE(uint64_t) tmTimerPollInternal(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta)
+{
+    VMCPUID idCpu = pVM->tm.s.idTimerCpu;
+    AssertReturn(idCpu < pVM->cCpus, 0);
+    PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, idCpu);
+
+    const uint64_t u64Now = TMVirtualGetNoCheck(pVM);
+    STAM_COUNTER_INC(&pVM->tm.s.StatPoll);
+
+    /*
+     * Return straight away if the timer FF is already set ...
+     */
+    if (VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+        return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollAlreadySet);
+
+    /*
+     * ... or if timers are being run.
+     */
+    if (ASMAtomicReadBool(&pVM->tm.s.fRunningQueues))
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatPollRunning);
+        return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta);
+    }
+
+    /*
+     * Check for TMCLOCK_VIRTUAL expiration.
+     */
+    const uint64_t  u64Expire1 = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].u64Expire);
+    const int64_t   i64Delta1  = u64Expire1 - u64Now;
+    if (i64Delta1 <= 0)
+    {
+        if (!VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+        {
+            Log5(("TMAll(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+            VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+        }
+        LogFlow(("TMTimerPoll: expire1=%'RU64 <= now=%'RU64\n", u64Expire1, u64Now));
+        return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtual);
+    }
+
+    /*
+     * Check for TMCLOCK_VIRTUAL_SYNC expiration.
+     * This isn't quite as straight forward if in a catch-up, not only do
+     * we have to adjust the 'now' but when have to adjust the delta as well.
+     */
+
+    /*
+     * Optimistic lockless approach.
+     */
+    uint64_t u64VirtualSyncNow;
+    uint64_t u64Expire2 = ASMAtomicUoReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+    if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking))
+    {
+        if (!ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+        {
+            u64VirtualSyncNow = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync);
+            if (RT_LIKELY(   ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)
+                          && !ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp)
+                          && u64VirtualSyncNow == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync)
+                          && u64Expire2 == ASMAtomicUoReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire)))
+            {
+                u64VirtualSyncNow = u64Now - u64VirtualSyncNow;
+                int64_t i64Delta2 = u64Expire2 - u64VirtualSyncNow;
+                if (i64Delta2 > 0)
+                {
+                    STAM_COUNTER_INC(&pVM->tm.s.StatPollSimple);
+                    STAM_COUNTER_INC(&pVM->tm.s.StatPollMiss);
+
+                    if (pVCpu == pVCpuDst)
+                        return tmTimerPollReturnMiss(pVM, u64Now, RT_MIN(i64Delta1, i64Delta2), pu64Delta);
+                    return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta);
+                }
+
+                if (    !pVM->tm.s.fRunningQueues
+                    &&  !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+                {
+                    Log5(("TMAll(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+                    VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+                }
+
+                STAM_COUNTER_INC(&pVM->tm.s.StatPollSimple);
+                LogFlow(("TMTimerPoll: expire2=%'RU64 <= now=%'RU64\n", u64Expire2, u64Now));
+                return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync);
+            }
+        }
+    }
+    else
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatPollSimple);
+        LogFlow(("TMTimerPoll: stopped\n"));
+        return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync);
+    }
+
+    /*
+     * Complicated lockless approach.
+     */
+    uint64_t    off;
+    uint32_t    u32Pct = 0;
+    bool        fCatchUp;
+    int         cOuterTries = 42;
+    for (;; cOuterTries--)
+    {
+        fCatchUp   = ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp);
+        off        = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync);
+        u64Expire2 = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+        if (fCatchUp)
+        {
+            /* No changes allowed, try get a consistent set of parameters. */
+            uint64_t const u64Prev    = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev);
+            uint64_t const offGivenUp = ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp);
+            u32Pct                    = ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage);
+            if (    (   u64Prev    == ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev)
+                     && offGivenUp == ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp)
+                     && u32Pct     == ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage)
+                     && off        == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync)
+                     && u64Expire2 == ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire)
+                     && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)
+                     && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking))
+                ||  cOuterTries <= 0)
+            {
+                uint64_t u64Delta = u64Now - u64Prev;
+                if (RT_LIKELY(!(u64Delta >> 32)))
+                {
+                    uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, u32Pct, 100);
+                    if (off > u64Sub + offGivenUp)
+                        off -= u64Sub;
+                    else /* we've completely caught up. */
+                        off = offGivenUp;
+                }
+                else
+                    /* More than 4 seconds since last time (or negative), ignore it. */
+                    Log(("TMVirtualGetSync: u64Delta=%RX64 (NoLock)\n", u64Delta));
+
+                /* Check that we're still running and in catch up. */
+                if (    ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)
+                    &&  ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+                    break;
+            }
+        }
+        else if (   off        == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync)
+                 && u64Expire2 == ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire)
+                 && !ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp)
+                 && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking))
+            break; /* Got an consistent offset */
+
+        /* Repeat the initial checks before iterating. */
+        if (VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+            return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollAlreadySet);
+        if (ASMAtomicUoReadBool(&pVM->tm.s.fRunningQueues))
+        {
+            STAM_COUNTER_INC(&pVM->tm.s.StatPollRunning);
+            return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta);
+        }
+        if (!ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking))
+        {
+            LogFlow(("TMTimerPoll: stopped\n"));
+            return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync);
+        }
+        if (cOuterTries <= 0)
+            break; /* that's enough */
+    }
+    if (cOuterTries <= 0)
+        STAM_COUNTER_INC(&pVM->tm.s.StatPollELoop);
+    u64VirtualSyncNow = u64Now - off;
+
+    /* Calc delta and see if we've got a virtual sync hit. */
+    int64_t i64Delta2 = u64Expire2 - u64VirtualSyncNow;
+    if (i64Delta2 <= 0)
+    {
+        if (    !pVM->tm.s.fRunningQueues
+            &&  !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+        {
+            Log5(("TMAll(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+            VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+        }
+        STAM_COUNTER_INC(&pVM->tm.s.StatPollVirtualSync);
+        LogFlow(("TMTimerPoll: expire2=%'RU64 <= now=%'RU64\n", u64Expire2, u64Now));
+        return tmTimerPollReturnHit(pVM, pVCpu, pVCpuDst, u64Now, pu64Delta, &pVM->tm.s.StatPollVirtualSync);
+    }
+
+    /*
+     * Return the time left to the next event.
+     */
+    STAM_COUNTER_INC(&pVM->tm.s.StatPollMiss);
+    if (pVCpu == pVCpuDst)
+    {
+        if (fCatchUp)
+            i64Delta2 = ASMMultU64ByU32DivByU32(i64Delta2, 100, u32Pct + 100);
+        return tmTimerPollReturnMiss(pVM, u64Now, RT_MIN(i64Delta1, i64Delta2), pu64Delta);
+    }
+    return tmTimerPollReturnOtherCpu(pVM, u64Now, pu64Delta);
+}
+
+
+/**
+ * Set FF if we've passed the next virtual event.
+ *
+ * This function is called before FFs are checked in the inner execution EM loops.
+ *
+ * @returns true if timers are pending, false if not.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @thread  The emulation thread.
+ */
+VMMDECL(bool) TMTimerPollBool(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    uint64_t off = 0;
+    tmTimerPollInternal(pVM, pVCpu, &off);
+    return off == 0;
+}
+
+
+/**
+ * Set FF if we've passed the next virtual event.
+ *
+ * This function is called before FFs are checked in the inner execution EM loops.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @thread  The emulation thread.
+ */
+VMM_INT_DECL(void) TMTimerPollVoid(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    uint64_t off;
+    tmTimerPollInternal(pVM, pVCpu, &off);
+}
+
+
+/**
+ * Set FF if we've passed the next virtual event.
+ *
+ * This function is called before FFs are checked in the inner execution EM loops.
+ *
+ * @returns The GIP timestamp of the next event.
+ *          0 if the next event has already expired.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ * @param   pu64Delta   Where to store the delta.
+ * @thread  The emulation thread.
+ */
+VMM_INT_DECL(uint64_t) TMTimerPollGIP(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *pu64Delta)
+{
+    return tmTimerPollInternal(pVM, pVCpu, pu64Delta);
+}
+
+#endif /* VBOX_HIGH_RES_TIMERS_HACK */
+
+/**
+ * Locks the timer clock.
+ *
+ * @returns VINF_SUCCESS on success, @a rcBusy if busy, and VERR_NOT_SUPPORTED
+ *          if the clock does not have a lock.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ * @param   rcBusy      What to return in ring-0 and raw-mode context if the
+ *                      lock is busy.  Pass VINF_SUCCESS to acquired the
+ *                      critical section thru a ring-3 call if necessary.
+ *
+ * @remarks Currently only supported on timers using the virtual sync clock.
+ */
+VMMDECL(int) TMTimerLock(PVMCC pVM, TMTIMERHANDLE hTimer, int rcBusy)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    AssertReturn(idxQueue == TMCLOCK_VIRTUAL_SYNC, VERR_NOT_SUPPORTED);
+    return PDMCritSectEnter(pVM, &pVM->tm.s.VirtualSyncLock, rcBusy);
+}
+
+
+/**
+ * Unlocks a timer clock locked by TMTimerLock.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(void) TMTimerUnlock(PVMCC pVM, TMTIMERHANDLE hTimer)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_VOID(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    AssertReturnVoid(idxQueue == TMCLOCK_VIRTUAL_SYNC);
+    PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock);
+}
+
+
+/**
+ * Checks if the current thread owns the timer clock lock.
+ *
+ * @returns @c true if its the owner, @c false if not.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(bool) TMTimerIsLockOwner(PVMCC pVM, TMTIMERHANDLE hTimer)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, false); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    AssertReturn(idxQueue == TMCLOCK_VIRTUAL_SYNC, false);
+    return PDMCritSectIsOwner(pVM, &pVM->tm.s.VirtualSyncLock);
+}
+
+
+/**
+ * Optimized TMTimerSet code path for starting an inactive timer.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pTimer          The timer handle.
+ * @param   u64Expire       The new expire time.
+ * @param   pQueue          Pointer to the shared timer queue data.
+ * @param   idxQueue        The queue index.
+ */
+static int tmTimerSetOptimizedStart(PVMCC pVM, PTMTIMER pTimer, uint64_t u64Expire, PTMTIMERQUEUE pQueue, uint32_t idxQueue)
+{
+    Assert(pTimer->idxPrev == UINT32_MAX);
+    Assert(pTimer->idxNext == UINT32_MAX);
+    Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE);
+
+    /*
+     * Calculate and set the expiration time.
+     */
+    if (idxQueue == TMCLOCK_VIRTUAL_SYNC)
+    {
+        uint64_t u64Last = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSync);
+        AssertMsgStmt(u64Expire >= u64Last,
+                      ("exp=%#llx last=%#llx\n", u64Expire, u64Last),
+                      u64Expire = u64Last);
+    }
+    ASMAtomicWriteU64(&pTimer->u64Expire, u64Expire);
+    Log2(("tmTimerSetOptimizedStart: %p:{.pszDesc='%s', .u64Expire=%'RU64}\n", pTimer, pTimer->szName, u64Expire));
+
+    /*
+     * Link the timer into the active list.
+     */
+    tmTimerQueueLinkActive(pVM, TM_GET_TIMER_QUEUE_CC(pVM, idxQueue, pQueue), pQueue, pTimer, u64Expire);
+
+    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetOpt);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * TMTimerSet for the virtual sync timer queue.
+ *
+ * This employs a greatly simplified state machine by always acquiring the
+ * queue lock and bypassing the scheduling list.
+ *
+ * @returns VBox status code
+ * @param   pVM                 The cross context VM structure.
+ * @param   pTimer              The timer handle.
+ * @param   u64Expire           The expiration time.
+ */
+static int tmTimerVirtualSyncSet(PVMCC pVM, PTMTIMER pTimer, uint64_t u64Expire)
+{
+    STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetVs), a);
+    VM_ASSERT_EMT(pVM);
+    TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer);
+    int rc = PDMCritSectEnter(pVM, &pVM->tm.s.VirtualSyncLock, VINF_SUCCESS);
+    AssertRCReturn(rc, rc);
+
+    PTMTIMERQUEUE const     pQueue   = &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC];
+    PTMTIMERQUEUECC const   pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, TMCLOCK_VIRTUAL_SYNC, pQueue);
+    TMTIMERSTATE const      enmState = pTimer->enmState;
+    switch (enmState)
+    {
+        case TMTIMERSTATE_EXPIRED_DELIVER:
+        case TMTIMERSTATE_STOPPED:
+            if (enmState == TMTIMERSTATE_EXPIRED_DELIVER)
+                STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetVsStExpDeliver);
+            else
+                STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetVsStStopped);
+
+            AssertMsg(u64Expire >= pVM->tm.s.u64VirtualSync,
+                      ("%'RU64 < %'RU64 %s\n", u64Expire, pVM->tm.s.u64VirtualSync, pTimer->szName));
+            pTimer->u64Expire = u64Expire;
+            TM_SET_STATE(pTimer, TMTIMERSTATE_ACTIVE);
+            tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_ACTIVE:
+            STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetVsStActive);
+            tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer);
+            pTimer->u64Expire = u64Expire;
+            tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_PENDING_RESCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP:
+        case TMTIMERSTATE_PENDING_SCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_DESTROY:
+        case TMTIMERSTATE_FREE:
+            AssertLogRelMsgFailed(("Invalid timer state %s: %s\n", tmTimerState(enmState), pTimer->szName));
+            rc = VERR_TM_INVALID_STATE;
+            break;
+
+        default:
+            AssertMsgFailed(("Unknown timer state %d: %s\n", enmState, pTimer->szName));
+            rc = VERR_TM_UNKNOWN_STATE;
+            break;
+    }
+
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetVs), a);
+    PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock);
+    return rc;
+}
+
+
+/**
+ * Arm a timer with a (new) expire time.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ * @param   u64Expire   New expire time.
+ */
+VMMDECL(int) TMTimerSet(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t u64Expire)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    STAM_COUNTER_INC(&pTimer->StatSetAbsolute);
+
+    /* Treat virtual sync timers specially. */
+    if (idxQueue == TMCLOCK_VIRTUAL_SYNC)
+        return tmTimerVirtualSyncSet(pVM, pTimer, u64Expire);
+
+    STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+    TMTIMER_ASSERT_CRITSECT(pVM, pTimer);
+
+    DBGFTRACE_U64_TAG2(pVM, u64Expire, "TMTimerSet", pTimer->szName);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Gather optimization info.
+     */
+    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSet);
+    TMTIMERSTATE enmOrgState = pTimer->enmState;
+    switch (enmOrgState)
+    {
+        case TMTIMERSTATE_STOPPED:                  STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStStopped); break;
+        case TMTIMERSTATE_EXPIRED_DELIVER:          STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStExpDeliver); break;
+        case TMTIMERSTATE_ACTIVE:                   STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStActive); break;
+        case TMTIMERSTATE_PENDING_STOP:             STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendStop); break;
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendStopSched); break;
+        case TMTIMERSTATE_PENDING_SCHEDULE:         STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendSched); break;
+        case TMTIMERSTATE_PENDING_RESCHEDULE:       STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStPendResched); break;
+        default:                                    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetStOther); break;
+    }
+#endif
+
+#if 1
+    /*
+     * The most common case is setting the timer again during the callback.
+     * The second most common case is starting a timer at some other time.
+     */
+    TMTIMERSTATE enmState1 = pTimer->enmState;
+    if (    enmState1 == TMTIMERSTATE_EXPIRED_DELIVER
+        ||  (   enmState1 == TMTIMERSTATE_STOPPED
+             && pTimer->pCritSect))
+    {
+        /* Try take the TM lock and check the state again. */
+        int rc = PDMCritSectTryEnter(pVM, &pQueue->TimerLock);
+        if (RT_SUCCESS_NP(rc))
+        {
+            if (RT_LIKELY(tmTimerTry(pTimer, TMTIMERSTATE_ACTIVE, enmState1)))
+            {
+                tmTimerSetOptimizedStart(pVM, pTimer, u64Expire, pQueue, idxQueue);
+                STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+                PDMCritSectLeave(pVM, &pQueue->TimerLock);
+                return VINF_SUCCESS;
+            }
+            PDMCritSectLeave(pVM, &pQueue->TimerLock);
+        }
+    }
+#endif
+
+    /*
+     * Unoptimized code path.
+     */
+    int cRetries = 1000;
+    do
+    {
+        /*
+         * Change to any of the SET_EXPIRE states if valid and then to SCHEDULE or RESCHEDULE.
+         */
+        TMTIMERSTATE enmState = pTimer->enmState;
+        Log2(("TMTimerSet: %p:{.enmState=%s, .pszDesc='%s'} cRetries=%d u64Expire=%'RU64\n",
+              pTimer, tmTimerState(enmState), pTimer->szName, cRetries, u64Expire));
+        switch (enmState)
+        {
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+            case TMTIMERSTATE_STOPPED:
+                if (tmTimerTryWithLink(pQueueCC, pQueue, pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState))
+                {
+                    Assert(pTimer->idxPrev == UINT32_MAX);
+                    Assert(pTimer->idxNext == UINT32_MAX);
+                    pTimer->u64Expire = u64Expire;
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE);
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = u64Expire;
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE);
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+
+            case TMTIMERSTATE_ACTIVE:
+                if (tmTimerTryWithLink(pQueueCC, pQueue, pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = u64Expire;
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE);
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+            case TMTIMERSTATE_PENDING_STOP:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = u64Expire;
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE);
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+#ifdef IN_RING3
+                if (!RTThreadYield())
+                    RTThreadSleep(1);
+#else
+/** @todo call host context and yield after a couple of iterations */
+#endif
+                break;
+
+            /*
+             * Invalid states.
+             */
+            case TMTIMERSTATE_DESTROY:
+            case TMTIMERSTATE_FREE:
+                AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, pTimer->szName));
+                return VERR_TM_INVALID_STATE;
+            default:
+                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName));
+                return VERR_TM_UNKNOWN_STATE;
+        }
+    } while (cRetries-- > 0);
+
+    AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName));
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSet), a);
+    return VERR_TM_TIMER_UNSTABLE_STATE;
+}
+
+
+/**
+ * Return the current time for the specified clock, setting pu64Now if not NULL.
+ *
+ * @returns Current time.
+ * @param   pVM             The cross context VM structure.
+ * @param   enmClock        The clock to query.
+ * @param   pu64Now         Optional pointer where to store the return time
+ */
+DECL_FORCE_INLINE(uint64_t) tmTimerSetRelativeNowWorker(PVMCC pVM, TMCLOCK enmClock, uint64_t *pu64Now)
+{
+    uint64_t u64Now;
+    switch (enmClock)
+    {
+        case TMCLOCK_VIRTUAL_SYNC:
+            u64Now = TMVirtualSyncGet(pVM);
+            break;
+        case TMCLOCK_VIRTUAL:
+            u64Now = TMVirtualGet(pVM);
+            break;
+        case TMCLOCK_REAL:
+            u64Now = TMRealGet(pVM);
+            break;
+        default:
+            AssertFatalMsgFailed(("%d\n", enmClock));
+    }
+
+    if (pu64Now)
+        *pu64Now = u64Now;
+    return u64Now;
+}
+
+
+/**
+ * Optimized TMTimerSetRelative code path.
+ *
+ * @returns VBox status code.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   pTimer          The timer handle.
+ * @param   cTicksToNext    Clock ticks until the next time expiration.
+ * @param   pu64Now         Where to return the current time stamp used.
+ *                          Optional.
+ * @param   pQueueCC        The context specific queue data (same as @a pQueue
+ *                          for ring-3).
+ * @param   pQueue          The shared queue data.
+ */
+static int tmTimerSetRelativeOptimizedStart(PVMCC pVM, PTMTIMER pTimer, uint64_t cTicksToNext, uint64_t *pu64Now,
+                                            PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue)
+{
+    Assert(pTimer->idxPrev == UINT32_MAX);
+    Assert(pTimer->idxNext == UINT32_MAX);
+    Assert(pTimer->enmState == TMTIMERSTATE_ACTIVE);
+
+    /*
+     * Calculate and set the expiration time.
+     */
+    uint64_t const  u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now);
+    pTimer->u64Expire         = u64Expire;
+    Log2(("tmTimerSetRelativeOptimizedStart: %p:{.pszDesc='%s', .u64Expire=%'RU64} cTicksToNext=%'RU64\n", pTimer, pTimer->szName, u64Expire, cTicksToNext));
+
+    /*
+     * Link the timer into the active list.
+     */
+    DBGFTRACE_U64_TAG2(pVM, u64Expire, "tmTimerSetRelativeOptimizedStart", pTimer->szName);
+    tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire);
+
+    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeOpt);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * TMTimerSetRelative for the virtual sync timer queue.
+ *
+ * This employs a greatly simplified state machine by always acquiring the
+ * queue lock and bypassing the scheduling list.
+ *
+ * @returns VBox status code
+ * @param   pVM                 The cross context VM structure.
+ * @param   pTimer              The timer to (re-)arm.
+ * @param   cTicksToNext        Clock ticks until the next time expiration.
+ * @param   pu64Now             Where to return the current time stamp used.
+ *                              Optional.
+ */
+static int tmTimerVirtualSyncSetRelative(PVMCC pVM, PTMTIMER pTimer, uint64_t cTicksToNext, uint64_t *pu64Now)
+{
+    STAM_PROFILE_START(pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelativeVs), a);
+    VM_ASSERT_EMT(pVM);
+    TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer);
+    int rc = PDMCritSectEnter(pVM, &pVM->tm.s.VirtualSyncLock, VINF_SUCCESS);
+    AssertRCReturn(rc, rc);
+
+    /* Calculate the expiration tick. */
+    uint64_t u64Expire = TMVirtualSyncGetNoCheck(pVM);
+    if (pu64Now)
+        *pu64Now = u64Expire;
+    u64Expire += cTicksToNext;
+
+    /* Update the timer. */
+    PTMTIMERQUEUE const     pQueue   = &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC];
+    PTMTIMERQUEUECC const   pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, TMCLOCK_VIRTUAL_SYNC, pQueue);
+    TMTIMERSTATE const      enmState = pTimer->enmState;
+    switch (enmState)
+    {
+        case TMTIMERSTATE_EXPIRED_DELIVER:
+        case TMTIMERSTATE_STOPPED:
+            if (enmState == TMTIMERSTATE_EXPIRED_DELIVER)
+                STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeVsStExpDeliver);
+            else
+                STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeVsStStopped);
+            pTimer->u64Expire = u64Expire;
+            TM_SET_STATE(pTimer, TMTIMERSTATE_ACTIVE);
+            tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_ACTIVE:
+            STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeVsStActive);
+            tmTimerQueueUnlinkActive(pVM, pQueueCC, pQueue, pTimer);
+            pTimer->u64Expire = u64Expire;
+            tmTimerQueueLinkActive(pVM, pQueueCC, pQueue, pTimer, u64Expire);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_PENDING_RESCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP:
+        case TMTIMERSTATE_PENDING_SCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_DESTROY:
+        case TMTIMERSTATE_FREE:
+            AssertLogRelMsgFailed(("Invalid timer state %s: %s\n", tmTimerState(enmState), pTimer->szName));
+            rc = VERR_TM_INVALID_STATE;
+            break;
+
+        default:
+            AssertMsgFailed(("Unknown timer state %d: %s\n", enmState, pTimer->szName));
+            rc = VERR_TM_UNKNOWN_STATE;
+            break;
+    }
+
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelativeVs), a);
+    PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock);
+    return rc;
+}
+
+
+/**
+ * Arm a timer with a expire time relative to the current time.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   pTimer          The timer to arm.
+ * @param   cTicksToNext    Clock ticks until the next time expiration.
+ * @param   pu64Now         Where to return the current time stamp used.
+ *                          Optional.
+ * @param   pQueueCC        The context specific queue data (same as @a pQueue
+ *                          for ring-3).
+ * @param   pQueue          The shared queue data.
+ */
+static int tmTimerSetRelative(PVMCC pVM, PTMTIMER pTimer, uint64_t cTicksToNext, uint64_t *pu64Now,
+                              PTMTIMERQUEUECC pQueueCC, PTMTIMERQUEUE pQueue)
+{
+    STAM_COUNTER_INC(&pTimer->StatSetRelative);
+
+    /* Treat virtual sync timers specially. */
+    if (pQueue->enmClock == TMCLOCK_VIRTUAL_SYNC)
+        return tmTimerVirtualSyncSetRelative(pVM, pTimer, cTicksToNext, pu64Now);
+
+    STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelative), a);
+    TMTIMER_ASSERT_CRITSECT(pVM, pTimer);
+
+    DBGFTRACE_U64_TAG2(pVM, cTicksToNext, "TMTimerSetRelative", pTimer->szName);
+
+#ifdef VBOX_WITH_STATISTICS
+    /*
+     * Gather optimization info.
+     */
+    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelative);
+    TMTIMERSTATE enmOrgState = pTimer->enmState;
+    switch (enmOrgState)
+    {
+        case TMTIMERSTATE_STOPPED:                  STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStStopped); break;
+        case TMTIMERSTATE_EXPIRED_DELIVER:          STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStExpDeliver); break;
+        case TMTIMERSTATE_ACTIVE:                   STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStActive); break;
+        case TMTIMERSTATE_PENDING_STOP:             STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendStop); break;
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendStopSched); break;
+        case TMTIMERSTATE_PENDING_SCHEDULE:         STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendSched); break;
+        case TMTIMERSTATE_PENDING_RESCHEDULE:       STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStPendResched); break;
+        default:                                    STAM_COUNTER_INC(&pVM->tm.s.StatTimerSetRelativeStOther); break;
+    }
+#endif
+
+    /*
+     * Try to take the TM lock and optimize the common cases.
+     *
+     * With the TM lock we can safely make optimizations like immediate
+     * scheduling and we can also be 100% sure that we're not racing the
+     * running of the timer queues. As an additional restraint we require the
+     * timer to have a critical section associated with to be 100% there aren't
+     * concurrent operations on the timer. (This latter isn't necessary any
+     * longer as this isn't supported for any timers, critsect or not.)
+     *
+     * Note! Lock ordering doesn't apply when we only _try_ to
+     *       get the innermost locks.
+     */
+    bool fOwnTMLock = RT_SUCCESS_NP(PDMCritSectTryEnter(pVM, &pQueue->TimerLock));
+#if 1
+    if (    fOwnTMLock
+        &&  pTimer->pCritSect)
+    {
+        TMTIMERSTATE enmState = pTimer->enmState;
+        if (RT_LIKELY(  (   enmState == TMTIMERSTATE_EXPIRED_DELIVER
+                         || enmState == TMTIMERSTATE_STOPPED)
+                      && tmTimerTry(pTimer, TMTIMERSTATE_ACTIVE, enmState)))
+        {
+            tmTimerSetRelativeOptimizedStart(pVM, pTimer, cTicksToNext, pu64Now, pQueueCC, pQueue);
+            STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelative), a);
+            PDMCritSectLeave(pVM, &pQueue->TimerLock);
+            return VINF_SUCCESS;
+        }
+
+        /* Optimize other states when it becomes necessary. */
+    }
+#endif
+
+    /*
+     * Unoptimized path.
+     */
+    int rc;
+    for (int cRetries = 1000; ; cRetries--)
+    {
+        /*
+         * Change to any of the SET_EXPIRE states if valid and then to SCHEDULE or RESCHEDULE.
+         */
+        TMTIMERSTATE enmState = pTimer->enmState;
+        switch (enmState)
+        {
+            case TMTIMERSTATE_STOPPED:
+                if (pQueue->enmClock == TMCLOCK_VIRTUAL_SYNC)
+                {
+                    /** @todo To fix assertion in tmR3TimerQueueRunVirtualSync:
+                     *              Figure a safe way of activating this timer while the queue is
+                     *              being run.
+                     *        (99.9% sure this that the assertion is caused by DevAPIC.cpp
+                     *        re-starting the timer in response to a initial_count write.) */
+                }
+                RT_FALL_THRU();
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+                if (tmTimerTryWithLink(pQueueCC, pQueue, pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState))
+                {
+                    Assert(pTimer->idxPrev == UINT32_MAX);
+                    Assert(pTimer->idxNext == UINT32_MAX);
+                    pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now);
+                    Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [EXP/STOP]\n",
+                          pTimer, tmTimerState(enmState), pTimer->szName, pTimer->u64Expire, cRetries));
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE);
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    rc = VINF_SUCCESS;
+                    break;
+                }
+                rc = VERR_TRY_AGAIN;
+                break;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now);
+                    Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [PEND_SCHED]\n",
+                          pTimer, tmTimerState(enmState), pTimer->szName, pTimer->u64Expire, cRetries));
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_SCHEDULE);
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    rc = VINF_SUCCESS;
+                    break;
+                }
+                rc = VERR_TRY_AGAIN;
+                break;
+
+
+            case TMTIMERSTATE_ACTIVE:
+                if (tmTimerTryWithLink(pQueueCC, pQueue, pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now);
+                    Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [ACTIVE]\n",
+                          pTimer, tmTimerState(enmState), pTimer->szName, pTimer->u64Expire, cRetries));
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE);
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    rc = VINF_SUCCESS;
+                    break;
+                }
+                rc = VERR_TRY_AGAIN;
+                break;
+
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+            case TMTIMERSTATE_PENDING_STOP:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE, enmState))
+                {
+                    pTimer->u64Expire = cTicksToNext + tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now);
+                    Log2(("TMTimerSetRelative: %p:{.enmState=%s, .pszDesc='%s', .u64Expire=%'RU64} cRetries=%d [PEND_RESCH/STOP]\n",
+                          pTimer, tmTimerState(enmState), pTimer->szName, pTimer->u64Expire, cRetries));
+                    TM_SET_STATE(pTimer, TMTIMERSTATE_PENDING_RESCHEDULE);
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    rc = VINF_SUCCESS;
+                    break;
+                }
+                rc = VERR_TRY_AGAIN;
+                break;
+
+
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+#ifdef IN_RING3
+                if (!RTThreadYield())
+                    RTThreadSleep(1);
+#else
+/** @todo call host context and yield after a couple of iterations */
+#endif
+                rc = VERR_TRY_AGAIN;
+                break;
+
+            /*
+             * Invalid states.
+             */
+            case TMTIMERSTATE_DESTROY:
+            case TMTIMERSTATE_FREE:
+                AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, pTimer->szName));
+                rc = VERR_TM_INVALID_STATE;
+                break;
+
+            default:
+                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName));
+                rc = VERR_TM_UNKNOWN_STATE;
+                break;
+        }
+
+        /* switch + loop is tedious to break out of. */
+        if (rc == VINF_SUCCESS)
+            break;
+
+        if (rc != VERR_TRY_AGAIN)
+        {
+            tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now);
+            break;
+        }
+        if (cRetries <= 0)
+        {
+            AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName));
+            rc = VERR_TM_TIMER_UNSTABLE_STATE;
+            tmTimerSetRelativeNowWorker(pVM, pQueue->enmClock, pu64Now);
+            break;
+        }
+
+        /*
+         * Retry to gain locks.
+         */
+        if (!fOwnTMLock)
+            fOwnTMLock = RT_SUCCESS_NP(PDMCritSectTryEnter(pVM, &pQueue->TimerLock));
+
+    } /* for (;;) */
+
+    /*
+     * Clean up and return.
+     */
+    if (fOwnTMLock)
+        PDMCritSectLeave(pVM, &pQueue->TimerLock);
+
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerSetRelative), a);
+    return rc;
+}
+
+
+/**
+ * Arm a timer with a expire time relative to the current time.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   hTimer          Timer handle as returned by one of the create functions.
+ * @param   cTicksToNext    Clock ticks until the next time expiration.
+ * @param   pu64Now         Where to return the current time stamp used.
+ *                          Optional.
+ */
+VMMDECL(int) TMTimerSetRelative(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cTicksToNext, uint64_t *pu64Now)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    return tmTimerSetRelative(pVM, pTimer, cTicksToNext, pu64Now, pQueueCC, pQueue);
+}
+
+
+/**
+ * Drops a hint about the frequency of the timer.
+ *
+ * This is used by TM and the VMM to calculate how often guest execution needs
+ * to be interrupted.  The hint is automatically cleared by TMTimerStop.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ * @param   uHzHint     The frequency hint.  Pass 0 to clear the hint.
+ *
+ * @remarks We're using an integer hertz value here since anything above 1 HZ
+ *          is not going to be any trouble satisfying scheduling wise.  The
+ *          range where it makes sense is >= 100 HZ.
+ */
+VMMDECL(int) TMTimerSetFrequencyHint(PVMCC pVM, TMTIMERHANDLE hTimer, uint32_t uHzHint)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    TMTIMER_ASSERT_CRITSECT(pVM, pTimer);
+
+    uint32_t const uHzOldHint = pTimer->uHzHint;
+    pTimer->uHzHint = uHzHint;
+
+    uint32_t const uMaxHzHint = pQueue->uMaxHzHint;
+    if (   uHzHint    >  uMaxHzHint
+        || uHzOldHint >= uMaxHzHint)
+        ASMAtomicOrU64(&pVM->tm.s.HzHint.u64Combined, RT_BIT_32(idxQueue) | RT_BIT_32(idxQueue + 16));
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * TMTimerStop for the virtual sync timer queue.
+ *
+ * This employs a greatly simplified state machine by always acquiring the
+ * queue lock and bypassing the scheduling list.
+ *
+ * @returns VBox status code
+ * @param   pVM                 The cross context VM structure.
+ * @param   pTimer              The timer handle.
+ */
+static int tmTimerVirtualSyncStop(PVMCC pVM, PTMTIMER pTimer)
+{
+    STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerStopVs), a);
+    VM_ASSERT_EMT(pVM);
+    TMTIMER_ASSERT_SYNC_CRITSECT_ORDER(pVM, pTimer);
+    int rc = PDMCritSectEnter(pVM, &pVM->tm.s.VirtualSyncLock, VINF_SUCCESS);
+    AssertRCReturn(rc, rc);
+
+    /* Reset the HZ hint. */
+    uint32_t uOldHzHint = pTimer->uHzHint;
+    if (uOldHzHint)
+    {
+        if (uOldHzHint >= pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].uMaxHzHint)
+            ASMAtomicOrU64(&pVM->tm.s.HzHint.u64Combined, RT_BIT_32(TMCLOCK_VIRTUAL_SYNC) | RT_BIT_32(TMCLOCK_VIRTUAL_SYNC + 16));
+        pTimer->uHzHint = 0;
+    }
+
+    /* Update the timer state. */
+    TMTIMERSTATE const enmState = pTimer->enmState;
+    switch (enmState)
+    {
+        case TMTIMERSTATE_ACTIVE:
+        {
+            PTMTIMERQUEUE const pQueue = &pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC];
+            tmTimerQueueUnlinkActive(pVM, TM_GET_TIMER_QUEUE_CC(pVM, TMCLOCK_VIRTUAL_SYNC, pQueue), pQueue, pTimer);
+            TM_SET_STATE(pTimer, TMTIMERSTATE_STOPPED);
+            rc = VINF_SUCCESS;
+            break;
+        }
+
+        case TMTIMERSTATE_EXPIRED_DELIVER:
+            TM_SET_STATE(pTimer, TMTIMERSTATE_STOPPED);
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_STOPPED:
+            rc = VINF_SUCCESS;
+            break;
+
+        case TMTIMERSTATE_PENDING_RESCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP:
+        case TMTIMERSTATE_PENDING_SCHEDULE:
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_DESTROY:
+        case TMTIMERSTATE_FREE:
+            AssertLogRelMsgFailed(("Invalid timer state %s: %s\n", tmTimerState(enmState), pTimer->szName));
+            rc = VERR_TM_INVALID_STATE;
+            break;
+
+        default:
+            AssertMsgFailed(("Unknown timer state %d: %s\n", enmState, pTimer->szName));
+            rc = VERR_TM_UNKNOWN_STATE;
+            break;
+    }
+
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStopVs), a);
+    PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock);
+    return rc;
+}
+
+
+/**
+ * Stop the timer.
+ * Use TMR3TimerArm() to "un-stop" the timer.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(int) TMTimerStop(PVMCC pVM, TMTIMERHANDLE hTimer)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    STAM_COUNTER_INC(&pTimer->StatStop);
+
+    /* Treat virtual sync timers specially. */
+    if (idxQueue == TMCLOCK_VIRTUAL_SYNC)
+        return tmTimerVirtualSyncStop(pVM, pTimer);
+
+    STAM_PROFILE_START(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+    TMTIMER_ASSERT_CRITSECT(pVM, pTimer);
+
+    /*
+     * Reset the HZ hint.
+     */
+    uint32_t const uOldHzHint = pTimer->uHzHint;
+    if (uOldHzHint)
+    {
+        if (uOldHzHint >= pQueue->uMaxHzHint)
+            ASMAtomicOrU64(&pVM->tm.s.HzHint.u64Combined, RT_BIT_32(idxQueue) | RT_BIT_32(idxQueue + 16));
+        pTimer->uHzHint = 0;
+    }
+
+    /** @todo see if this function needs optimizing. */
+    int cRetries = 1000;
+    do
+    {
+        /*
+         * Change to any of the SET_EXPIRE states if valid and then to SCHEDULE or RESCHEDULE.
+         */
+        TMTIMERSTATE    enmState = pTimer->enmState;
+        Log2(("TMTimerStop: %p:{.enmState=%s, .pszDesc='%s'} cRetries=%d\n",
+              pTimer, tmTimerState(enmState), pTimer->szName, cRetries));
+        switch (enmState)
+        {
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+                //AssertMsgFailed(("You don't stop an expired timer dude!\n"));
+                return VERR_INVALID_PARAMETER;
+
+            case TMTIMERSTATE_STOPPED:
+            case TMTIMERSTATE_PENDING_STOP:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+                return VINF_SUCCESS;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_STOP_SCHEDULE, enmState))
+                {
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+                if (tmTimerTry(pTimer, TMTIMERSTATE_PENDING_STOP, enmState))
+                {
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case TMTIMERSTATE_ACTIVE:
+                if (tmTimerTryWithLink(pQueueCC, pQueue, pTimer, TMTIMERSTATE_PENDING_STOP, enmState))
+                {
+                    tmSchedule(pVM, pQueueCC, pQueue, pTimer);
+                    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+                    return VINF_SUCCESS;
+                }
+                break;
+
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+#ifdef IN_RING3
+                if (!RTThreadYield())
+                    RTThreadSleep(1);
+#else
+/** @todo call host and yield cpu after a while. */
+#endif
+                break;
+
+            /*
+             * Invalid states.
+             */
+            case TMTIMERSTATE_DESTROY:
+            case TMTIMERSTATE_FREE:
+                AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, pTimer->szName));
+                return VERR_TM_INVALID_STATE;
+            default:
+                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName));
+                return VERR_TM_UNKNOWN_STATE;
+        }
+    } while (cRetries-- > 0);
+
+    AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName));
+    STAM_PROFILE_STOP(&pVM->tm.s.CTX_SUFF_Z(StatTimerStop), a);
+    return VERR_TM_TIMER_UNSTABLE_STATE;
+}
+
+
+/**
+ * Get the current clock time.
+ * Handy for calculating the new expire time.
+ *
+ * @returns Current clock time.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGet(PVMCC pVM, TMTIMERHANDLE hTimer)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    STAM_COUNTER_INC(&pTimer->StatGet);
+
+    uint64_t u64;
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+            u64 = TMVirtualGet(pVM);
+            break;
+        case TMCLOCK_VIRTUAL_SYNC:
+            u64 = TMVirtualSyncGet(pVM);
+            break;
+        case TMCLOCK_REAL:
+            u64 = TMRealGet(pVM);
+            break;
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return UINT64_MAX;
+    }
+    //Log2(("TMTimerGet: returns %'RU64 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+    //      u64, pTimer, tmTimerState(pTimer->enmState), pTimer->szName));
+    return u64;
+}
+
+
+/**
+ * Get the frequency of the timer clock.
+ *
+ * @returns Clock frequency (as Hz of course).
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGetFreq(PVMCC pVM, TMTIMERHANDLE hTimer)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            return TMCLOCK_FREQ_VIRTUAL;
+
+        case TMCLOCK_REAL:
+            return TMCLOCK_FREQ_REAL;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Get the expire time of the timer.
+ * Only valid for active timers.
+ *
+ * @returns Expire time of the timer.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGetExpire(PVMCC pVM, TMTIMERHANDLE hTimer)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, UINT64_MAX); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    TMTIMER_ASSERT_CRITSECT(pVM, pTimer);
+    int cRetries = 1000;
+    do
+    {
+        TMTIMERSTATE enmState = pTimer->enmState;
+        switch (enmState)
+        {
+            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+            case TMTIMERSTATE_EXPIRED_DELIVER:
+            case TMTIMERSTATE_STOPPED:
+            case TMTIMERSTATE_PENDING_STOP:
+            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                Log2(("TMTimerGetExpire: returns ~0 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                      pTimer, tmTimerState(pTimer->enmState), pTimer->szName));
+                return UINT64_MAX;
+
+            case TMTIMERSTATE_ACTIVE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE:
+            case TMTIMERSTATE_PENDING_SCHEDULE:
+                Log2(("TMTimerGetExpire: returns %'RU64 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                      pTimer->u64Expire, pTimer, tmTimerState(pTimer->enmState), pTimer->szName));
+                return pTimer->u64Expire;
+
+            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+#ifdef IN_RING3
+                if (!RTThreadYield())
+                    RTThreadSleep(1);
+#endif
+                break;
+
+            /*
+             * Invalid states.
+             */
+            case TMTIMERSTATE_DESTROY:
+            case TMTIMERSTATE_FREE:
+                AssertMsgFailed(("Invalid timer state %d (%s)\n", enmState, pTimer->szName));
+                Log2(("TMTimerGetExpire: returns ~0 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                      pTimer, tmTimerState(pTimer->enmState), pTimer->szName));
+                return UINT64_MAX;
+            default:
+                AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName));
+                return UINT64_MAX;
+        }
+    } while (cRetries-- > 0);
+
+    AssertMsgFailed(("Failed waiting for stable state. state=%d (%s)\n", pTimer->enmState, pTimer->szName));
+    Log2(("TMTimerGetExpire: returns ~0 (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+          pTimer, tmTimerState(pTimer->enmState), pTimer->szName));
+    return UINT64_MAX;
+}
+
+
+/**
+ * Checks if a timer is active or not.
+ *
+ * @returns True if active.
+ * @returns False if not active.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(bool) TMTimerIsActive(PVMCC pVM, TMTIMERHANDLE hTimer)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, false); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    TMTIMERSTATE enmState = pTimer->enmState;
+    switch (enmState)
+    {
+        case TMTIMERSTATE_STOPPED:
+        case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+        case TMTIMERSTATE_EXPIRED_DELIVER:
+        case TMTIMERSTATE_PENDING_STOP:
+        case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+            Log2(("TMTimerIsActive: returns false (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                  pTimer, tmTimerState(pTimer->enmState), pTimer->szName));
+            return false;
+
+        case TMTIMERSTATE_ACTIVE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE:
+        case TMTIMERSTATE_PENDING_SCHEDULE:
+        case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+        case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+            Log2(("TMTimerIsActive: returns true (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                  pTimer, tmTimerState(pTimer->enmState), pTimer->szName));
+            return true;
+
+        /*
+         * Invalid states.
+         */
+        case TMTIMERSTATE_DESTROY:
+        case TMTIMERSTATE_FREE:
+            AssertMsgFailed(("Invalid timer state %s (%s)\n", tmTimerState(enmState), pTimer->szName));
+            Log2(("TMTimerIsActive: returns false (pTimer=%p:{.enmState=%s, .pszDesc='%s'})\n",
+                  pTimer, tmTimerState(pTimer->enmState), pTimer->szName));
+            return false;
+        default:
+            AssertMsgFailed(("Unknown timer state %d (%s)\n", enmState, pTimer->szName));
+            return false;
+    }
+}
+
+
+/* -=-=-=-=-=-=- Convenience APIs -=-=-=-=-=-=- */
+
+
+/**
+ * Arm a timer with a (new) expire time relative to current time.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   hTimer          Timer handle as returned by one of the create functions.
+ * @param   cMilliesToNext  Number of milliseconds to the next tick.
+ */
+VMMDECL(int) TMTimerSetMillies(PVMCC pVM, TMTIMERHANDLE hTimer, uint32_t cMilliesToNext)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return tmTimerSetRelative(pVM, pTimer, cMilliesToNext * UINT64_C(1000000), NULL, pQueueCC, pQueue);
+
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return tmTimerSetRelative(pVM, pTimer, cMilliesToNext * UINT64_C(1000000), NULL, pQueueCC, pQueue);
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return tmTimerSetRelative(pVM, pTimer, cMilliesToNext, NULL, pQueueCC, pQueue);
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return VERR_TM_TIMER_BAD_CLOCK;
+    }
+}
+
+
+/**
+ * Arm a timer with a (new) expire time relative to current time.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   hTimer          Timer handle as returned by one of the create functions.
+ * @param   cMicrosToNext   Number of microseconds to the next tick.
+ */
+VMMDECL(int) TMTimerSetMicro(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cMicrosToNext)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return tmTimerSetRelative(pVM, pTimer, cMicrosToNext * 1000, NULL, pQueueCC, pQueue);
+
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return tmTimerSetRelative(pVM, pTimer, cMicrosToNext * 1000, NULL, pQueueCC, pQueue);
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return tmTimerSetRelative(pVM, pTimer, cMicrosToNext / 1000, NULL, pQueueCC, pQueue);
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return VERR_TM_TIMER_BAD_CLOCK;
+    }
+}
+
+
+/**
+ * Arm a timer with a (new) expire time relative to current time.
+ *
+ * @returns VBox status code.
+ * @param   pVM             The cross context VM structure.
+ * @param   hTimer          Timer handle as returned by one of the create functions.
+ * @param   cNanosToNext    Number of nanoseconds to the next tick.
+ */
+VMMDECL(int) TMTimerSetNano(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cNanosToNext)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN(pVM, hTimer); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return tmTimerSetRelative(pVM, pTimer, cNanosToNext, NULL, pQueueCC, pQueue);
+
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return tmTimerSetRelative(pVM, pTimer, cNanosToNext, NULL, pQueueCC, pQueue);
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return tmTimerSetRelative(pVM, pTimer, cNanosToNext / 1000000, NULL, pQueueCC, pQueue);
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return VERR_TM_TIMER_BAD_CLOCK;
+    }
+}
+
+
+/**
+ * Get the current clock time as nanoseconds.
+ *
+ * @returns The timer clock as nanoseconds.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGetNano(PVMCC pVM, TMTIMERHANDLE hTimer)
+{
+    return TMTimerToNano(pVM, hTimer, TMTimerGet(pVM, hTimer));
+}
+
+
+/**
+ * Get the current clock time as microseconds.
+ *
+ * @returns The timer clock as microseconds.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGetMicro(PVMCC pVM, TMTIMERHANDLE hTimer)
+{
+    return TMTimerToMicro(pVM, hTimer, TMTimerGet(pVM, hTimer));
+}
+
+
+/**
+ * Get the current clock time as milliseconds.
+ *
+ * @returns The timer clock as milliseconds.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ */
+VMMDECL(uint64_t) TMTimerGetMilli(PVMCC pVM, TMTIMERHANDLE hTimer)
+{
+    return TMTimerToMilli(pVM, hTimer, TMTimerGet(pVM, hTimer));
+}
+
+
+/**
+ * Converts the specified timer clock time to nanoseconds.
+ *
+ * @returns nanoseconds.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ * @param   cTicks      The clock ticks.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMMDECL(uint64_t) TMTimerToNano(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cTicks)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return cTicks;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return cTicks * 1000000;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Converts the specified timer clock time to microseconds.
+ *
+ * @returns microseconds.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ * @param   cTicks      The clock ticks.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMMDECL(uint64_t) TMTimerToMicro(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cTicks)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return cTicks / 1000;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return cTicks * 1000;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Converts the specified timer clock time to milliseconds.
+ *
+ * @returns milliseconds.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ * @param   cTicks      The clock ticks.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMMDECL(uint64_t) TMTimerToMilli(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cTicks)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return cTicks / 1000000;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return cTicks;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Converts the specified nanosecond timestamp to timer clock ticks.
+ *
+ * @returns timer clock ticks.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ * @param   cNanoSecs   The nanosecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMMDECL(uint64_t) TMTimerFromNano(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cNanoSecs)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return cNanoSecs;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return cNanoSecs / 1000000;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Converts the specified microsecond timestamp to timer clock ticks.
+ *
+ * @returns timer clock ticks.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ * @param   cMicroSecs  The microsecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMMDECL(uint64_t) TMTimerFromMicro(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cMicroSecs)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return cMicroSecs * 1000;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return cMicroSecs / 1000;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Converts the specified millisecond timestamp to timer clock ticks.
+ *
+ * @returns timer clock ticks.
+ * @param   pVM         The cross context VM structure.
+ * @param   hTimer      Timer handle as returned by one of the create functions.
+ * @param   cMilliSecs  The millisecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMMDECL(uint64_t) TMTimerFromMilli(PVMCC pVM, TMTIMERHANDLE hTimer, uint64_t cMilliSecs)
+{
+    TMTIMER_HANDLE_TO_VARS_RETURN_EX(pVM, hTimer, 0); /* => pTimer, pQueueCC, pQueue, idxTimer, idxQueue */
+    switch (pQueue->enmClock)
+    {
+        case TMCLOCK_VIRTUAL:
+        case TMCLOCK_VIRTUAL_SYNC:
+            AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+            return cMilliSecs * 1000000;
+
+        case TMCLOCK_REAL:
+            AssertCompile(TMCLOCK_FREQ_REAL == 1000);
+            return cMilliSecs;
+
+        default:
+            AssertMsgFailed(("Invalid enmClock=%d\n", pQueue->enmClock));
+            return 0;
+    }
+}
+
+
+/**
+ * Convert state to string.
+ *
+ * @returns Readonly status name.
+ * @param   enmState    State.
+ */
+const char *tmTimerState(TMTIMERSTATE enmState)
+{
+    switch (enmState)
+    {
+#define CASE(num, state) \
+            case TMTIMERSTATE_##state: \
+                AssertCompile(TMTIMERSTATE_##state == (num)); \
+                return #num "-" #state
+        CASE( 0,INVALID);
+        CASE( 1,STOPPED);
+        CASE( 2,ACTIVE);
+        CASE( 3,EXPIRED_GET_UNLINK);
+        CASE( 4,EXPIRED_DELIVER);
+        CASE( 5,PENDING_STOP);
+        CASE( 6,PENDING_STOP_SCHEDULE);
+        CASE( 7,PENDING_SCHEDULE_SET_EXPIRE);
+        CASE( 8,PENDING_SCHEDULE);
+        CASE( 9,PENDING_RESCHEDULE_SET_EXPIRE);
+        CASE(10,PENDING_RESCHEDULE);
+        CASE(11,DESTROY);
+        CASE(12,FREE);
+        default:
+            AssertMsgFailed(("Invalid state enmState=%d\n", enmState));
+            return "Invalid state!";
+#undef CASE
+    }
+}
+
+
+#if defined(IN_RING0) || defined(IN_RING3)
+/**
+ * Copies over old timers and initialized newly allocted ones.
+ *
+ * Helper for TMR0TimerQueueGrow an tmR3TimerQueueGrow.
+ *
+ * @param   paTimers            The new timer allocation.
+ * @param   paOldTimers         The old timers.
+ * @param   cNewTimers          Number of new timers.
+ * @param   cOldTimers          Number of old timers.
+ */
+void tmHCTimerQueueGrowInit(PTMTIMER paTimers, TMTIMER const *paOldTimers, uint32_t cNewTimers, uint32_t cOldTimers)
+{
+    Assert(cOldTimers < cNewTimers);
+
+    /*
+     * Copy over the old info and initialize the new handles.
+     */
+    if (cOldTimers > 0)
+        memcpy(paTimers, paOldTimers, sizeof(TMTIMER) * cOldTimers);
+
+    size_t i = cNewTimers;
+    while (i-- > cOldTimers)
+    {
+        paTimers[i].u64Expire       = UINT64_MAX;
+        paTimers[i].enmType         = TMTIMERTYPE_INVALID;
+        paTimers[i].enmState        = TMTIMERSTATE_FREE;
+        paTimers[i].idxScheduleNext = UINT32_MAX;
+        paTimers[i].idxNext         = UINT32_MAX;
+        paTimers[i].idxPrev         = UINT32_MAX;
+        paTimers[i].hSelf           = NIL_TMTIMERHANDLE;
+    }
+
+    /*
+     * Mark the zero'th entry as allocated but invalid if we just allocated it.
+     */
+    if (cOldTimers == 0)
+    {
+        paTimers[0].enmState = TMTIMERSTATE_INVALID;
+        paTimers[0].szName[0] = 'n';
+        paTimers[0].szName[1] = 'i';
+        paTimers[0].szName[2] = 'l';
+        paTimers[0].szName[3] = '\0';
+    }
+}
+#endif /* IN_RING0 || IN_RING3 */
+
+
+/**
+ * The slow path of tmGetFrequencyHint() where we try to recalculate the value.
+ *
+ * @returns The highest frequency.  0 if no timers care.
+ * @param   pVM             The cross context VM structure.
+ * @param   uOldMaxHzHint   The old global hint.
+ */
+DECL_NO_INLINE(static, uint32_t) tmGetFrequencyHintSlow(PVMCC pVM, uint32_t uOldMaxHzHint)
+{
+    /* Set two bits, though not entirely sure it's needed (too exhaused to think clearly)
+       but it should force other callers thru the slow path while we're recalculating and
+       help us detect changes while we're recalculating. */
+    AssertCompile(RT_ELEMENTS(pVM->tm.s.aTimerQueues) <= 16);
+
+    /*
+     * The "right" highest frequency value isn't so important that we'll block
+     * waiting on the timer semaphores.
+     */
+    uint32_t uMaxHzHint = 0;
+    for (uint32_t idxQueue = 0; idxQueue < RT_ELEMENTS(pVM->tm.s.aTimerQueues); idxQueue++)
+    {
+        PTMTIMERQUEUE pQueue = &pVM->tm.s.aTimerQueues[idxQueue];
+
+        /* Get the max Hz hint for the queue. */
+        uint32_t uMaxHzHintQueue;
+        if (  !(ASMAtomicUoReadU64(&pVM->tm.s.HzHint.u64Combined) & (RT_BIT_32(idxQueue) | RT_BIT_32(idxQueue + 16)))
+            || RT_FAILURE_NP(PDMCritSectTryEnter(pVM, &pQueue->TimerLock)))
+            uMaxHzHintQueue = ASMAtomicReadU32(&pQueue->uMaxHzHint);
+        else
+        {
+            /* Is it still necessary to do updating? */
+            if (ASMAtomicUoReadU64(&pVM->tm.s.HzHint.u64Combined) & (RT_BIT_32(idxQueue) | RT_BIT_32(idxQueue + 16)))
+            {
+                ASMAtomicAndU64(&pVM->tm.s.HzHint.u64Combined, ~RT_BIT_64(idxQueue + 16)); /* clear one flag up front */
+
+                PTMTIMERQUEUECC pQueueCC = TM_GET_TIMER_QUEUE_CC(pVM, idxQueue, pQueue);
+                uMaxHzHintQueue = 0;
+                for (PTMTIMER pCur = tmTimerQueueGetHead(pQueueCC, pQueue);
+                     pCur;
+                     pCur = tmTimerGetNext(pQueueCC, pCur))
+                {
+                    uint32_t uHzHint = ASMAtomicUoReadU32(&pCur->uHzHint);
+                    if (uHzHint > uMaxHzHintQueue)
+                    {
+                        TMTIMERSTATE enmState = pCur->enmState;
+                        switch (enmState)
+                        {
+                            case TMTIMERSTATE_ACTIVE:
+                            case TMTIMERSTATE_EXPIRED_GET_UNLINK:
+                            case TMTIMERSTATE_EXPIRED_DELIVER:
+                            case TMTIMERSTATE_PENDING_SCHEDULE_SET_EXPIRE:
+                            case TMTIMERSTATE_PENDING_SCHEDULE:
+                            case TMTIMERSTATE_PENDING_RESCHEDULE_SET_EXPIRE:
+                            case TMTIMERSTATE_PENDING_RESCHEDULE:
+                                uMaxHzHintQueue = uHzHint;
+                                break;
+
+                            case TMTIMERSTATE_STOPPED:
+                            case TMTIMERSTATE_PENDING_STOP:
+                            case TMTIMERSTATE_PENDING_STOP_SCHEDULE:
+                            case TMTIMERSTATE_DESTROY:
+                            case TMTIMERSTATE_FREE:
+                            case TMTIMERSTATE_INVALID:
+                                break;
+                            /* no default, want gcc warnings when adding more states. */
+                        }
+                    }
+                }
+
+                /* Write the new Hz hint for the quest and clear the other update flag. */
+                ASMAtomicUoWriteU32(&pQueue->uMaxHzHint, uMaxHzHintQueue);
+                ASMAtomicAndU64(&pVM->tm.s.HzHint.u64Combined, ~RT_BIT_64(idxQueue));
+            }
+            else
+                uMaxHzHintQueue = ASMAtomicUoReadU32(&pQueue->uMaxHzHint);
+
+            PDMCritSectLeave(pVM, &pQueue->TimerLock);
+        }
+
+        /* Update the global max Hz hint. */
+        if (uMaxHzHint < uMaxHzHintQueue)
+            uMaxHzHint = uMaxHzHintQueue;
+    }
+
+    /*
+     * Update the frequency hint if no pending frequency changes and we didn't race anyone thru here.
+     */
+    uint64_t u64Actual = RT_MAKE_U64(0 /*no pending updates*/, uOldMaxHzHint);
+    if (ASMAtomicCmpXchgExU64(&pVM->tm.s.HzHint.u64Combined, RT_MAKE_U64(0, uMaxHzHint), u64Actual, &u64Actual))
+        Log(("tmGetFrequencyHintSlow: New value %u Hz\n", uMaxHzHint));
+    else
+        for (uint32_t iTry = 1;; iTry++)
+        {
+            if (RT_LO_U32(u64Actual) != 0)
+                Log(("tmGetFrequencyHintSlow: Outdated value %u Hz (%#x, try %u)\n", uMaxHzHint, RT_LO_U32(u64Actual), iTry));
+            else if (iTry >= 4)
+                Log(("tmGetFrequencyHintSlow: Unable to set %u Hz (try %u)\n", uMaxHzHint, iTry));
+            else if (ASMAtomicCmpXchgExU64(&pVM->tm.s.HzHint.u64Combined, RT_MAKE_U64(0, uMaxHzHint), u64Actual, &u64Actual))
+                Log(("tmGetFrequencyHintSlow: New value %u Hz (try %u)\n", uMaxHzHint, iTry));
+            else
+                continue;
+            break;
+        }
+    return uMaxHzHint;
+}
+
+
+/**
+ * Gets the highest frequency hint for all the important timers.
+ *
+ * @returns The highest frequency.  0 if no timers care.
+ * @param   pVM         The cross context VM structure.
+ */
+DECLINLINE(uint32_t) tmGetFrequencyHint(PVMCC pVM)
+{
+    /*
+     * Query the value, recalculate it if necessary.
+     */
+    uint64_t u64Combined = ASMAtomicReadU64(&pVM->tm.s.HzHint.u64Combined);
+    if (RT_HI_U32(u64Combined) == 0)
+        return RT_LO_U32(u64Combined); /* hopefully somewhat likely */
+    return tmGetFrequencyHintSlow(pVM, RT_LO_U32(u64Combined));
+}
+
+
+/**
+ * Calculates a host timer frequency that would be suitable for the current
+ * timer load.
+ *
+ * This will take the highest timer frequency, adjust for catch-up and warp
+ * driver, and finally add a little fudge factor.  The caller (VMM) will use
+ * the result to adjust the per-cpu preemption timer.
+ *
+ * @returns The highest frequency.  0 if no important timers around.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+VMM_INT_DECL(uint32_t) TMCalcHostTimerFrequency(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    uint32_t uHz = tmGetFrequencyHint(pVM);
+
+    /* Catch up, we have to be more aggressive than the % indicates at the
+       beginning of the effort. */
+    if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+    {
+        uint32_t u32Pct = ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage);
+        if (ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+        {
+            if (u32Pct <= 100)
+                u32Pct = u32Pct * pVM->tm.s.cPctHostHzFudgeFactorCatchUp100 / 100;
+            else if (u32Pct <= 200)
+                u32Pct = u32Pct * pVM->tm.s.cPctHostHzFudgeFactorCatchUp200 / 100;
+            else if (u32Pct <= 400)
+                u32Pct = u32Pct * pVM->tm.s.cPctHostHzFudgeFactorCatchUp400 / 100;
+            uHz *= u32Pct + 100;
+            uHz /= 100;
+        }
+    }
+
+    /* Warp drive. */
+    if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualWarpDrive))
+    {
+        uint32_t u32Pct = ASMAtomicReadU32(&pVM->tm.s.u32VirtualWarpDrivePercentage);
+        if (ASMAtomicReadBool(&pVM->tm.s.fVirtualWarpDrive))
+        {
+            uHz *= u32Pct;
+            uHz /= 100;
+        }
+    }
+
+    /* Fudge factor. */
+    if (pVCpu->idCpu == pVM->tm.s.idTimerCpu)
+        uHz *= pVM->tm.s.cPctHostHzFudgeFactorTimerCpu;
+    else
+        uHz *= pVM->tm.s.cPctHostHzFudgeFactorOtherCpu;
+    uHz /= 100;
+
+    /* Make sure it isn't too high. */
+    if (uHz > pVM->tm.s.cHostHzMax)
+        uHz = pVM->tm.s.cHostHzMax;
+
+    return uHz;
+}
+
+
+/**
+ * Whether the guest virtual clock is ticking.
+ *
+ * @returns true if ticking, false otherwise.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(bool) TMVirtualIsTicking(PVM pVM)
+{
+    return RT_BOOL(pVM->tm.s.cVirtualTicking);
+}
+
diff --git a/src/VBox/VMM/VMMAll/TMAllCpu.cpp b/src/VBox/VMM/VMMAll/TMAllCpu.cpp
new file mode 100644
index 00000000..7b68afe1
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/TMAllCpu.cpp
@@ -0,0 +1,661 @@
+/* $Id: TMAllCpu.cpp $ */
+/** @file
+ * TM - Timeout Manager, CPU Time, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TM
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/gim.h>
+#include <VBox/vmm/dbgf.h>
+#include <VBox/vmm/nem.h>
+#if   defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+# include <iprt/asm-amd64-x86.h> /* for SUPGetCpuHzFromGIP; ASMReadTSC */
+#elif defined(RT_ARCH_ARM64) || defined(RT_ARCH_ARM32)
+# include <iprt/asm-arm.h>
+#endif
+#include "TMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/sup.h>
+
+#include <VBox/param.h>
+#include <VBox/err.h>
+#include <iprt/asm-math.h>
+#include <iprt/assert.h>
+#include <VBox/log.h>
+
+
+
+/**
+ * Converts from virtual time to raw CPU ticks.
+ *
+ * Mainly to have the ASMMultU64ByU32DivByU32 overflow trickery in one place.
+ *
+ * @returns raw CPU ticks.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64VirtualTime  The virtual time to convert.
+ */
+DECLINLINE(uint64_t) tmCpuTickCalcFromVirtual(PVMCC pVM, uint64_t u64VirtualTime)
+{
+    if (pVM->tm.s.cTSCTicksPerSecond <= UINT32_MAX)
+        return ASMMultU64ByU32DivByU32(u64VirtualTime, (uint32_t)pVM->tm.s.cTSCTicksPerSecond, TMCLOCK_FREQ_VIRTUAL);
+    Assert(pVM->tm.s.cTSCTicksPerSecond <= ((uint64_t)UINT32_MAX << 2)); /* <= 15.99 GHz */
+    return ASMMultU64ByU32DivByU32(u64VirtualTime, (uint32_t)(pVM->tm.s.cTSCTicksPerSecond >> 2), TMCLOCK_FREQ_VIRTUAL >> 2);
+}
+
+
+/**
+ * Gets the raw cpu tick from current virtual time.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fCheckTimers    Whether to check timers.
+ */
+DECLINLINE(uint64_t) tmCpuTickGetRawVirtual(PVMCC pVM, bool fCheckTimers)
+{
+    if (fCheckTimers)
+        return tmCpuTickCalcFromVirtual(pVM, TMVirtualSyncGet(pVM));
+    return tmCpuTickCalcFromVirtual(pVM, TMVirtualSyncGetNoCheck(pVM));
+}
+
+
+#ifdef IN_RING3
+/**
+ * Used by tmR3CpuTickParavirtEnable and tmR3CpuTickParavirtDisable.
+ *
+ * @param   pVM     The cross context VM structure.
+ */
+uint64_t tmR3CpuTickGetRawVirtualNoCheck(PVM pVM)
+{
+    return tmCpuTickGetRawVirtual(pVM, false /*fCheckTimers*/);
+}
+#endif
+
+
+/**
+ * Resumes the CPU timestamp counter ticking.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @internal
+ */
+int tmCpuTickResume(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (!pVCpu->tm.s.fTSCTicking)
+    {
+        pVCpu->tm.s.fTSCTicking = true;
+
+        /** @todo Test that pausing and resuming doesn't cause lag! (I.e. that we're
+         *        unpaused before the virtual time and stopped after it. */
+        switch (pVM->tm.s.enmTSCMode)
+        {
+            case TMTSCMODE_REAL_TSC_OFFSET:
+                pVCpu->tm.s.offTSCRawSrc = SUPReadTsc() - pVCpu->tm.s.u64TSC;
+                break;
+            case TMTSCMODE_VIRT_TSC_EMULATED:
+            case TMTSCMODE_DYNAMIC:
+                pVCpu->tm.s.offTSCRawSrc = tmCpuTickGetRawVirtual(pVM, false /* don't check for pending timers */)
+                                         - pVCpu->tm.s.u64TSC;
+                break;
+            case TMTSCMODE_NATIVE_API:
+                pVCpu->tm.s.offTSCRawSrc = 0; /** @todo ?? */
+                /* Looks like this is only used by weird modes and MSR TSC writes.  We cannot support either on NEM/win. */
+                break;
+            default:
+                AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+        }
+        return VINF_SUCCESS;
+    }
+    AssertFailed();
+    return VERR_TM_TSC_ALREADY_TICKING;
+}
+
+
+/**
+ * Resumes the CPU timestamp counter ticking.
+ *
+ * @returns VINF_SUCCESS or VERR_TM_VIRTUAL_TICKING_IPE (asserted).
+ * @param   pVM     The cross context VM structure.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+int tmCpuTickResumeLocked(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (!pVCpu->tm.s.fTSCTicking)
+    {
+        /* TSC must be ticking before calling tmCpuTickGetRawVirtual()! */
+        pVCpu->tm.s.fTSCTicking = true;
+        uint32_t c = ASMAtomicIncU32(&pVM->tm.s.cTSCsTicking);
+        AssertMsgReturn(c <= pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE);
+        if (c == 1)
+        {
+            /* The first VCPU to resume. */
+            uint64_t    offTSCRawSrcOld = pVCpu->tm.s.offTSCRawSrc;
+
+            STAM_COUNTER_INC(&pVM->tm.s.StatTSCResume);
+
+            /* When resuming, use the TSC value of the last stopped VCPU to avoid the TSC going back. */
+            switch (pVM->tm.s.enmTSCMode)
+            {
+                case TMTSCMODE_REAL_TSC_OFFSET:
+                    pVCpu->tm.s.offTSCRawSrc = SUPReadTsc() - pVM->tm.s.u64LastPausedTSC;
+                    break;
+                case TMTSCMODE_VIRT_TSC_EMULATED:
+                case TMTSCMODE_DYNAMIC:
+                    pVCpu->tm.s.offTSCRawSrc = tmCpuTickGetRawVirtual(pVM, false /* don't check for pending timers */)
+                                             - pVM->tm.s.u64LastPausedTSC;
+                    break;
+                case TMTSCMODE_NATIVE_API:
+                {
+                    int rc = NEMHCResumeCpuTickOnAll(pVM, pVCpu, pVM->tm.s.u64LastPausedTSC);
+                    AssertRCReturn(rc, rc);
+                    pVCpu->tm.s.offTSCRawSrc = offTSCRawSrcOld = 0;
+                    break;
+                }
+                default:
+                    AssertFailedReturn(VERR_IPE_NOT_REACHED_DEFAULT_CASE);
+            }
+
+            /* Calculate the offset addendum for other VCPUs to use. */
+            pVM->tm.s.offTSCPause = pVCpu->tm.s.offTSCRawSrc - offTSCRawSrcOld;
+        }
+        else
+        {
+            /* All other VCPUs (if any). */
+            pVCpu->tm.s.offTSCRawSrc += pVM->tm.s.offTSCPause;
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Pauses the CPU timestamp counter ticking.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @internal
+ */
+int tmCpuTickPause(PVMCPUCC pVCpu)
+{
+    if (pVCpu->tm.s.fTSCTicking)
+    {
+        pVCpu->tm.s.u64TSC = TMCpuTickGetNoCheck(pVCpu);
+        pVCpu->tm.s.fTSCTicking = false;
+        return VINF_SUCCESS;
+    }
+    AssertFailed();
+    return VERR_TM_TSC_ALREADY_PAUSED;
+}
+
+
+/**
+ * Pauses the CPU timestamp counter ticking.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @internal
+ */
+int tmCpuTickPauseLocked(PVMCC pVM, PVMCPUCC pVCpu)
+{
+    if (pVCpu->tm.s.fTSCTicking)
+    {
+        pVCpu->tm.s.u64TSC = TMCpuTickGetNoCheck(pVCpu);
+        pVCpu->tm.s.fTSCTicking = false;
+
+        uint32_t c = ASMAtomicDecU32(&pVM->tm.s.cTSCsTicking);
+        AssertMsgReturn(c < pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE);
+        if (c == 0)
+        {
+            /* When the last TSC stops, remember the value. */
+            STAM_COUNTER_INC(&pVM->tm.s.StatTSCPause);
+            pVM->tm.s.u64LastPausedTSC = pVCpu->tm.s.u64TSC;
+        }
+        return VINF_SUCCESS;
+    }
+    AssertFailed();
+    return VERR_TM_TSC_ALREADY_PAUSED;
+}
+
+
+#ifdef IN_RING0 /* Only used in ring-0 at present (AMD-V and VT-x). */
+
+# ifdef VBOX_WITH_STATISTICS
+/**
+ * Record why we refused to use offsetted TSC.
+ *
+ * Used by TMCpuTickCanUseRealTSC() and TMCpuTickGetDeadlineAndTscOffset().
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure of the calling EMT.
+ */
+DECLINLINE(void) tmCpuTickRecordOffsettedTscRefusal(PVM pVM, PVMCPU pVCpu)
+{
+    /* Sample the reason for refusing. */
+    if (pVM->tm.s.enmTSCMode != TMTSCMODE_DYNAMIC)
+       STAM_COUNTER_INC(&pVM->tm.s.StatTSCNotFixed);
+    else if (!pVCpu->tm.s.fTSCTicking)
+       STAM_COUNTER_INC(&pVM->tm.s.StatTSCNotTicking);
+    else if (pVM->tm.s.enmTSCMode != TMTSCMODE_REAL_TSC_OFFSET)
+    {
+        if (pVM->tm.s.fVirtualSyncCatchUp)
+        {
+           if (pVM->tm.s.u32VirtualSyncCatchUpPercentage <= 10)
+               STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupLE010);
+           else if (pVM->tm.s.u32VirtualSyncCatchUpPercentage <= 25)
+               STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupLE025);
+           else if (pVM->tm.s.u32VirtualSyncCatchUpPercentage <= 100)
+               STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupLE100);
+           else
+               STAM_COUNTER_INC(&pVM->tm.s.StatTSCCatchupOther);
+        }
+        else if (!pVM->tm.s.fVirtualSyncTicking)
+           STAM_COUNTER_INC(&pVM->tm.s.StatTSCSyncNotTicking);
+        else if (pVM->tm.s.fVirtualWarpDrive)
+           STAM_COUNTER_INC(&pVM->tm.s.StatTSCWarp);
+    }
+}
+# endif /* VBOX_WITH_STATISTICS */
+
+/**
+ * Checks if AMD-V / VT-x can use an offsetted hardware TSC or not.
+ *
+ * @returns true/false accordingly.
+ * @param   pVM             The cross context VM structure.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   poffRealTsc     The offset against the TSC of the current host CPU,
+ *                          if pfOffsettedTsc is set to true.
+ * @param   pfParavirtTsc   Where to return whether paravirt TSC is enabled.
+ *
+ * @thread  EMT(pVCpu).
+ * @see     TMCpuTickGetDeadlineAndTscOffset().
+ */
+VMM_INT_DECL(bool) TMCpuTickCanUseRealTSC(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *poffRealTsc, bool *pfParavirtTsc)
+{
+    Assert(pVCpu->tm.s.fTSCTicking || DBGFIsStepping(pVCpu));
+
+    *pfParavirtTsc = pVM->tm.s.fParavirtTscEnabled;
+
+    /*
+     * In real TSC mode it's easy, we just need the delta & offTscRawSrc and
+     * the CPU will add them to RDTSC and RDTSCP at runtime.
+     *
+     * In tmCpuTickGetInternal we do:
+     *          SUPReadTsc() - pVCpu->tm.s.offTSCRawSrc;
+     * Where SUPReadTsc() does:
+     *          ASMReadTSC() - pGipCpu->i64TscDelta;
+     * Which means tmCpuTickGetInternal actually does:
+     *          ASMReadTSC() - pGipCpu->i64TscDelta - pVCpu->tm.s.offTSCRawSrc;
+     * So, the offset to be ADDED to RDTSC[P] is:
+     *          offRealTsc = -(pGipCpu->i64TscDelta + pVCpu->tm.s.offTSCRawSrc)
+     */
+    if (pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET)
+    {
+        /** @todo We should negate both deltas!  It's soo weird that we do the
+         *        exact opposite of what the hardware implements. */
+# ifdef IN_RING3
+        *poffRealTsc = (uint64_t)0 - pVCpu->tm.s.offTSCRawSrc - (uint64_t)SUPGetTscDelta(g_pSUPGlobalInfoPage);
+# else
+        *poffRealTsc = (uint64_t)0 - pVCpu->tm.s.offTSCRawSrc - (uint64_t)SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet);
+# endif
+        return true;
+    }
+
+    /*
+     * We require:
+     *     1. A fixed TSC, this is checked at init time.
+     *     2. That the TSC is ticking (we shouldn't be here if it isn't)
+     *     3. Either that we're using the real TSC as time source or
+     *          a) we don't have any lag to catch up, and
+     *          b) the virtual sync clock hasn't been halted by an expired timer, and
+     *          c) we're not using warp drive (accelerated virtual guest time).
+     */
+    if (   pVM->tm.s.enmTSCMode == TMTSCMODE_DYNAMIC
+        && !pVM->tm.s.fVirtualSyncCatchUp
+        && RT_LIKELY(pVM->tm.s.fVirtualSyncTicking)
+        && !pVM->tm.s.fVirtualWarpDrive)
+    {
+        /* The source is the timer synchronous virtual clock. */
+        uint64_t uTscNow;
+        uint64_t u64Now = tmCpuTickCalcFromVirtual(pVM, TMVirtualSyncGetNoCheckWithTsc(pVM, &uTscNow))
+                        - pVCpu->tm.s.offTSCRawSrc;
+        /** @todo When we start collecting statistics on how much time we spend executing
+         * guest code before exiting, we should check this against the next virtual sync
+         * timer timeout. If it's lower than the avg. length, we should trap rdtsc to increase
+         * the chance that we'll get interrupted right after the timer expired. */
+        if (u64Now >= pVCpu->tm.s.u64TSCLastSeen)
+        {
+# ifdef IN_RING3
+            *poffRealTsc = u64Now - (uTscNow + (uint64_t)SUPGetTscDelta(g_pSUPGlobalInfoPage);
+# else
+            *poffRealTsc = u64Now - (uTscNow + (uint64_t)SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet));
+# endif
+            return true;    /** @todo count this? */
+        }
+    }
+
+# ifdef VBOX_WITH_STATISTICS
+    tmCpuTickRecordOffsettedTscRefusal(pVM, pVCpu);
+# endif
+    return false;
+}
+
+
+/**
+ * Calculates the number of host CPU ticks till the next virtual sync deadline.
+ *
+ * @note    To save work, this function will not bother calculating the accurate
+ *          tick count for deadlines that are more than a second ahead.
+ *
+ * @returns The number of host cpu ticks to the next deadline.  Max one second.
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   cNsToDeadline   The number of nano seconds to the next virtual
+ *                          sync deadline.
+ */
+DECLINLINE(uint64_t) tmCpuCalcTicksToDeadline(PVMCPUCC pVCpu, uint64_t cNsToDeadline)
+{
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL <= _4G);
+# ifdef IN_RING3
+    RT_NOREF_PV(pVCpu);
+    PSUPGIP const pGip = g_pSUPGlobalInfoPage;
+    uint64_t uCpuHz = pGip ? SUPGetCpuHzFromGip(pGip) : pVCpu->pVMR3->tm.s.cTSCTicksPerSecondHost;
+# else
+    uint64_t uCpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
+# endif
+    if (RT_UNLIKELY(cNsToDeadline >= TMCLOCK_FREQ_VIRTUAL))
+        return uCpuHz;
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL <= UINT32_MAX);
+    uint64_t cTicks = ASMMultU64ByU32DivByU32(uCpuHz, (uint32_t)cNsToDeadline, TMCLOCK_FREQ_VIRTUAL);
+    if (cTicks > 4000)
+        cTicks -= 4000; /* fudge to account for overhead */
+    else
+        cTicks >>= 1;
+    return cTicks;
+}
+
+
+/**
+ * Gets the next deadline in host CPU clock ticks and the TSC offset if we can
+ * use the raw TSC.
+ *
+ * @returns The number of host CPU clock ticks to the next timer deadline.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pVCpu               The cross context virtual CPU structure of the calling EMT.
+ * @param   poffRealTsc         The offset against the TSC of the current host CPU,
+ *                              if pfOffsettedTsc is set to true.
+ * @param   pfOffsettedTsc      Where to return whether TSC offsetting can be used.
+ * @param   pfParavirtTsc       Where to return whether paravirt TSC is enabled.
+ * @param   puTscNow            Where to return the TSC value that the return
+ *                              value is relative to.   This is delta adjusted.
+ * @param   puDeadlineVersion   Where to return the deadline "version" number.
+ *                              Use with TMVirtualSyncIsCurrentDeadlineVersion()
+ *                              to check if the absolute deadline is still up to
+ *                              date and the caller can skip calling this
+ *                              function.
+ *
+ * @thread  EMT(pVCpu).
+ * @see     TMCpuTickCanUseRealTSC().
+ */
+VMM_INT_DECL(uint64_t) TMCpuTickGetDeadlineAndTscOffset(PVMCC pVM, PVMCPUCC pVCpu, uint64_t *poffRealTsc,
+                                                        bool *pfOffsettedTsc, bool *pfParavirtTsc,
+                                                        uint64_t *puTscNow, uint64_t *puDeadlineVersion)
+{
+    Assert(pVCpu->tm.s.fTSCTicking || DBGFIsStepping(pVCpu));
+
+    *pfParavirtTsc = pVM->tm.s.fParavirtTscEnabled;
+
+    /*
+     * Same logic as in TMCpuTickCanUseRealTSC.
+     */
+    if (pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET)
+    {
+        /** @todo We should negate both deltas!  It's soo weird that we do the
+         *        exact opposite of what the hardware implements. */
+# ifdef IN_RING3
+        *poffRealTsc     = (uint64_t)0 - pVCpu->tm.s.offTSCRawSrc - (uint64_t)SUPGetTscDelta(g_pSUPGlobalInfoPage);
+# else
+        *poffRealTsc     = (uint64_t)0 - pVCpu->tm.s.offTSCRawSrc - (uint64_t)SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet);
+# endif
+        *pfOffsettedTsc  = true;
+        return tmCpuCalcTicksToDeadline(pVCpu, TMVirtualSyncGetNsToDeadline(pVM, puDeadlineVersion, puTscNow));
+    }
+
+    /*
+     * Same logic as in TMCpuTickCanUseRealTSC.
+     */
+    if (   pVM->tm.s.enmTSCMode == TMTSCMODE_DYNAMIC
+        && !pVM->tm.s.fVirtualSyncCatchUp
+        && RT_LIKELY(pVM->tm.s.fVirtualSyncTicking)
+        && !pVM->tm.s.fVirtualWarpDrive)
+    {
+        /* The source is the timer synchronous virtual clock. */
+        uint64_t cNsToDeadline;
+        uint64_t u64NowVirtSync = TMVirtualSyncGetWithDeadlineNoCheck(pVM, &cNsToDeadline, puDeadlineVersion, puTscNow);
+        uint64_t u64Now = tmCpuTickCalcFromVirtual(pVM, u64NowVirtSync);
+        u64Now -= pVCpu->tm.s.offTSCRawSrc;
+
+# ifdef IN_RING3
+        *poffRealTsc     = u64Now - (*puTscNow + (uint64_t)SUPGetTscDelta(g_pSUPGlobalInfoPage)); /* undoing delta */
+# else
+        *poffRealTsc     = u64Now - (*puTscNow + (uint64_t)SUPGetTscDeltaByCpuSetIndex(pVCpu->iHostCpuSet)); /* undoing delta */
+# endif
+        *pfOffsettedTsc  = u64Now >= pVCpu->tm.s.u64TSCLastSeen;
+        return tmCpuCalcTicksToDeadline(pVCpu, cNsToDeadline);
+    }
+
+# ifdef VBOX_WITH_STATISTICS
+    tmCpuTickRecordOffsettedTscRefusal(pVM, pVCpu);
+# endif
+    *pfOffsettedTsc  = false;
+    *poffRealTsc     = 0;
+    return tmCpuCalcTicksToDeadline(pVCpu, TMVirtualSyncGetNsToDeadline(pVM, puDeadlineVersion, puTscNow));
+}
+
+#endif /* IN_RING0 - at the moment */
+
+/**
+ * Read the current CPU timestamp counter.
+ *
+ * @returns Gets the CPU tsc.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   fCheckTimers    Whether to check timers.
+ */
+DECLINLINE(uint64_t) tmCpuTickGetInternal(PVMCPUCC pVCpu, bool fCheckTimers)
+{
+    uint64_t u64;
+
+    if (RT_LIKELY(pVCpu->tm.s.fTSCTicking))
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        switch (pVM->tm.s.enmTSCMode)
+        {
+            case TMTSCMODE_REAL_TSC_OFFSET:
+                u64 = SUPReadTsc();
+                break;
+            case TMTSCMODE_VIRT_TSC_EMULATED:
+            case TMTSCMODE_DYNAMIC:
+                u64 = tmCpuTickGetRawVirtual(pVM, fCheckTimers);
+                break;
+            case TMTSCMODE_NATIVE_API:
+            {
+                u64 = 0;
+                int rcNem = NEMHCQueryCpuTick(pVCpu, &u64, NULL);
+                AssertLogRelRCReturn(rcNem, SUPReadTsc());
+                break;
+            }
+            default:
+                AssertFailedBreakStmt(u64 = SUPReadTsc());
+        }
+        u64 -= pVCpu->tm.s.offTSCRawSrc;
+
+        /* Always return a value higher than what the guest has already seen. */
+        if (RT_LIKELY(u64 > pVCpu->tm.s.u64TSCLastSeen))
+            pVCpu->tm.s.u64TSCLastSeen = u64;
+        else
+        {
+            STAM_COUNTER_INC(&pVM->tm.s.StatTSCUnderflow);
+            pVCpu->tm.s.u64TSCLastSeen += 64;   /** @todo choose a good increment here */
+            u64 = pVCpu->tm.s.u64TSCLastSeen;
+        }
+    }
+    else
+        u64 = pVCpu->tm.s.u64TSC;
+    return u64;
+}
+
+
+/**
+ * Read the current CPU timestamp counter.
+ *
+ * @returns Gets the CPU tsc.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMMDECL(uint64_t) TMCpuTickGet(PVMCPUCC pVCpu)
+{
+    return tmCpuTickGetInternal(pVCpu, true /* fCheckTimers */);
+}
+
+
+/**
+ * Read the current CPU timestamp counter, don't check for expired timers.
+ *
+ * @returns Gets the CPU tsc.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(uint64_t) TMCpuTickGetNoCheck(PVMCPUCC pVCpu)
+{
+    return tmCpuTickGetInternal(pVCpu, false /* fCheckTimers */);
+}
+
+
+/**
+ * Sets the current CPU timestamp counter.
+ *
+ * @returns VBox status code.
+ * @param   pVM         The cross context VM structure.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u64Tick     The new timestamp value.
+ *
+ * @thread  EMT which TSC is to be set.
+ */
+VMM_INT_DECL(int) TMCpuTickSet(PVMCC pVM, PVMCPUCC pVCpu, uint64_t u64Tick)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+    STAM_COUNTER_INC(&pVM->tm.s.StatTSCSet);
+
+    /*
+     * This is easier to do when the TSC is paused since resume will
+     * do all the calculations for us. Actually, we don't need to
+     * call tmCpuTickPause here since we overwrite u64TSC anyway.
+     */
+    bool        fTSCTicking    = pVCpu->tm.s.fTSCTicking;
+    pVCpu->tm.s.fTSCTicking    = false;
+    pVCpu->tm.s.u64TSC         = u64Tick;
+    pVCpu->tm.s.u64TSCLastSeen = u64Tick;
+    if (fTSCTicking)
+        tmCpuTickResume(pVM, pVCpu);
+    /** @todo Try help synchronizing it better among the virtual CPUs? */
+
+    return VINF_SUCCESS;
+}
+
+/**
+ * Sets the last seen CPU timestamp counter.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   u64LastSeenTick     The last seen timestamp value.
+ *
+ * @thread  EMT which TSC is to be set.
+ */
+VMM_INT_DECL(int) TMCpuTickSetLastSeen(PVMCPUCC pVCpu, uint64_t u64LastSeenTick)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    LogFlow(("TMCpuTickSetLastSeen %RX64\n", u64LastSeenTick));
+    /** @todo deal with wraparound!   */
+    if (pVCpu->tm.s.u64TSCLastSeen < u64LastSeenTick)
+        pVCpu->tm.s.u64TSCLastSeen = u64LastSeenTick;
+    return VINF_SUCCESS;
+}
+
+/**
+ * Gets the last seen CPU timestamp counter of the guest.
+ *
+ * @returns the last seen TSC.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ *
+ * @thread  EMT(pVCpu).
+ */
+VMM_INT_DECL(uint64_t) TMCpuTickGetLastSeen(PVMCPUCC pVCpu)
+{
+    VMCPU_ASSERT_EMT(pVCpu);
+
+    return pVCpu->tm.s.u64TSCLastSeen;
+}
+
+
+/**
+ * Get the timestamp frequency.
+ *
+ * @returns Number of ticks per second.
+ * @param   pVM     The cross context VM structure.
+ */
+VMMDECL(uint64_t) TMCpuTicksPerSecond(PVMCC pVM)
+{
+    if (pVM->tm.s.enmTSCMode == TMTSCMODE_REAL_TSC_OFFSET)
+    {
+        PSUPGLOBALINFOPAGE const pGip = g_pSUPGlobalInfoPage;
+        if (pGip && pGip->u32Mode != SUPGIPMODE_INVARIANT_TSC)
+        {
+#ifdef IN_RING3
+            uint64_t cTSCTicksPerSecond = SUPGetCpuHzFromGip(pGip);
+#elif defined(IN_RING0)
+            uint64_t cTSCTicksPerSecond = SUPGetCpuHzFromGipBySetIndex(pGip, (uint32_t)RTMpCpuIdToSetIndex(RTMpCpuId()));
+#else
+            uint64_t cTSCTicksPerSecond = SUPGetCpuHzFromGipBySetIndex(pGip, VMMGetCpu(pVM)->iHostCpuSet);
+#endif
+            if (RT_LIKELY(cTSCTicksPerSecond != ~(uint64_t)0))
+                return cTSCTicksPerSecond;
+        }
+    }
+    return pVM->tm.s.cTSCTicksPerSecond;
+}
+
+
+/**
+ * Whether the TSC is ticking for the VCPU.
+ *
+ * @returns true if ticking, false otherwise.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ */
+VMM_INT_DECL(bool) TMCpuTickIsTicking(PVMCPUCC pVCpu)
+{
+    return pVCpu->tm.s.fTSCTicking;
+}
+
diff --git a/src/VBox/VMM/VMMAll/TMAllReal.cpp b/src/VBox/VMM/VMMAll/TMAllReal.cpp
new file mode 100644
index 00000000..7540c87c
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/TMAllReal.cpp
@@ -0,0 +1,63 @@
+/* $Id: TMAllReal.cpp $ */
+/** @file
+ * TM - Timeout Manager, Real Time, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TM
+#include <VBox/vmm/tm.h>
+#include "TMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <iprt/time.h>
+
+
+/**
+ * Gets the current TMCLOCK_REAL time.
+ *
+ * @returns Real time.
+ * @param   pVM             The cross context VM structure.
+ */
+VMM_INT_DECL(uint64_t) TMRealGet(PVM pVM)
+{
+    NOREF(pVM);
+    return RTTimeMilliTS();
+}
+
+
+/**
+ * Gets the frequency of the TMCLOCK_REAL clock.
+ *
+ * @returns frequency.
+ * @param   pVM             The cross context VM structure.
+ */
+VMM_INT_DECL(uint64_t) TMRealGetFreq(PVM pVM)
+{
+    NOREF(pVM);
+    return TMCLOCK_FREQ_REAL;
+}
+
diff --git a/src/VBox/VMM/VMMAll/TMAllVirtual.cpp b/src/VBox/VMM/VMMAll/TMAllVirtual.cpp
new file mode 100644
index 00000000..9244bd85
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/TMAllVirtual.cpp
@@ -0,0 +1,1154 @@
+/* $Id: TMAllVirtual.cpp $ */
+/** @file
+ * TM - Timeout Manager, Virtual Time, All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TM
+#include <VBox/vmm/tm.h>
+#include <VBox/vmm/dbgftrace.h>
+#ifdef IN_RING3
+# include <iprt/thread.h>
+#endif
+#include "TMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/vmm/vmm.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+#include <VBox/sup.h>
+
+#include <iprt/time.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/asm-math.h>
+
+
+
+/**
+ * @interface_method_impl{RTTIMENANOTSDATA,pfnBad}
+ */
+DECLCALLBACK(DECLEXPORT(void)) tmVirtualNanoTSBad(PRTTIMENANOTSDATA pData, uint64_t u64NanoTS, uint64_t u64DeltaPrev,
+                                                  uint64_t u64PrevNanoTS)
+{
+    PVMCC pVM = RT_FROM_CPP_MEMBER(pData, VMCC, VMCC_CTX(tm).s.VirtualGetRawData);
+    pData->cBadPrev++;
+    if ((int64_t)u64DeltaPrev < 0)
+        LogRel(("TM: u64DeltaPrev=%RI64 u64PrevNanoTS=0x%016RX64 u64NanoTS=0x%016RX64 pVM=%p\n",
+                u64DeltaPrev, u64PrevNanoTS, u64NanoTS, pVM));
+    else
+        Log(("TM: u64DeltaPrev=%RI64 u64PrevNanoTS=0x%016RX64 u64NanoTS=0x%016RX64 pVM=%p (debugging?)\n",
+             u64DeltaPrev, u64PrevNanoTS, u64NanoTS, pVM));
+}
+
+
+#ifdef IN_RING3
+/**
+ * @callback_method_impl{FNTIMENANOTSINTERNAL, For driverless mode.}
+ */
+static DECLCALLBACK(uint64_t) tmR3VirtualNanoTSDriverless(PRTTIMENANOTSDATA pData, PRTITMENANOTSEXTRA pExtra)
+{
+    RT_NOREF(pData);
+    if (pExtra)
+        pExtra->uTSCValue = ASMReadTSC();
+    return RTTimeSystemNanoTS();
+}
+#endif
+
+
+/**
+ * @interface_method_impl{RTTIMENANOTSDATA,pfnRediscover}
+ *
+ * This is the initial worker, so the first call in each context ends up here.
+ * It is also used should the delta rating of the host CPUs change or if the
+ * fGetGipCpu feature the current worker relies upon becomes unavailable.  The
+ * last two events may occur as CPUs are taken online.
+ */
+DECLCALLBACK(DECLEXPORT(uint64_t)) tmVirtualNanoTSRediscover(PRTTIMENANOTSDATA pData, PRTITMENANOTSEXTRA pExtra)
+{
+    PVMCC                 pVM = RT_FROM_CPP_MEMBER(pData, VMCC, VMCC_CTX(tm).s.VirtualGetRawData);
+    PFNTIMENANOTSINTERNAL pfnWorker;
+
+    /*
+     * We require a valid GIP for the selection below.
+     * Invalid GIP is fatal, though we have to allow no GIP in driverless mode (ring-3 only).
+     */
+    PSUPGLOBALINFOPAGE pGip = g_pSUPGlobalInfoPage;
+#ifdef IN_RING3
+    if (pGip)
+#endif
+    {
+        AssertFatalMsg(RT_VALID_PTR(pGip), ("pVM=%p pGip=%p\n", pVM, pGip));
+        AssertFatalMsg(pGip->u32Magic == SUPGLOBALINFOPAGE_MAGIC, ("pVM=%p pGip=%p u32Magic=%#x\n", pVM, pGip, pGip->u32Magic));
+        AssertFatalMsg(pGip->u32Mode > SUPGIPMODE_INVALID && pGip->u32Mode < SUPGIPMODE_END,
+                       ("pVM=%p pGip=%p u32Mode=%#x\n", pVM, pGip, pGip->u32Mode));
+
+        /*
+         * Determine the new worker.
+         */
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+        bool const fLFence = RT_BOOL(ASMCpuId_EDX(1) & X86_CPUID_FEATURE_EDX_SSE2);
+#endif
+        switch (pGip->u32Mode)
+        {
+#if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)
+            case SUPGIPMODE_SYNC_TSC:
+            case SUPGIPMODE_INVARIANT_TSC:
+# ifdef IN_RING0
+                if (pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO)
+                    pfnWorker = fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta    : RTTimeNanoTSLegacySyncInvarNoDelta;
+                else
+                    pfnWorker = fLFence ? RTTimeNanoTSLFenceSyncInvarWithDelta  : RTTimeNanoTSLegacySyncInvarWithDelta;
+# else
+                if (pGip->fGetGipCpu & SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS)
+                    pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_PRACTICALLY_ZERO
+                              ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta             : RTTimeNanoTSLegacySyncInvarNoDelta
+                              : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseIdtrLim : RTTimeNanoTSLegacySyncInvarWithDeltaUseIdtrLim;
+                else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS)
+                    pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_PRACTICALLY_ZERO
+                              ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta            : RTTimeNanoTSLegacySyncInvarNoDelta
+                              : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseRdtscp : RTTimeNanoTSLegacySyncInvarWithDeltaUseRdtscp;
+                else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_0B)
+                    pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO
+                              ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta                 : RTTimeNanoTSLegacySyncInvarNoDelta
+                              : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseApicIdExt0B : RTTimeNanoTSLegacySyncInvarWithDeltaUseApicIdExt0B;
+                else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_8000001E)
+                    pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO
+                              ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta                       : RTTimeNanoTSLegacySyncInvarNoDelta
+                              : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseApicIdExt8000001E : RTTimeNanoTSLegacySyncInvarWithDeltaUseApicIdExt8000001E;
+                else
+                    pfnWorker = pGip->enmUseTscDelta <= SUPGIPUSETSCDELTA_ROUGHLY_ZERO
+                              ? fLFence ? RTTimeNanoTSLFenceSyncInvarNoDelta            : RTTimeNanoTSLegacySyncInvarNoDelta
+                              : fLFence ? RTTimeNanoTSLFenceSyncInvarWithDeltaUseApicId : RTTimeNanoTSLegacySyncInvarWithDeltaUseApicId;
+# endif
+                break;
+
+            case SUPGIPMODE_ASYNC_TSC:
+# ifdef IN_RING0
+                pfnWorker = fLFence ? RTTimeNanoTSLFenceAsync : RTTimeNanoTSLegacyAsync;
+# else
+                if (pGip->fGetGipCpu & SUPGIPGETCPU_IDTR_LIMIT_MASK_MAX_SET_CPUS)
+                    pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseIdtrLim     : RTTimeNanoTSLegacyAsyncUseIdtrLim;
+                else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_MASK_MAX_SET_CPUS)
+                    pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseRdtscp      : RTTimeNanoTSLegacyAsyncUseRdtscp;
+                else if (pGip->fGetGipCpu & SUPGIPGETCPU_RDTSCP_GROUP_IN_CH_NUMBER_IN_CL)
+                    pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseRdtscpGroupChNumCl : RTTimeNanoTSLegacyAsyncUseRdtscpGroupChNumCl;
+                else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_0B)
+                    pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseApicIdExt0B : RTTimeNanoTSLegacyAsyncUseApicIdExt0B;
+                else if (pGip->fGetGipCpu & SUPGIPGETCPU_APIC_ID_EXT_8000001E)
+                    pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseApicIdExt8000001E  : RTTimeNanoTSLegacyAsyncUseApicIdExt8000001E;
+                else
+                    pfnWorker = fLFence ? RTTimeNanoTSLFenceAsyncUseApicId      : RTTimeNanoTSLegacyAsyncUseApicId;
+# endif
+                break;
+#endif
+            default:
+                AssertFatalMsgFailed(("pVM=%p pGip=%p u32Mode=%#x\n", pVM, pGip, pGip->u32Mode));
+        }
+    }
+#ifdef IN_RING3
+    else
+        pfnWorker = tmR3VirtualNanoTSDriverless;
+#endif
+
+    /*
+     * Update the pfnVirtualGetRaw pointer and call the worker we selected.
+     */
+    ASMAtomicWritePtr((void * volatile *)&pVM->VMCC_CTX(tm).s.pfnVirtualGetRaw, (void *)(uintptr_t)pfnWorker);
+    return pfnWorker(pData, pExtra);
+}
+
+
+/**
+ * @interface_method_impl{RTTIMENANOTSDATA,pfnBadCpuIndex}
+ */
+DECLCALLBACK(DECLEXPORT(uint64_t)) tmVirtualNanoTSBadCpuIndex(PRTTIMENANOTSDATA pData, PRTITMENANOTSEXTRA pExtra,
+                                                              uint16_t idApic, uint16_t iCpuSet, uint16_t iGipCpu)
+{
+    PVMCC pVM = RT_FROM_CPP_MEMBER(pData, VMCC, VMCC_CTX(tm).s.VirtualGetRawData);
+    AssertFatalMsgFailed(("pVM=%p idApic=%#x iCpuSet=%#x iGipCpu=%#x pExtra=%p\n", pVM, idApic, iCpuSet, iGipCpu, pExtra));
+#ifndef _MSC_VER
+    return UINT64_MAX;
+#endif
+}
+
+
+/**
+ * Wrapper around the IPRT GIP time methods.
+ */
+DECLINLINE(uint64_t) tmVirtualGetRawNanoTS(PVMCC pVM)
+{
+#ifdef IN_RING3
+    uint64_t u64 = pVM->tm.s.pfnVirtualGetRaw(&pVM->tm.s.VirtualGetRawData, NULL /*pExtra*/);
+#elif defined(IN_RING0)
+    uint32_t cPrevSteps = pVM->tmr0.s.VirtualGetRawData.c1nsSteps;
+    uint64_t u64 = pVM->tmr0.s.pfnVirtualGetRaw(&pVM->tmr0.s.VirtualGetRawData, NULL /*pExtra*/);
+    if (cPrevSteps != pVM->tmr0.s.VirtualGetRawData.c1nsSteps)
+        VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
+#else
+# error "unsupported context"
+#endif
+    /*DBGFTRACE_POS_U64(pVM, u64);*/
+    return u64;
+}
+
+
+/**
+ * Wrapper around the IPRT GIP time methods, extended version.
+ */
+DECLINLINE(uint64_t) tmVirtualGetRawNanoTSEx(PVMCC pVM, uint64_t *puTscNow)
+{
+    RTITMENANOTSEXTRA Extra;
+#ifdef IN_RING3
+    uint64_t u64 = pVM->tm.s.pfnVirtualGetRaw(&pVM->tm.s.VirtualGetRawData, &Extra);
+#elif defined(IN_RING0)
+    uint32_t cPrevSteps = pVM->tmr0.s.VirtualGetRawData.c1nsSteps;
+    uint64_t u64 = pVM->tmr0.s.pfnVirtualGetRaw(&pVM->tmr0.s.VirtualGetRawData, &Extra);
+    if (cPrevSteps != pVM->tmr0.s.VirtualGetRawData.c1nsSteps)
+        VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
+#else
+# error "unsupported context"
+#endif
+    if (puTscNow)
+        *puTscNow = Extra.uTSCValue;
+    /*DBGFTRACE_POS_U64(pVM, u64);*/
+    return u64;
+}
+
+
+/**
+ * Get the time when we're not running at 100%
+ *
+ * @returns The timestamp.
+ * @param   pVM         The cross context VM structure.
+ * @param   puTscNow    Where to return the TSC corresponding to the returned
+ *                      timestamp (delta adjusted). Optional.
+ */
+static uint64_t tmVirtualGetRawNonNormal(PVMCC pVM, uint64_t *puTscNow)
+{
+    /*
+     * Recalculate the RTTimeNanoTS() value for the period where
+     * warp drive has been enabled.
+     */
+    uint64_t u64 = tmVirtualGetRawNanoTSEx(pVM, puTscNow);
+    u64 -= pVM->tm.s.u64VirtualWarpDriveStart;
+    u64 *= pVM->tm.s.u32VirtualWarpDrivePercentage;
+    u64 /= 100;
+    u64 += pVM->tm.s.u64VirtualWarpDriveStart;
+
+    /*
+     * Now we apply the virtual time offset.
+     * (Which is the negated tmVirtualGetRawNanoTS() value for when the virtual
+     * machine started if it had been running continuously without any suspends.)
+     */
+    u64 -= pVM->tm.s.u64VirtualOffset;
+    return u64;
+}
+
+
+/**
+ * Get the raw virtual time.
+ *
+ * @returns The current time stamp.
+ * @param   pVM         The cross context VM structure.
+ */
+DECLINLINE(uint64_t) tmVirtualGetRaw(PVMCC pVM)
+{
+    if (RT_LIKELY(!pVM->tm.s.fVirtualWarpDrive))
+        return tmVirtualGetRawNanoTS(pVM) - pVM->tm.s.u64VirtualOffset;
+    return tmVirtualGetRawNonNormal(pVM, NULL /*puTscNow*/);
+}
+
+
+/**
+ * Get the raw virtual time, extended version.
+ *
+ * @returns The current time stamp.
+ * @param   pVM         The cross context VM structure.
+ * @param   puTscNow    Where to return the TSC corresponding to the returned
+ *                      timestamp (delta adjusted). Optional.
+ */
+DECLINLINE(uint64_t) tmVirtualGetRawEx(PVMCC pVM, uint64_t *puTscNow)
+{
+    if (RT_LIKELY(!pVM->tm.s.fVirtualWarpDrive))
+        return tmVirtualGetRawNanoTSEx(pVM, puTscNow) - pVM->tm.s.u64VirtualOffset;
+    return tmVirtualGetRawNonNormal(pVM, puTscNow);
+}
+
+
+/**
+ * Inlined version of tmVirtualGetEx.
+ */
+DECLINLINE(uint64_t) tmVirtualGet(PVMCC pVM, bool fCheckTimers)
+{
+    uint64_t u64;
+    if (RT_LIKELY(pVM->tm.s.cVirtualTicking))
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualGet);
+        u64 = tmVirtualGetRaw(pVM);
+
+        /*
+         * Use the chance to check for expired timers.
+         */
+        if (fCheckTimers)
+        {
+            PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+            if (    !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)
+                &&  !pVM->tm.s.fRunningQueues
+                &&  (   pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].u64Expire <= u64
+                     || (   pVM->tm.s.fVirtualSyncTicking
+                         && pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire <= u64 - pVM->tm.s.offVirtualSync
+                        )
+                    )
+                &&  !pVM->tm.s.fRunningQueues
+               )
+            {
+                STAM_COUNTER_INC(&pVM->tm.s.StatVirtualGetSetFF);
+                Log5(("TMAllVirtual(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+                VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+#ifdef IN_RING3
+                VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+            }
+        }
+    }
+    else
+        u64 = pVM->tm.s.u64Virtual;
+    return u64;
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL time
+ *
+ * @returns The timestamp.
+ * @param   pVM     The cross context VM structure.
+ *
+ * @remark  While the flow of time will never go backwards, the speed of the
+ *          progress varies due to inaccurate RTTimeNanoTS and TSC. The latter can be
+ *          influenced by power saving (SpeedStep, PowerNow!), while the former
+ *          makes use of TSC and kernel timers.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualGet(PVMCC pVM)
+{
+    return tmVirtualGet(pVM, true /*fCheckTimers*/);
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL time without checking
+ * timers or anything.
+ *
+ * Meaning, this has no side effect on FFs like TMVirtualGet may have.
+ *
+ * @returns The timestamp.
+ * @param   pVM     The cross context VM structure.
+ *
+ * @remarks See TMVirtualGet.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualGetNoCheck(PVMCC pVM)
+{
+    return tmVirtualGet(pVM, false /*fCheckTimers*/);
+}
+
+
+/**
+ * Converts the dead line interval from TMCLOCK_VIRTUAL to host nano seconds.
+ *
+ * @returns Host nano second count.
+ * @param   pVM                     The cross context VM structure.
+ * @param   cVirtTicksToDeadline    The TMCLOCK_VIRTUAL interval.
+ */
+DECLINLINE(uint64_t) tmVirtualVirtToNsDeadline(PVM pVM, uint64_t cVirtTicksToDeadline)
+{
+    if (RT_UNLIKELY(pVM->tm.s.fVirtualWarpDrive))
+        return ASMMultU64ByU32DivByU32(cVirtTicksToDeadline, 100, pVM->tm.s.u32VirtualWarpDrivePercentage);
+    return cVirtTicksToDeadline;
+}
+
+
+/**
+ * tmVirtualSyncGetLocked worker for handling catch-up when owning the lock.
+ *
+ * @returns The timestamp.
+ * @param   pVM                 The cross context VM structure.
+ * @param   u64                 raw virtual time.
+ * @param   off                 offVirtualSync.
+ * @param   pcNsToDeadline      Where to return the number of nano seconds to
+ *                              the next virtual sync timer deadline. Can be
+ *                              NULL.
+ * @param   pnsAbsDeadline      Where to return the absolute deadline.
+ *                              Optional.
+ */
+DECLINLINE(uint64_t) tmVirtualSyncGetHandleCatchUpLocked(PVMCC pVM, uint64_t u64, uint64_t off,
+                                                         uint64_t *pcNsToDeadline, uint64_t *pnsAbsDeadline)
+{
+    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked);
+
+    /*
+     * Don't make updates until we've check the timer queue.
+     */
+    bool            fUpdatePrev = true;
+    bool            fUpdateOff  = true;
+    bool            fStop       = false;
+    const uint64_t  u64Prev     = pVM->tm.s.u64VirtualSyncCatchUpPrev;
+    uint64_t        u64Delta    = u64 - u64Prev;
+    if (RT_LIKELY(!(u64Delta >> 32)))
+    {
+        uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, pVM->tm.s.u32VirtualSyncCatchUpPercentage, 100);
+        if (off > u64Sub + pVM->tm.s.offVirtualSyncGivenUp)
+        {
+            off -= u64Sub;
+            Log4(("TM: %'RU64/-%'8RU64: sub %RU32 [vsghcul]\n", u64 - off, off - pVM->tm.s.offVirtualSyncGivenUp, u64Sub));
+        }
+        else
+        {
+            /* we've completely caught up. */
+            STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+            off = pVM->tm.s.offVirtualSyncGivenUp;
+            fStop = true;
+            Log4(("TM: %'RU64/0: caught up [vsghcul]\n", u64));
+        }
+    }
+    else
+    {
+        /* More than 4 seconds since last time (or negative), ignore it. */
+        fUpdateOff = false;
+        fUpdatePrev = !(u64Delta & RT_BIT_64(63));
+        Log(("TMVirtualGetSync: u64Delta=%RX64\n", u64Delta));
+    }
+
+    /*
+     * Complete the calculation of the current TMCLOCK_VIRTUAL_SYNC time. The current
+     * approach is to never pass the head timer. So, when we do stop the clock and
+     * set the timer pending flag.
+     */
+    u64 -= off;
+
+    uint64_t u64Last = ASMAtomicUoReadU64(&pVM->tm.s.u64VirtualSync);
+    if (u64Last > u64)
+    {
+        u64 = u64Last + 1;
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetAdjLast);
+    }
+
+    uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+    if (pnsAbsDeadline)
+        *pnsAbsDeadline = u64Expire; /* Always return the unadjusted absolute deadline, or HM will waste time going
+                                        thru this code over an over again even if there aren't any timer changes. */
+    if (u64 < u64Expire)
+    {
+        ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64);
+        if (fUpdateOff)
+            ASMAtomicWriteU64(&pVM->tm.s.offVirtualSync, off);
+        if (fStop)
+            ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncCatchUp, false);
+        if (fUpdatePrev)
+            ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev, u64);
+        if (pcNsToDeadline)
+        {
+            uint64_t cNsToDeadline = u64Expire - u64;
+            if (pVM->tm.s.fVirtualSyncCatchUp)
+                cNsToDeadline = ASMMultU64ByU32DivByU32(cNsToDeadline, 100,
+                                                        pVM->tm.s.u32VirtualSyncCatchUpPercentage + 100);
+            *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, cNsToDeadline);
+        }
+        PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock);
+    }
+    else
+    {
+        u64 = u64Expire;
+        ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64);
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
+
+        VM_FF_SET(pVM, VM_FF_TM_VIRTUAL_SYNC);
+        PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+        VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+        Log5(("TMAllVirtual(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+        Log4(("TM: %'RU64/-%'8RU64: exp tmr=>ff [vsghcul]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp));
+        PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock);
+
+        if (pcNsToDeadline)
+            *pcNsToDeadline = 0;
+#ifdef IN_RING3
+        VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF);
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetExpired);
+    }
+    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked);
+
+    Log6(("tmVirtualSyncGetHandleCatchUpLocked -> %'RU64\n", u64));
+    DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetHandleCatchUpLocked");
+    return u64;
+}
+
+
+/**
+ * tmVirtualSyncGetEx worker for when we get the lock.
+ *
+ * @returns timesamp.
+ * @param   pVM                 The cross context VM structure.
+ * @param   u64                 The virtual clock timestamp.
+ * @param   pcNsToDeadline      Where to return the number of nano seconds to
+ *                              the next virtual sync timer deadline.  Can be
+ *                              NULL.
+ * @param   pnsAbsDeadline      Where to return the absolute deadline.
+ *                              Optional.
+ */
+DECLINLINE(uint64_t) tmVirtualSyncGetLocked(PVMCC pVM, uint64_t u64, uint64_t *pcNsToDeadline, uint64_t *pnsAbsDeadline)
+{
+    /*
+     * Not ticking?
+     */
+    if (!pVM->tm.s.fVirtualSyncTicking)
+    {
+        u64 = ASMAtomicUoReadU64(&pVM->tm.s.u64VirtualSync);
+        PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock);
+        if (pcNsToDeadline)
+            *pcNsToDeadline = 0;
+        if (pnsAbsDeadline)
+            *pnsAbsDeadline = u64;
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked);
+        Log6(("tmVirtualSyncGetLocked -> %'RU64 [stopped]\n", u64));
+        DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetLocked-stopped");
+        return u64;
+    }
+
+    /*
+     * Handle catch up in a separate function.
+     */
+    uint64_t off = ASMAtomicUoReadU64(&pVM->tm.s.offVirtualSync);
+    if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+        return tmVirtualSyncGetHandleCatchUpLocked(pVM, u64, off, pcNsToDeadline, pnsAbsDeadline);
+
+    /*
+     * Complete the calculation of the current TMCLOCK_VIRTUAL_SYNC time. The current
+     * approach is to never pass the head timer. So, when we do stop the clock and
+     * set the timer pending flag.
+     */
+    u64 -= off;
+
+    uint64_t u64Last = ASMAtomicUoReadU64(&pVM->tm.s.u64VirtualSync);
+    if (u64Last > u64)
+    {
+        u64 = u64Last + 1;
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetAdjLast);
+    }
+
+    uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+    if (pnsAbsDeadline)
+        *pnsAbsDeadline = u64Expire;
+    if (u64 < u64Expire)
+    {
+        ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64);
+        PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock);
+        if (pcNsToDeadline)
+            *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, u64Expire - u64);
+    }
+    else
+    {
+        u64 = u64Expire;
+        ASMAtomicWriteU64(&pVM->tm.s.u64VirtualSync, u64);
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
+
+        VM_FF_SET(pVM, VM_FF_TM_VIRTUAL_SYNC);
+        PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+        VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+        Log5(("TMAllVirtual(%u): FF: %d -> 1\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+        Log4(("TM: %'RU64/-%'8RU64: exp tmr=>ff [vsgl]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp));
+        PDMCritSectLeave(pVM, &pVM->tm.s.VirtualSyncLock);
+
+#ifdef IN_RING3
+        VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+        if (pcNsToDeadline)
+            *pcNsToDeadline = 0;
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF);
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetExpired);
+    }
+    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLocked);
+    Log6(("tmVirtualSyncGetLocked -> %'RU64\n", u64));
+    DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetLocked");
+    return u64;
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time.
+ *
+ * @returns The timestamp.
+ * @param   pVM                 The cross context VM structure.
+ * @param   fCheckTimers        Check timers or not
+ * @param   pcNsToDeadline      Where to return the number of nano seconds to
+ *                              the next virtual sync timer deadline.  Can be
+ *                              NULL.
+ * @param   pnsAbsDeadline      Where to return the absolute deadline.
+ *                              Optional.
+ * @param   puTscNow            Where to return the TSC corresponding to the
+ *                              returned timestamp (delta adjusted). Optional.
+ * @thread  EMT.
+ */
+DECLINLINE(uint64_t) tmVirtualSyncGetEx(PVMCC pVM, bool fCheckTimers, uint64_t *pcNsToDeadline,
+                                        uint64_t *pnsAbsDeadline, uint64_t *puTscNow)
+{
+    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGet);
+
+    uint64_t u64;
+    if (!pVM->tm.s.fVirtualSyncTicking)
+    {
+        if (pcNsToDeadline)
+            *pcNsToDeadline = 0;
+        u64 = pVM->tm.s.u64VirtualSync;
+        DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetEx-stopped1");
+        return u64;
+    }
+
+    /*
+     * Query the virtual clock and do the usual expired timer check.
+     */
+    Assert(pVM->tm.s.cVirtualTicking);
+    u64 = tmVirtualGetRawEx(pVM, puTscNow);
+    if (fCheckTimers)
+    {
+        PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+        if (    !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)
+            &&  pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL].u64Expire <= u64)
+        {
+            Log5(("TMAllVirtual(%u): FF: 0 -> 1\n", __LINE__));
+            VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+#ifdef IN_RING3
+            VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM /** @todo |VMNOTIFYFF_FLAGS_POKE*/);
+#endif
+            STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF);
+        }
+    }
+
+    /*
+     * If we can get the lock, get it.  The result is much more reliable.
+     *
+     * Note! This is where all clock source devices branch off because they
+     *       will be owning the lock already.  The 'else' is taken by code
+     *       which is less picky or hasn't been adjusted yet
+     */
+    /** @todo switch this around, have the tmVirtualSyncGetLocked code inlined
+     *        here and the remainder of this function in a static worker. */
+    if (PDMCritSectTryEnter(pVM, &pVM->tm.s.VirtualSyncLock) == VINF_SUCCESS)
+        return tmVirtualSyncGetLocked(pVM, u64, pcNsToDeadline, pnsAbsDeadline);
+
+    /*
+     * When the clock is ticking, not doing catch ups and not running into an
+     * expired time, we can get away without locking. Try this first.
+     */
+    uint64_t off;
+    if (ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking))
+    {
+        if (!ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+        {
+            off = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync);
+            if (RT_LIKELY(   ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)
+                          && !ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncCatchUp)
+                          && off == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync)))
+            {
+                off = u64 - off;
+                uint64_t const u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+                if (off < u64Expire)
+                {
+                    if (pnsAbsDeadline)
+                        *pnsAbsDeadline = u64Expire;
+                    if (pcNsToDeadline)
+                        *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, u64Expire - off);
+                    STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLockless);
+                    Log6(("tmVirtualSyncGetEx -> %'RU64 [lockless]\n", off));
+                    DBGFTRACE_U64_TAG(pVM, off, "tmVirtualSyncGetEx-lockless");
+                    return off;
+                }
+            }
+        }
+    }
+    else
+    {
+        off = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSync);
+        if (RT_LIKELY(!ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking)))
+        {
+            if (pcNsToDeadline)
+                *pcNsToDeadline = 0;
+            if (pnsAbsDeadline)
+                *pnsAbsDeadline = off;
+            STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetLockless);
+            Log6(("tmVirtualSyncGetEx -> %'RU64 [lockless/stopped]\n", off));
+            DBGFTRACE_U64_TAG(pVM, off, "tmVirtualSyncGetEx-stopped2");
+            return off;
+        }
+    }
+
+    /*
+     * Read the offset and adjust if we're playing catch-up.
+     *
+     * The catch-up adjusting work by us decrementing the offset by a percentage of
+     * the time elapsed since the previous TMVirtualGetSync call.
+     *
+     * It's possible to get a very long or even negative interval between two read
+     * for the following reasons:
+     *  - Someone might have suspended the process execution, frequently the case when
+     *    debugging the process.
+     *  - We might be on a different CPU which TSC isn't quite in sync with the
+     *    other CPUs in the system.
+     *  - Another thread is racing us and we might have been preempted while inside
+     *    this function.
+     *
+     * Assuming nano second virtual time, we can simply ignore any intervals which has
+     * any of the upper 32 bits set.
+     */
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    int cOuterTries = 42;
+    for (;; cOuterTries--)
+    {
+        /* Try grab the lock, things get simpler when owning the lock. */
+        int rcLock = PDMCritSectTryEnter(pVM, &pVM->tm.s.VirtualSyncLock);
+        if (RT_SUCCESS_NP(rcLock))
+            return tmVirtualSyncGetLocked(pVM, u64, pcNsToDeadline, pnsAbsDeadline);
+
+        /* Re-check the ticking flag. */
+        if (!ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking))
+        {
+            off = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSync);
+            if (   ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncTicking)
+                && cOuterTries > 0)
+                continue;
+            if (pcNsToDeadline)
+                *pcNsToDeadline = 0;
+            if (pnsAbsDeadline)
+                *pnsAbsDeadline = off;
+            Log6(("tmVirtualSyncGetEx -> %'RU64 [stopped]\n", off));
+            DBGFTRACE_U64_TAG(pVM, off, "tmVirtualSyncGetEx-stopped3");
+            return off;
+        }
+
+        off = ASMAtomicReadU64(&pVM->tm.s.offVirtualSync);
+        if (ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+        {
+            /* No changes allowed, try get a consistent set of parameters. */
+            uint64_t const u64Prev    = ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev);
+            uint64_t const offGivenUp = ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp);
+            uint32_t const u32Pct     = ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage);
+            if (    (   u64Prev    == ASMAtomicReadU64(&pVM->tm.s.u64VirtualSyncCatchUpPrev)
+                     && offGivenUp == ASMAtomicReadU64(&pVM->tm.s.offVirtualSyncGivenUp)
+                     && u32Pct     == ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage)
+                     && ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+                ||  cOuterTries <= 0)
+            {
+                uint64_t u64Delta = u64 - u64Prev;
+                if (RT_LIKELY(!(u64Delta >> 32)))
+                {
+                    uint64_t u64Sub = ASMMultU64ByU32DivByU32(u64Delta, u32Pct, 100);
+                    if (off > u64Sub + offGivenUp)
+                    {
+                        off -= u64Sub;
+                        Log4(("TM: %'RU64/-%'8RU64: sub %RU32 [NoLock]\n", u64 - off, pVM->tm.s.offVirtualSync - offGivenUp, u64Sub));
+                    }
+                    else
+                    {
+                        /* we've completely caught up. */
+                        STAM_PROFILE_ADV_STOP(&pVM->tm.s.StatVirtualSyncCatchup, c);
+                        off = offGivenUp;
+                        Log4(("TM: %'RU64/0: caught up [NoLock]\n", u64));
+                    }
+                }
+                else
+                    /* More than 4 seconds since last time (or negative), ignore it. */
+                    Log(("TMVirtualGetSync: u64Delta=%RX64 (NoLock)\n", u64Delta));
+
+                /* Check that we're still running and in catch up. */
+                if (    ASMAtomicUoReadBool(&pVM->tm.s.fVirtualSyncTicking)
+                    &&  ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+                    break;
+                if (cOuterTries <= 0)
+                    break; /* enough */
+            }
+        }
+        else if (   off == ASMAtomicReadU64(&pVM->tm.s.offVirtualSync)
+                 && !ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+            break; /* Got an consistent offset */
+        else if (cOuterTries <= 0)
+            break; /* enough */
+    }
+    if (cOuterTries <= 0)
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetELoop);
+
+    /*
+     * Complete the calculation of the current TMCLOCK_VIRTUAL_SYNC time. The current
+     * approach is to never pass the head timer. So, when we do stop the clock and
+     * set the timer pending flag.
+     */
+    u64 -= off;
+/** @todo u64VirtualSyncLast */
+    uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+    if (pnsAbsDeadline)
+        *pnsAbsDeadline = u64Expire;
+    if (u64 >= u64Expire)
+    {
+        PVMCPUCC pVCpuDst = VMCC_GET_CPU(pVM, pVM->tm.s.idTimerCpu);
+        if (!VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+        {
+            Log5(("TMAllVirtual(%u): FF: %d -> 1 (NoLock)\n", __LINE__, VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)));
+            VM_FF_SET(pVM, VM_FF_TM_VIRTUAL_SYNC); /* Hmm? */
+            VMCPU_FF_SET(pVCpuDst, VMCPU_FF_TIMER);
+#ifdef IN_RING3
+            VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
+#endif
+            STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetSetFF);
+            Log4(("TM: %'RU64/-%'8RU64: exp tmr=>ff [NoLock]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp));
+        }
+        else
+            Log4(("TM: %'RU64/-%'8RU64: exp tmr [NoLock]\n", u64, pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp));
+        if (pcNsToDeadline)
+            *pcNsToDeadline = 0;
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualSyncGetExpired);
+    }
+    else if (pcNsToDeadline)
+    {
+        uint64_t cNsToDeadline = u64Expire - u64;
+        if (ASMAtomicReadBool(&pVM->tm.s.fVirtualSyncCatchUp))
+            cNsToDeadline = ASMMultU64ByU32DivByU32(cNsToDeadline, 100,
+                                                    ASMAtomicReadU32(&pVM->tm.s.u32VirtualSyncCatchUpPercentage) + 100);
+        *pcNsToDeadline = tmVirtualVirtToNsDeadline(pVM, cNsToDeadline);
+    }
+
+    Log6(("tmVirtualSyncGetEx -> %'RU64\n", u64));
+    DBGFTRACE_U64_TAG(pVM, u64, "tmVirtualSyncGetEx-nolock");
+    return u64;
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time.
+ *
+ * @returns The timestamp.
+ * @param   pVM             The cross context VM structure.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGet(PVMCC pVM)
+{
+    return tmVirtualSyncGetEx(pVM, true /*fCheckTimers*/, NULL /*pcNsToDeadline*/, NULL /*pnsAbsDeadline*/, NULL /*puTscNow*/);
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time without checking timers running on
+ * TMCLOCK_VIRTUAL.
+ *
+ * @returns The timestamp.
+ * @param   pVM             The cross context VM structure.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGetNoCheck(PVMCC pVM)
+{
+    return tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, NULL /*pcNsToDeadline*/, NULL /*pnsAbsDeadline*/, NULL /*puTscNow*/);
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time without checking timers running on
+ * TMCLOCK_VIRTUAL, also returning corresponding TSC value.
+ *
+ * @returns The timestamp.
+ * @param   pVM             The cross context VM structure.
+ * @param   puTscNow        Where to return the TSC value that the return
+ *                          value is relative to.   This is delta adjusted.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGetNoCheckWithTsc(PVMCC pVM, uint64_t *puTscNow)
+{
+    return tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, NULL /*pcNsToDeadline*/, NULL /*pnsAbsDeadline*/, puTscNow);
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time.
+ *
+ * @returns The timestamp.
+ * @param   pVM     The cross context VM structure.
+ * @param   fCheckTimers    Check timers on the virtual clock or not.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGetEx(PVMCC pVM, bool fCheckTimers)
+{
+    return tmVirtualSyncGetEx(pVM, fCheckTimers, NULL /*pcNsToDeadline*/, NULL /*pnsAbsDeadline*/, NULL /*puTscNow*/);
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL_SYNC time and ticks to the next deadline
+ * without checking timers running on TMCLOCK_VIRTUAL.
+ *
+ * @returns The timestamp.
+ * @param   pVM                 The cross context VM structure.
+ * @param   pcNsToDeadline      Where to return the number of nano seconds to
+ *                              the next virtual sync timer deadline.
+ * @param   puTscNow            Where to return the TSC value that the return
+ *                              value is relative to.   This is delta adjusted.
+ * @param   puDeadlineVersion   Where to return the deadline "version" number.
+ *                              Use with TMVirtualSyncIsCurrentDeadlineVersion()
+ *                              to check if the absolute deadline is still up to
+ *                              date and the caller can skip calling this
+ *                              function.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGetWithDeadlineNoCheck(PVMCC pVM, uint64_t *pcNsToDeadline,
+                                                           uint64_t *puDeadlineVersion, uint64_t *puTscNow)
+{
+    uint64_t cNsToDeadlineTmp;       /* try convince the compiler to skip the if tests. */
+    uint64_t u64Now = tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, &cNsToDeadlineTmp, puDeadlineVersion, puTscNow);
+    *pcNsToDeadline = cNsToDeadlineTmp;
+    return u64Now;
+}
+
+
+/**
+ * Gets the number of nano seconds to the next virtual sync deadline.
+ *
+ * @returns The number of TMCLOCK_VIRTUAL ticks.
+ * @param   pVM                 The cross context VM structure.
+ * @param   puTscNow            Where to return the TSC value that the return
+ *                              value is relative to.   This is delta adjusted.
+ * @param   puDeadlineVersion   Where to return the deadline "version" number.
+ *                              Use with TMVirtualSyncIsCurrentDeadlineVersion()
+ *                              to check if the absolute deadline is still up to
+ *                              date and the caller can skip calling this
+ *                              function.
+ * @thread  EMT.
+ * @remarks May set the timer and virtual sync FFs.
+ */
+VMMDECL(uint64_t) TMVirtualSyncGetNsToDeadline(PVMCC pVM, uint64_t *puDeadlineVersion, uint64_t *puTscNow)
+{
+    uint64_t cNsToDeadline;
+    tmVirtualSyncGetEx(pVM, false /*fCheckTimers*/, &cNsToDeadline, puDeadlineVersion, puTscNow);
+    return cNsToDeadline;
+}
+
+
+/**
+ * Checks if the given deadline is still current.
+ *
+ * @retval  true if the deadline is still current.
+ * @retval  false if the deadline is outdated.
+ * @param   pVM                 The cross context VM structure.
+ * @param   uDeadlineVersion    The deadline version to check.
+ */
+VMM_INT_DECL(bool) TMVirtualSyncIsCurrentDeadlineVersion(PVMCC pVM, uint64_t uDeadlineVersion)
+{
+    /** @todo Try use ASMAtomicUoReadU64 instead. */
+    uint64_t u64Expire = ASMAtomicReadU64(&pVM->tm.s.aTimerQueues[TMCLOCK_VIRTUAL_SYNC].u64Expire);
+    return u64Expire == uDeadlineVersion;
+}
+
+
+/**
+ * Gets the current lag of the synchronous virtual clock (relative to the virtual clock).
+ *
+ * @return  The current lag.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualSyncGetLag(PVMCC pVM)
+{
+    return pVM->tm.s.offVirtualSync - pVM->tm.s.offVirtualSyncGivenUp;
+}
+
+
+/**
+ * Get the current catch-up percent.
+ *
+ * @return  The current catch0up percent. 0 means running at the same speed as the virtual clock.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(uint32_t) TMVirtualSyncGetCatchUpPct(PVMCC pVM)
+{
+    if (pVM->tm.s.fVirtualSyncCatchUp)
+        return pVM->tm.s.u32VirtualSyncCatchUpPercentage;
+    return 0;
+}
+
+
+/**
+ * Gets the current TMCLOCK_VIRTUAL frequency.
+ *
+ * @returns The frequency.
+ * @param   pVM     The cross context VM structure.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualGetFreq(PVM pVM)
+{
+    NOREF(pVM);
+    return TMCLOCK_FREQ_VIRTUAL;
+}
+
+
+/**
+ * Worker for TMR3PauseClocks.
+ *
+ * @returns VINF_SUCCESS or VERR_TM_VIRTUAL_TICKING_IPE (asserted).
+ * @param   pVM     The cross context VM structure.
+ */
+int tmVirtualPauseLocked(PVMCC pVM)
+{
+    uint32_t c = ASMAtomicDecU32(&pVM->tm.s.cVirtualTicking);
+    AssertMsgReturn(c < pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE);
+    if (c == 0)
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualPause);
+        pVM->tm.s.u64Virtual = tmVirtualGetRaw(pVM);
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, false);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Worker for TMR3ResumeClocks.
+ *
+ * @returns VINF_SUCCESS or VERR_TM_VIRTUAL_TICKING_IPE (asserted).
+ * @param   pVM     The cross context VM structure.
+ */
+int tmVirtualResumeLocked(PVMCC pVM)
+{
+    uint32_t c = ASMAtomicIncU32(&pVM->tm.s.cVirtualTicking);
+    AssertMsgReturn(c <= pVM->cCpus, ("%u vs %u\n", c, pVM->cCpus), VERR_TM_VIRTUAL_TICKING_IPE);
+    if (c == 1)
+    {
+        STAM_COUNTER_INC(&pVM->tm.s.StatVirtualResume);
+        pVM->tm.s.u64VirtualRawPrev         = 0;
+        pVM->tm.s.u64VirtualWarpDriveStart  = tmVirtualGetRawNanoTS(pVM);
+        pVM->tm.s.u64VirtualOffset          = pVM->tm.s.u64VirtualWarpDriveStart - pVM->tm.s.u64Virtual;
+        ASMAtomicWriteBool(&pVM->tm.s.fVirtualSyncTicking, true);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Converts from virtual ticks to nanoseconds.
+ *
+ * @returns nanoseconds.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64VirtualTicks The virtual ticks to convert.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualToNano(PVM pVM, uint64_t u64VirtualTicks)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64VirtualTicks;
+}
+
+
+/**
+ * Converts from virtual ticks to microseconds.
+ *
+ * @returns microseconds.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64VirtualTicks The virtual ticks to convert.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualToMicro(PVM pVM, uint64_t u64VirtualTicks)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64VirtualTicks / 1000;
+}
+
+
+/**
+ * Converts from virtual ticks to milliseconds.
+ *
+ * @returns milliseconds.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64VirtualTicks The virtual ticks to convert.
+ * @remark  There could be rounding errors here. We just do a simple integer divide
+ *          without any adjustments.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualToMilli(PVM pVM, uint64_t u64VirtualTicks)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64VirtualTicks / 1000000;
+}
+
+
+/**
+ * Converts from nanoseconds to virtual ticks.
+ *
+ * @returns virtual ticks.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64NanoTS       The nanosecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualFromNano(PVM pVM, uint64_t u64NanoTS)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64NanoTS;
+}
+
+
+/**
+ * Converts from microseconds to virtual ticks.
+ *
+ * @returns virtual ticks.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64MicroTS      The microsecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualFromMicro(PVM pVM, uint64_t u64MicroTS)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64MicroTS * 1000;
+}
+
+
+/**
+ * Converts from milliseconds to virtual ticks.
+ *
+ * @returns virtual ticks.
+ * @param   pVM             The cross context VM structure.
+ * @param   u64MilliTS      The millisecond value ticks to convert.
+ * @remark  There could be rounding and overflow errors here.
+ */
+VMM_INT_DECL(uint64_t) TMVirtualFromMilli(PVM pVM, uint64_t u64MilliTS)
+{
+    NOREF(pVM);
+    AssertCompile(TMCLOCK_FREQ_VIRTUAL == 1000000000);
+    return u64MilliTS * 1000000;
+}
+
diff --git a/src/VBox/VMM/VMMAll/TRPMAll.cpp b/src/VBox/VMM/VMMAll/TRPMAll.cpp
new file mode 100644
index 00000000..a7901d52
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/TRPMAll.cpp
@@ -0,0 +1,438 @@
+/* $Id: TRPMAll.cpp $ */
+/** @file
+ * TRPM - Trap Monitor - Any Context.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_TRPM
+#include <VBox/vmm/trpm.h>
+#include <VBox/vmm/pgm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/selm.h>
+#include <VBox/vmm/stam.h>
+#include <VBox/vmm/dbgf.h>
+#include "TRPMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/vmm/em.h>
+#include <VBox/log.h>
+#include <iprt/assert.h>
+#include <iprt/asm.h>
+#include <iprt/param.h>
+#include <iprt/x86.h>
+
+
+
+/**
+ * Query info about the current active trap/interrupt.
+ * If no trap is active active an error code is returned.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   pu8TrapNo               Where to store the trap number.
+ * @param   penmType                Where to store the trap type
+ */
+VMMDECL(int) TRPMQueryTrap(PVMCPU pVCpu, uint8_t *pu8TrapNo, TRPMEVENT *penmType)
+{
+    /*
+     * Check if we have a trap at present.
+     */
+    if (pVCpu->trpm.s.uActiveVector != ~0U)
+    {
+        if (pu8TrapNo)
+            *pu8TrapNo = (uint8_t)pVCpu->trpm.s.uActiveVector;
+        if (penmType)
+            *penmType = pVCpu->trpm.s.enmActiveType;
+        return VINF_SUCCESS;
+    }
+
+    return VERR_TRPM_NO_ACTIVE_TRAP;
+}
+
+
+/**
+ * Gets the trap number for the current trap.
+ *
+ * The caller is responsible for making sure there is an active trap which
+ * takes an error code when making this request.
+ *
+ * @returns The current trap number.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+VMMDECL(uint8_t) TRPMGetTrapNo(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    return (uint8_t)pVCpu->trpm.s.uActiveVector;
+}
+
+
+/**
+ * Gets the error code for the current trap.
+ *
+ * The caller is responsible for making sure there is an active trap which
+ * takes an error code when making this request.
+ *
+ * @returns Error code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+VMMDECL(uint32_t) TRPMGetErrorCode(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+#ifdef VBOX_STRICT
+    switch (pVCpu->trpm.s.uActiveVector)
+    {
+        case X86_XCPT_TS:
+        case X86_XCPT_NP:
+        case X86_XCPT_SS:
+        case X86_XCPT_GP:
+        case X86_XCPT_PF:
+        case X86_XCPT_AC:
+        case X86_XCPT_DF:
+            break;
+        default:
+            AssertMsgFailed(("This trap (%#x) doesn't have any error code\n", pVCpu->trpm.s.uActiveVector));
+            break;
+    }
+#endif
+    return pVCpu->trpm.s.uActiveErrorCode;
+}
+
+
+/**
+ * Gets the fault address for the current trap.
+ *
+ * The caller is responsible for making sure there is an active trap 0x0e when
+ * making this request.
+ *
+ * @returns Fault address associated with the trap.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+VMMDECL(RTGCUINTPTR) TRPMGetFaultAddress(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    AssertMsg(pVCpu->trpm.s.uActiveVector == X86_XCPT_PF, ("Not page-fault trap!\n"));
+    return pVCpu->trpm.s.uActiveCR2;
+}
+
+
+/**
+ * Gets the instruction-length for the current trap (only relevant for software
+ * interrupts and software exceptions \#BP and \#OF).
+ *
+ * The caller is responsible for making sure there is an active trap 0x0e when
+ * making this request.
+ *
+ * @returns Fault address associated with the trap.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+VMMDECL(uint8_t) TRPMGetInstrLength(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    return pVCpu->trpm.s.cbInstr;
+}
+
+
+/**
+ * Checks if the current \#DB exception is due to an INT1/ICEBP instruction.
+ *
+ * The caller is responsible for making sure there is an active trap.
+ *
+ * @returns @c true if it's due to INT1/ICEBP, @c false if not.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMDECL(bool) TRPMIsTrapDueToIcebp(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    return pVCpu->trpm.s.fIcebp;
+}
+
+
+/**
+ * Clears the current active trap/exception/interrupt.
+ *
+ * The caller is responsible for making sure there is an active trap
+ * when making this request.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ */
+VMMDECL(int) TRPMResetTrap(PVMCPU pVCpu)
+{
+    /*
+     * Cannot reset non-existing trap!
+     */
+    if (pVCpu->trpm.s.uActiveVector == ~0U)
+    {
+        AssertMsgFailed(("No active trap!\n"));
+        return VERR_TRPM_NO_ACTIVE_TRAP;
+    }
+
+    /*
+     * Reset it.
+     */
+    pVCpu->trpm.s.uActiveVector = ~0U;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Assert trap/exception/interrupt.
+ *
+ * The caller is responsible for making sure there is no active trap
+ * when making this request.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   u8TrapNo            The trap vector to assert.
+ * @param   enmType             Trap type.
+ */
+VMMDECL(int) TRPMAssertTrap(PVMCPUCC pVCpu, uint8_t u8TrapNo, TRPMEVENT enmType)
+{
+    Log2(("TRPMAssertTrap: u8TrapNo=%02x type=%d\n", u8TrapNo, enmType));
+
+    /*
+     * Cannot assert a trap when one is already active.
+     */
+    if (pVCpu->trpm.s.uActiveVector != ~0U)
+    {
+        AssertMsgFailed(("CPU%d: Active trap %#x\n", pVCpu->idCpu, pVCpu->trpm.s.uActiveVector));
+        return VERR_TRPM_ACTIVE_TRAP;
+    }
+
+    pVCpu->trpm.s.uActiveVector               = u8TrapNo;
+    pVCpu->trpm.s.enmActiveType               = enmType;
+    pVCpu->trpm.s.uActiveErrorCode            = ~0U;
+    pVCpu->trpm.s.uActiveCR2                  = 0xdeadface;
+    pVCpu->trpm.s.cbInstr                     = UINT8_MAX;
+    pVCpu->trpm.s.fIcebp                      = false;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Assert a page-fault exception.
+ *
+ * The caller is responsible for making sure there is no active trap
+ * when making this request.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uCR2                The new fault address.
+ * @param   uErrorCode          The error code for the page-fault.
+ */
+VMMDECL(int) TRPMAssertXcptPF(PVMCPUCC pVCpu, RTGCUINTPTR uCR2, uint32_t uErrorCode)
+{
+    Log2(("TRPMAssertXcptPF: uCR2=%RGv uErrorCode=%#RX32\n", uCR2, uErrorCode));
+
+    /*
+     * Cannot assert a trap when one is already active.
+     */
+    if (pVCpu->trpm.s.uActiveVector != ~0U)
+    {
+        AssertMsgFailed(("CPU%d: Active trap %#x\n", pVCpu->idCpu, pVCpu->trpm.s.uActiveVector));
+        return VERR_TRPM_ACTIVE_TRAP;
+    }
+
+    pVCpu->trpm.s.uActiveVector               = X86_XCPT_PF;
+    pVCpu->trpm.s.enmActiveType               = TRPM_TRAP;
+    pVCpu->trpm.s.uActiveErrorCode            = uErrorCode;
+    pVCpu->trpm.s.uActiveCR2                  = uCR2;
+    pVCpu->trpm.s.cbInstr                     = UINT8_MAX;
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the error code of the current trap.
+ * (This function is for use in trap handlers and such.)
+ *
+ * The caller is responsible for making sure there is an active trap
+ * which takes an errorcode when making this request.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uErrorCode          The new error code.
+ */
+VMMDECL(void) TRPMSetErrorCode(PVMCPU pVCpu, uint32_t uErrorCode)
+{
+    Log2(("TRPMSetErrorCode: uErrorCode=%#RX32\n", uErrorCode));
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    AssertMsg(   pVCpu->trpm.s.enmActiveType == TRPM_TRAP
+              || (   pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT && pVCpu->trpm.s.uActiveVector == X86_XCPT_DB),
+              ("Not hardware exception or privileged software exception (INT1/ICEBP)!\n"));
+    pVCpu->trpm.s.uActiveErrorCode = uErrorCode;
+#ifdef VBOX_STRICT
+    if (pVCpu->trpm.s.enmActiveType == TRPM_TRAP)
+    {
+        switch (pVCpu->trpm.s.uActiveVector)
+        {
+            case X86_XCPT_TS: case X86_XCPT_NP: case X86_XCPT_SS: case X86_XCPT_GP: case X86_XCPT_PF:
+                AssertMsg(uErrorCode != ~0U, ("Invalid uErrorCode=%#x u8TrapNo=%u\n", uErrorCode, pVCpu->trpm.s.uActiveVector));
+                break;
+            case X86_XCPT_AC: case X86_XCPT_DF:
+                AssertMsg(uErrorCode == 0,   ("Invalid uErrorCode=%#x u8TrapNo=%u\n", uErrorCode, pVCpu->trpm.s.uActiveVector));
+                break;
+            default:
+                AssertMsg(uErrorCode == ~0U, ("Invalid uErrorCode=%#x u8TrapNo=%u\n", uErrorCode, pVCpu->trpm.s.uActiveVector));
+                break;
+        }
+    }
+#endif
+}
+
+
+/**
+ * Sets the fault address of the current \#PF trap. (This function is for use in
+ * trap handlers and such.)
+ *
+ * The caller is responsible for making sure there is an active trap 0e
+ * when making this request.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   uCR2                The new fault address (cr2 register).
+ */
+VMMDECL(void) TRPMSetFaultAddress(PVMCPU pVCpu, RTGCUINTPTR uCR2)
+{
+    Log2(("TRPMSetFaultAddress: uCR2=%RGv\n", uCR2));
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    AssertMsg(pVCpu->trpm.s.enmActiveType == TRPM_TRAP, ("Not hardware exception!\n"));
+    AssertMsg(pVCpu->trpm.s.uActiveVector == X86_XCPT_PF, ("Not trap 0e!\n"));
+    pVCpu->trpm.s.uActiveCR2 = uCR2;
+}
+
+
+/**
+ * Sets the instruction-length of the current trap (relevant for software
+ * interrupts and software exceptions like \#BP, \#OF).
+ *
+ * The caller is responsible for making sure there is an active trap when making
+ * this request.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   cbInstr             The instruction length.
+ */
+VMMDECL(void) TRPMSetInstrLength(PVMCPU pVCpu, uint8_t cbInstr)
+{
+    Log2(("TRPMSetInstrLength: cbInstr=%u\n", cbInstr));
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    AssertMsg(   pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT
+              || (   pVCpu->trpm.s.enmActiveType == TRPM_TRAP
+                  && (   pVCpu->trpm.s.uActiveVector == X86_XCPT_BP
+                      || pVCpu->trpm.s.uActiveVector == X86_XCPT_OF)),
+              ("Invalid trap type %#x\n", pVCpu->trpm.s.enmActiveType));
+    pVCpu->trpm.s.cbInstr = cbInstr;
+}
+
+
+/**
+ * Sets if the current \#DB exception is due to an INT1/ICEBP instruction.
+ *
+ * The caller is responsible for making sure there is an active trap and it's a
+ * \#DB.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMDECL(void) TRPMSetTrapDueToIcebp(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT, ("Trap type for INT1/ICEBP invalid!"));
+    AssertMsg(pVCpu->trpm.s.uActiveVector == X86_XCPT_DB, ("INT1/ICEBP must be indicated by a #DB!\n"));
+    pVCpu->trpm.s.fIcebp = true;
+}
+
+
+/**
+ * Checks if the current active trap/interrupt/exception/fault/whatever is a software
+ * interrupt or not.
+ *
+ * The caller is responsible for making sure there is an active trap
+ * when making this request.
+ *
+ * @returns true if software interrupt, false if not.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMDECL(bool) TRPMIsSoftwareInterrupt(PVMCPU pVCpu)
+{
+    AssertMsg(pVCpu->trpm.s.uActiveVector != ~0U, ("No active trap!\n"));
+    return (pVCpu->trpm.s.enmActiveType == TRPM_SOFTWARE_INT);
+}
+
+
+/**
+ * Check if there is an active trap.
+ *
+ * @returns true if trap active, false if not.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ */
+VMMDECL(bool) TRPMHasTrap(PVMCPU pVCpu)
+{
+    return pVCpu->trpm.s.uActiveVector != ~0U;
+}
+
+
+/**
+ * Query all info about the current active trap/interrupt.
+ * If no trap is active active an error code is returned.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pu8TrapNo       Where to store the trap number.
+ * @param   pEnmType        Where to store the trap type.
+ * @param   puErrorCode     Where to store the error code associated with some
+ *                          traps. ~0U is stored if the trap has no error code.
+ * @param   puCR2           Where to store the CR2 associated with a trap 0E.
+ * @param   pcbInstr        Where to store the instruction-length associated with
+ *                          some traps.
+ * @param   pfIcebp         Where to store whether the trap is a \#DB caused by an
+ *                          INT1/ICEBP instruction.
+ */
+VMMDECL(int) TRPMQueryTrapAll(PVMCPU pVCpu, uint8_t *pu8TrapNo, TRPMEVENT *pEnmType, uint32_t *puErrorCode, PRTGCUINTPTR puCR2,
+                              uint8_t *pcbInstr, bool *pfIcebp)
+{
+    /*
+     * Check if we have an active trap.
+     */
+    if (pVCpu->trpm.s.uActiveVector == ~0U)
+        return VERR_TRPM_NO_ACTIVE_TRAP;
+
+    if (pu8TrapNo)
+        *pu8TrapNo      = (uint8_t)pVCpu->trpm.s.uActiveVector;
+    if (pEnmType)
+        *pEnmType       = pVCpu->trpm.s.enmActiveType;
+    if (puErrorCode)
+        *puErrorCode    = pVCpu->trpm.s.uActiveErrorCode;
+    if (puCR2)
+        *puCR2          = pVCpu->trpm.s.uActiveCR2;
+    if (pcbInstr)
+        *pcbInstr       = pVCpu->trpm.s.cbInstr;
+    if (pfIcebp)
+        *pfIcebp        = pVCpu->trpm.s.fIcebp;
+    return VINF_SUCCESS;
+}
+
diff --git a/src/VBox/VMM/VMMAll/VMAll.cpp b/src/VBox/VMM/VMMAll/VMAll.cpp
new file mode 100644
index 00000000..67909b83
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/VMAll.cpp
@@ -0,0 +1,444 @@
+/* $Id: VMAll.cpp $ */
+/** @file
+ * VM - Virtual Machine All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VM
+#include "VMInternal.h"
+#include <VBox/vmm/vmm.h>
+#include <VBox/vmm/mm.h>
+#include <VBox/vmm/vmcc.h>
+#include <VBox/err.h>
+#include <VBox/log.h>
+
+#include <iprt/assert.h>
+#include <iprt/string.h>
+#include <iprt/thread.h>
+
+
+#ifdef IN_RING3
+
+/**
+ * Sets the error message.
+ *
+ * @returns rc. Meaning you can do:
+ *    @code
+ *    return VM_SET_ERROR(pVM, VERR_OF_YOUR_CHOICE, "descriptive message");
+ *    @endcode
+ * @param   pVM             The cross context VM structure.
+ * @param   rc              VBox status code.
+ * @param   SRC_POS         Use RT_SRC_POS.
+ * @param   pszFormat       Error message format string.
+ * @param   ...             Error message arguments.
+ * @thread  Any
+ */
+VMMDECL(int) VMSetError(PVMCC pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, ...)
+{
+    va_list args;
+    va_start(args, pszFormat);
+    int rc2 = VMSetErrorV(pVM, rc, RT_SRC_POS_ARGS, pszFormat, args); Assert(rc == rc2); NOREF(rc2);
+    va_end(args);
+    return rc;
+}
+
+
+/**
+ * Sets the error message.
+ *
+ * @returns rc. Meaning you can do:
+ *    @code
+ *    return VM_SET_ERROR(pVM, VERR_OF_YOUR_CHOICE, "descriptive message");
+ *    @endcode
+ * @param   pVM             The cross context VM structure.
+ * @param   rc              VBox status code.
+ * @param   SRC_POS         Use RT_SRC_POS.
+ * @param   pszFormat       Error message format string.
+ * @param   args            Error message arguments.
+ * @thread  Any
+ */
+VMMDECL(int) VMSetErrorV(PVMCC pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args)
+{
+# ifdef IN_RING3
+    /*
+     * Switch to EMT.
+     */
+    va_list va2;
+    va_copy(va2, args); /* Have to make a copy here or GCC will break. */
+    VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)vmR3SetErrorUV, 7,   /* ASSUMES 3 source pos args! */
+                            pVM->pUVM, rc, RT_SRC_POS_ARGS, pszFormat, &va2);
+    va_end(va2);
+
+# else
+    /*
+     * We're already on the EMT thread and can safely create a VMERROR chunk.
+     */
+    vmSetErrorCopy(pVM, rc, RT_SRC_POS_ARGS, pszFormat, args);
+    VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_VM_SET_ERROR, 0);
+# endif
+    return rc;
+}
+
+
+/**
+ * Copies the error to a VMERROR structure.
+ *
+ * This is mainly intended for Ring-0 and GC where the error must be copied to
+ * memory accessible from ring-3. But it's just possible that we might add
+ * APIs for retrieving the VMERROR copy later.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   rc              VBox status code.
+ * @param   SRC_POS         Use RT_SRC_POS.
+ * @param   pszFormat       Error message format string.
+ * @param   args            Error message arguments.
+ * @thread  EMT
+ */
+void vmSetErrorCopy(PVM pVM, int rc, RT_SRC_POS_DECL, const char *pszFormat, va_list args)
+{
+    NOREF(pVM); NOREF(rc); RT_SRC_POS_NOREF(); NOREF(pszFormat); NOREF(args);
+# if 0 /// @todo implement Ring-0 and GC VMSetError
+    /*
+     * Create the untranslated message copy.
+     */
+    /* free any old message. */
+    MMHyperFree(pVM, MMHyperR32Ctx(pVM, pVM->vm.s.pError));
+    pVM->vm.s.pError = NULL;
+
+    /* calc reasonable start size. */
+    size_t cchFile = pszFile ? strlen(pszFile) : 0;
+    size_t cchFunction = pszFunction ? strlen(pszFunction) : 0;
+    size_t cchFormat = strlen(pszFormat);
+    size_t cb = sizeof(VMERROR)
+              + cchFile + 1
+              + cchFunction + 1
+              + cchFormat + 32;
+
+    /* allocate it */
+    void *pv;
+    int rc2 = MMHyperAlloc(pVM, cb, 0, MM_TAG_VM, &pv);
+    if (RT_SUCCESS(rc2))
+    {
+        /* initialize it. */
+        PVMERROR pErr = (PVMERROR)pv;
+        pErr->cbAllocated = cb;
+        pErr->iLine = iLine;
+        pErr->off = sizeof(VMERROR);
+        pErr->offFile = pErr->offFunction = 0;
+
+        if (cchFile)
+        {
+            pErr->offFile = pErr->off;
+            memcpy((uint8_t *)pErr + pErr->off, pszFile, cchFile + 1);
+            pErr->off += cchFile + 1;
+        }
+
+        if (cchFunction)
+        {
+            pErr->offFunction = pErr->off;
+            memcpy((uint8_t *)pErr + pErr->off, pszFunction, cchFunction + 1);
+            pErr->off += cchFunction + 1;
+        }
+
+        pErr->offMessage = pErr->off;
+
+        /* format the message (pErr might be reallocated) */
+        VMSETERRORFMTARGS Args;
+        Args.pVM = pVM;
+        Args.pErr = pErr;
+
+        va_list va2;
+        va_copy(va2, args);
+        RTStrFormatV(vmSetErrorFmtOut, &pErr, NULL, NULL, &pszFormatTmp, args);
+        va_end(va2);
+
+        /* done. */
+        pVM->vm.s.pErrorR3 = MMHyper2HC(pVM, (uintptr_t)pArgs.pErr);
+    }
+# endif
+}
+
+#endif /* IN_RING3 */
+#ifdef IN_RING3
+
+/**
+ * Sets the runtime error message.
+ *
+ * As opposed VMSetError(), this method is intended to inform the VM user about
+ * errors and error-like conditions that happen at an arbitrary point during VM
+ * execution (like "host memory low" or "out of host disk space").
+ *
+ * @returns VBox status code. For some flags the status code <b>must</b> be
+ *          propagated up the stack.
+ *
+ * @param   pVM             The cross context VM structure.
+ *
+ * @param   fFlags          Flags indicating which actions to take.
+ *                          See VMSETRTERR_FLAGS_* for details on each flag.
+ *
+ * @param   pszErrorId      Unique error identification string. This is used by
+ *                          the frontends and maybe other devices or drivers, so
+ *                          once an ID has been selected it's essentially
+ *                          unchangable. Employ camelcase when constructing the
+ *                          string, leave out spaces.
+ *
+ *                          The registered runtime error callbacks should string
+ *                          switch on this and handle the ones it knows
+ *                          specifically and the unknown ones generically.
+ *
+ * @param   pszFormat       Error message format string.
+ * @param   ...             Error message arguments.
+ *
+ * @thread  Any
+ */
+VMMDECL(int) VMSetRuntimeError(PVMCC pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, ...)
+{
+    va_list va;
+    va_start(va, pszFormat);
+    int rc = VMSetRuntimeErrorV(pVM, fFlags, pszErrorId, pszFormat, va);
+    va_end(va);
+    return rc;
+}
+
+
+/**
+ * va_list version of VMSetRuntimeError.
+ *
+ * @returns VBox status code. For some flags the status code <b>must</b> be
+ *          propagated up the stack.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fFlags          Flags indicating which actions to take. See
+ *                          VMSETRTERR_FLAGS_*.
+ * @param   pszErrorId      Error ID string.
+ * @param   pszFormat       Error message format string.
+ * @param   va              Error message arguments.
+ *
+ * @thread  Any
+ */
+VMMDECL(int) VMSetRuntimeErrorV(PVMCC pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    Log(("VMSetRuntimeErrorV: fFlags=%#x pszErrorId=%s\n", fFlags, pszErrorId));
+
+    /*
+     * Relaxed parameter validation.
+     */
+    AssertPtr(pVM);
+    AssertMsg(!(fFlags & ~(VMSETRTERR_FLAGS_NO_WAIT | VMSETRTERR_FLAGS_SUSPEND | VMSETRTERR_FLAGS_FATAL)), ("%#x\n", fFlags));
+    Assert(!(fFlags & VMSETRTERR_FLAGS_NO_WAIT) || !VM_IS_EMT(pVM));
+    Assert(!(fFlags & VMSETRTERR_FLAGS_SUSPEND) || !(fFlags & VMSETRTERR_FLAGS_FATAL));
+    AssertPtr(pszErrorId);
+    Assert(*pszErrorId);
+    Assert(RTStrEnd(pszErrorId, 128) != NULL);
+    AssertPtr(pszFormat);
+    Assert(RTStrEnd(pszFormat, 512) != NULL);
+
+# ifdef IN_RING3
+    /*
+     * Switch to EMT.
+     *
+     * If it's a no-wait request, we have to format the message into a buffer
+     * here since the variable arguments list will become invalid once we call
+     * va_end and return.
+     */
+    int rc;
+    if (    !(fFlags & VMSETRTERR_FLAGS_NO_WAIT)
+        ||  VM_IS_EMT(pVM))
+    {
+        fFlags &= ~VMSETRTERR_FLAGS_NO_WAIT;
+
+        va_list va2;
+        va_copy(va2, va); /* Have to make a copy here or GCC will break. */
+        rc = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY,
+                                     (PFNRT)vmR3SetRuntimeErrorV, 5, pVM, fFlags, pszErrorId, pszFormat, &va2);
+        va_end(va2);
+    }
+    else
+    {
+        char *pszMessage = MMR3HeapAPrintfV(pVM, MM_TAG_VM, pszFormat, va);
+        rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY,
+                               (PFNRT)vmR3SetRuntimeError, 4, pVM, fFlags, pszErrorId, pszMessage);
+        if (RT_FAILURE(rc))
+            MMR3HeapFree(pszMessage);
+    }
+
+# else
+    /*
+     * We're already on the EMT and can safely create a VMRUNTIMEERROR chunk.
+     */
+    AssertReleaseMsgFailed(("Congratulations! You will have the pleasure of debugging the RC/R0 path.\n"));
+    vmSetRuntimeErrorCopy(pVM, fFlags, pszErrorId, pszFormat, va);
+
+    int rc = VMMRZCallRing3NoCpu(pVM, VMMCALLRING3_VM_SET_RUNTIME_ERROR, 0);
+# endif
+
+    Log(("VMSetRuntimeErrorV: returns %Rrc (pszErrorId=%s)\n", rc, pszErrorId));
+    return rc;
+}
+
+
+/**
+ * Copies the error to a VMRUNTIMEERROR structure.
+ *
+ * This is mainly intended for Ring-0 and RC where the error must be copied to
+ * memory accessible from ring-3. But it's just possible that we might add
+ * APIs for retrieving the VMRUNTIMEERROR copy later.
+ *
+ * @param   pVM             The cross context VM structure.
+ * @param   fFlags          The error flags.
+ * @param   pszErrorId      Error ID string.
+ * @param   pszFormat       Error message format string.
+ * @param   va              Error message arguments. This is of course spoiled
+ *                          by this call.
+ * @thread  EMT
+ */
+void vmSetRuntimeErrorCopy(PVM pVM, uint32_t fFlags, const char *pszErrorId, const char *pszFormat, va_list va)
+{
+    NOREF(pVM); NOREF(fFlags); NOREF(pszErrorId); NOREF(pszFormat); NOREF(va);
+# if 0 /// @todo implement Ring-0 and GC VMSetError
+    /*
+     * Create the untranslated message copy.
+     */
+    /* free any old message. */
+    MMHyperFree(pVM, MMHyperR32Ctx(pVM, pVM->vm.s.pRuntimeErrorR3));
+    pVM->vm.s.pRuntimeErrorR3 = NULL;
+
+    /* calc reasonable start size. */
+    size_t cchErrorID = pszErrorId ? strlen(pszErrorId) : 0;
+    size_t cchFormat = strlen(pszFormat);
+    size_t cb = sizeof(VMRUNTIMEERROR)
+              + cchErrorID + 1
+              + cchFormat + 32;
+
+    /* allocate it */
+    void *pv;
+    int rc2 = MMHyperAlloc(pVM, cb, 0, MM_TAG_VM, &pv);
+    if (RT_SUCCESS(rc2))
+    {
+        /* initialize it. */
+        PVMRUNTIMEERROR pErr = (PVMRUNTIMEERROR)pv;
+        pErr->cbAllocated = cb;
+        pErr->fFlags = fFlags;
+        pErr->off = sizeof(PVMRUNTIMEERROR);
+        pErr->offErrorID = 0;
+
+        if (cchErrorID)
+        {
+            pErr->offErrorID = pErr->off;
+            memcpy((uint8_t *)pErr + pErr->off, pszErrorId, cchErrorID + 1);
+            pErr->off += cchErrorID + 1;
+        }
+
+        pErr->offMessage = pErr->off;
+
+        /* format the message (pErr might be reallocated) */
+        VMSETRUNTIMEERRORFMTARGS Args;
+        Args.pVM = pVM;
+        Args.pErr = pErr;
+
+        va_list va2;
+        va_copy(va2, args);
+        RTStrFormatV(vmSetRuntimeErrorFmtOut, &pErr, NULL, NULL, &pszFormatTmp, args);
+        va_end(va2);
+
+        /* done. */
+        pVM->vm.s.pErrorRuntimeR3 = MMHyper2HC(pVM, (uintptr_t)pArgs.pErr);
+    }
+# endif
+}
+
+#endif /* IN_RING3 */
+
+/**
+ * Gets the name of VM state.
+ *
+ * @returns Pointer to a read-only string with the state name.
+ * @param   enmState    The state.
+ */
+VMMDECL(const char *) VMGetStateName(VMSTATE enmState)
+{
+    switch (enmState)
+    {
+#define MY_CASE(enm) case VMSTATE_##enm: return #enm;
+        MY_CASE(CREATING);
+        MY_CASE(CREATED);
+        MY_CASE(RUNNING);
+        MY_CASE(LOADING);
+        MY_CASE(LOAD_FAILURE);
+        MY_CASE(SAVING);
+        MY_CASE(SUSPENDED);
+        MY_CASE(RESETTING);
+        MY_CASE(GURU_MEDITATION);
+        MY_CASE(OFF);
+        MY_CASE(DESTROYING);
+        MY_CASE(TERMINATED);
+#undef MY_CASE
+        default:
+            return "Unknown";
+    }
+}
+
+
+/**
+ * Gets the total reset count.
+ *
+ * @returns Reset count. UINT32_MAX if @a pVM is invalid.
+ * @param   pVM         The VM handle.
+ */
+VMMDECL(uint32_t) VMGetResetCount(PVMCC pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+    return pVM->vm.s.cResets;
+}
+
+
+/**
+ * Gets the soft reset count.
+ *
+ * @returns Soft reset count. UINT32_MAX if @a pVM is invalid.
+ * @param   pVM         The VM handle.
+ */
+VMMDECL(uint32_t) VMGetSoftResetCount(PVMCC pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+    return pVM->vm.s.cSoftResets;
+}
+
+
+/**
+ * Gets the hard reset count.
+ *
+ * @returns Hard reset count. UINT32_MAX if @a pVM is invalid.
+ * @param   pVM         The VM handle.
+ */
+VMMDECL(uint32_t) VMGetHardResetCount(PVMCC pVM)
+{
+    VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
+    return pVM->vm.s.cHardResets;
+}
+
diff --git a/src/VBox/VMM/VMMAll/VMMAll.cpp b/src/VBox/VMM/VMMAll/VMMAll.cpp
new file mode 100644
index 00000000..b8ba8eb8
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/VMMAll.cpp
@@ -0,0 +1,296 @@
+/* $Id: VMMAll.cpp $ */
+/** @file
+ * VMM All Contexts.
+ */
+
+/*
+ * Copyright (C) 2006-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Header Files                                                                                                                 *
+*********************************************************************************************************************************/
+#define LOG_GROUP LOG_GROUP_VMM
+#include <VBox/vmm/vmm.h>
+#include "VMMInternal.h"
+#include <VBox/vmm/vmcc.h>
+#ifdef IN_RING0
+# include <VBox/vmm/gvm.h>
+#endif
+#include <VBox/vmm/hm.h>
+#include <VBox/vmm/vmcpuset.h>
+#include <VBox/param.h>
+#include <iprt/thread.h>
+#include <iprt/mp.h>
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+/** User counter for the vmmInitFormatTypes function (pro forma). */
+static volatile uint32_t g_cFormatTypeUsers = 0;
+
+
+/**
+ * Helper that formats a decimal number in the range 0..9999.
+ *
+ * @returns The length of the formatted number.
+ * @param   pszBuf              Output buffer with sufficient space.
+ * @param   uNumber             The number to format.
+ */
+static unsigned vmmFormatTypeShortNumber(char *pszBuf, uint32_t uNumber)
+{
+    unsigned  off = 0;
+    if (uNumber >= 10)
+    {
+        if (uNumber >= 100)
+        {
+            if (uNumber >= 1000)
+                pszBuf[off++] = ((uNumber / 1000) % 10) + '0';
+            pszBuf[off++] = ((uNumber / 100) % 10) + '0';
+        }
+        pszBuf[off++] = ((uNumber / 10) % 10) + '0';
+    }
+    pszBuf[off++] = (uNumber % 10) + '0';
+    pszBuf[off] = '\0';
+    return off;
+}
+
+
+/**
+ * @callback_method_impl{FNRTSTRFORMATTYPE, vmsetcpu}
+ */
+static DECLCALLBACK(size_t) vmmFormatTypeVmCpuSet(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
+                                                  const char *pszType, void const *pvValue,
+                                                  int cchWidth, int cchPrecision, unsigned fFlags,
+                                                  void *pvUser)
+{
+    NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(fFlags);
+
+    PCVMCPUSET  pSet   = (PCVMCPUSET)pvValue;
+    uint32_t    cCpus  = 0;
+    uint32_t    iCpu   = RT_ELEMENTS(pSet->au32Bitmap) * 32;
+    while (iCpu--)
+        if (VMCPUSET_IS_PRESENT(pSet, iCpu))
+            cCpus++;
+
+    char szTmp[32];
+    AssertCompile(RT_ELEMENTS(pSet->au32Bitmap) * 32 < 999);
+    if (cCpus == 1)
+    {
+        iCpu = RT_ELEMENTS(pSet->au32Bitmap) * 32;
+        while (iCpu--)
+            if (VMCPUSET_IS_PRESENT(pSet, iCpu))
+            {
+                szTmp[0] = 'c';
+                szTmp[1] = 'p';
+                szTmp[2] = 'u';
+                return pfnOutput(pvArgOutput, szTmp, 3 + vmmFormatTypeShortNumber(&szTmp[3], iCpu));
+            }
+        cCpus = 0;
+    }
+    if (cCpus == 0)
+        return pfnOutput(pvArgOutput, RT_STR_TUPLE("<empty>"));
+    if (cCpus == RT_ELEMENTS(pSet->au32Bitmap) * 32)
+        return pfnOutput(pvArgOutput, RT_STR_TUPLE("<full>"));
+
+    /*
+     * Print cpus that are present: {1,2,7,9 ... }
+     */
+    size_t cchRet = pfnOutput(pvArgOutput, "{", 1);
+
+    cCpus = 0;
+    iCpu  = 0;
+    while (iCpu < RT_ELEMENTS(pSet->au32Bitmap) * 32)
+    {
+        if (VMCPUSET_IS_PRESENT(pSet, iCpu))
+        {
+            /* Output the first cpu number. */
+            int off = 0;
+            if (cCpus != 0)
+                szTmp[off++] = ',';
+            cCpus++;
+            off += vmmFormatTypeShortNumber(&szTmp[off], iCpu);
+
+            /* Check for sequence. */
+            uint32_t const iStart = ++iCpu;
+            while (   iCpu < RT_ELEMENTS(pSet->au32Bitmap) * 32
+                   && VMCPUSET_IS_PRESENT(pSet, iCpu))
+            {
+                iCpu++;
+                cCpus++;
+            }
+            if (iCpu != iStart)
+            {
+                szTmp[off++] = '-';
+                off += vmmFormatTypeShortNumber(&szTmp[off], iCpu);
+            }
+
+            /* Terminate and output. */
+            szTmp[off] = '\0';
+            cchRet += pfnOutput(pvArgOutput, szTmp, off);
+        }
+        iCpu++;
+    }
+
+    cchRet += pfnOutput(pvArgOutput, "}", 1);
+    NOREF(pvUser);
+    return cchRet;
+}
+
+
+/**
+ * Registers the VMM wide format types.
+ *
+ * Called by VMMR3Init, VMMR0Init and VMMRCInit.
+ */
+int vmmInitFormatTypes(void)
+{
+    int rc = VINF_SUCCESS;
+    if (ASMAtomicIncU32(&g_cFormatTypeUsers) == 1)
+        rc = RTStrFormatTypeRegister("vmcpuset", vmmFormatTypeVmCpuSet, NULL);
+    return rc;
+}
+
+
+/**
+ * Counterpart to vmmInitFormatTypes, called by VMMR3Term and VMMR0Term.
+ */
+void vmmTermFormatTypes(void)
+{
+    if (ASMAtomicDecU32(&g_cFormatTypeUsers) == 0)
+        RTStrFormatTypeDeregister("vmcpuset");
+}
+
+
+/**
+ * Gets the ID of the virtual CPU associated with the calling thread.
+ *
+ * @returns The CPU ID. NIL_VMCPUID if the thread isn't an EMT.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @internal
+ */
+VMMDECL(VMCPUID) VMMGetCpuId(PVMCC pVM)
+{
+#if defined(IN_RING3)
+    return VMR3GetVMCPUId(pVM);
+
+#elif defined(IN_RING0)
+    PVMCPUCC pVCpu = GVMMR0GetGVCpuByGVMandEMT(pVM, NIL_RTNATIVETHREAD);
+    return pVCpu ? pVCpu->idCpu : NIL_VMCPUID;
+
+#else /* RC: Always EMT(0) */
+    NOREF(pVM);
+    return 0;
+#endif
+}
+
+
+/**
+ * Returns the VMCPU of the calling EMT.
+ *
+ * @returns The VMCPU pointer. NULL if not an EMT.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @internal
+ */
+VMMDECL(PVMCPUCC) VMMGetCpu(PVMCC pVM)
+{
+#ifdef IN_RING3
+    VMCPUID idCpu = VMR3GetVMCPUId(pVM);
+    if (idCpu == NIL_VMCPUID)
+        return NULL;
+    Assert(idCpu < pVM->cCpus);
+    return VMCC_GET_CPU(pVM, idCpu);
+
+#elif defined(IN_RING0)
+    return GVMMR0GetGVCpuByGVMandEMT(pVM, NIL_RTNATIVETHREAD);
+
+#else /* RC: Always EMT(0) */
+    RT_NOREF(pVM);
+    return &g_VCpu0;
+#endif /* IN_RING0 */
+}
+
+
+/**
+ * Returns the VMCPU of the first EMT thread.
+ *
+ * @returns The VMCPU pointer.
+ * @param   pVM         The cross context VM structure.
+ * @internal
+ */
+VMMDECL(PVMCPUCC) VMMGetCpu0(PVMCC pVM)
+{
+    Assert(pVM->cCpus == 1);
+    return VMCC_GET_CPU_0(pVM);
+}
+
+
+/**
+ * Returns the VMCPU of the specified virtual CPU.
+ *
+ * @returns The VMCPU pointer. NULL if idCpu is invalid.
+ *
+ * @param   pVM         The cross context VM structure.
+ * @param   idCpu       The ID of the virtual CPU.
+ * @internal
+ */
+VMMDECL(PVMCPUCC) VMMGetCpuById(PVMCC pVM, RTCPUID idCpu)
+{
+    AssertReturn(idCpu < pVM->cCpus, NULL);
+    return VMCC_GET_CPU(pVM, idCpu);
+}
+
+
+/**
+ * Gets the VBOX_SVN_REV.
+ *
+ * This is just to avoid having to compile a bunch of big files
+ * and requires less Makefile mess.
+ *
+ * @returns VBOX_SVN_REV.
+ */
+VMM_INT_DECL(uint32_t) VMMGetSvnRev(void)
+{
+    return VBOX_SVN_REV;
+}
+
+
+/**
+ * Returns the build type for matching components.
+ *
+ * @returns Build type value.
+ */
+uint32_t vmmGetBuildType(void)
+{
+    uint32_t uRet = 0xbeef0000;
+#ifdef DEBUG
+    uRet |= RT_BIT_32(0);
+#endif
+#ifdef VBOX_WITH_STATISTICS
+    uRet |= RT_BIT_32(1);
+#endif
+    return uRet;
+}
+
diff --git a/src/VBox/VMM/VMMAll/VMMAllA.asm b/src/VBox/VMM/VMMAll/VMMAllA.asm
new file mode 100644
index 00000000..93a25e76
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/VMMAllA.asm
@@ -0,0 +1,93 @@
+; $Id: VMMAllA.asm $
+;; @file
+; VMM - All Contexts Assembly Routines.
+;
+
+;
+; Copyright (C) 2009-2023 Oracle and/or its affiliates.
+;
+; This file is part of VirtualBox base platform packages, as
+; available from https://www.virtualbox.org.
+;
+; This program is free software; you can redistribute it and/or
+; modify it under the terms of the GNU General Public License
+; as published by the Free Software Foundation, in version 3 of the
+; License.
+;
+; This program is distributed in the hope that it will be useful, but
+; WITHOUT ANY WARRANTY; without even the implied warranty of
+; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+; General Public License for more details.
+;
+; You should have received a copy of the GNU General Public License
+; along with this program; if not, see <https://www.gnu.org/licenses>.
+;
+; SPDX-License-Identifier: GPL-3.0-only
+;
+
+;*******************************************************************************
+;* Header Files                                                                *
+;*******************************************************************************
+%include "VBox/asmdefs.mac"
+
+
+;*******************************************************************************
+;* Defined Constants And Macros                                                *
+;*******************************************************************************
+%ifdef IN_RING3
+ %ifdef RT_ARCH_AMD64
+  %define VMM_TRASH_XMM_REGS
+ %endif
+%endif
+%ifdef IN_RING0
+ %ifdef RT_ARCH_AMD64
+  %ifdef RT_OS_WINDOWS
+   %define VMM_TRASH_XMM_REGS
+  %endif
+ %endif
+%endif
+%ifndef VMM_TRASH_XMM_REGS
+ %ifdef VBOX_WITH_KERNEL_USING_XMM
+  %define VMM_TRASH_XMM_REGS
+ %endif
+%endif
+
+BEGINCODE
+
+
+;;
+; Trashes the volatile XMM registers in the current ABI.
+;
+BEGINPROC VMMTrashVolatileXMMRegs
+%ifdef VMM_TRASH_XMM_REGS
+        push    xBP
+        mov     xBP, xSP
+
+        ; take whatever is on the stack.
+        and     xSP, ~15
+        sub     xSP, 80h
+
+        movdqa  xmm0, [xSP + 0]
+        movdqa  xmm1, [xSP + 010h]
+        movdqa  xmm2, [xSP + 020h]
+        movdqa  xmm3, [xSP + 030h]
+        movdqa  xmm4, [xSP + 040h]
+        movdqa  xmm5, [xSP + 050h]
+ %ifdef ASM_CALL64_GCC
+        movdqa  xmm6, [xSP + 060h]
+        movdqa  xmm7, [xSP + 070h]
+        movdqa  xmm8, [xSP + 000h]
+        movdqa  xmm9, [xSP + 010h]
+        movdqa  xmm10,[xSP + 020h]
+        movdqa  xmm11,[xSP + 030h]
+        movdqa  xmm12,[xSP + 040h]
+        movdqa  xmm13,[xSP + 050h]
+        movdqa  xmm14,[xSP + 060h]
+        movdqa  xmm15,[xSP + 070h]
+ %endif
+        leave
+%endif ; VMM_TRASH_XMM_REGS
+        xor     eax, eax                ; for good measure.
+        ret
+ENDPROC   VMMTrashVolatileXMMRegs
+
diff --git a/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h b/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h
new file mode 100644
index 00000000..85db8a47
--- /dev/null
+++ b/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h
@@ -0,0 +1,12003 @@
+/* $Id: VMXAllTemplate.cpp.h $ */
+/** @file
+ * HM VMX (Intel VT-x) - Code template for our own hypervisor and the NEM darwin backend using Apple's Hypervisor.framework.
+ */
+
+/*
+ * Copyright (C) 2012-2023 Oracle and/or its affiliates.
+ *
+ * This file is part of VirtualBox base platform packages, as
+ * available from https://www.virtualbox.org.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, in version 3 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses>.
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+
+/*********************************************************************************************************************************
+*   Defined Constants And Macros                                                                                                 *
+*********************************************************************************************************************************/
+#if !defined(VMX_VMCS_WRITE_16) || !defined(VMX_VMCS_WRITE_32) || !defined(VMX_VMCS_WRITE_64) || !defined(VMX_VMCS_WRITE_64)
+# error "At least one of the VMX_VMCS_WRITE_16, VMX_VMCS_WRITE_32, VMX_VMCS_WRITE_64 or VMX_VMCS_WRITE_64 is missing"
+#endif
+
+
+#if !defined(VMX_VMCS_READ_16) || !defined(VMX_VMCS_READ_32) || !defined(VMX_VMCS_READ_64) || !defined(VMX_VMCS_READ_64)
+# error "At least one of the VMX_VMCS_READ_16, VMX_VMCS_READ_32, VMX_VMCS_READ_64 or VMX_VMCS_READ_64 is missing"
+#endif
+
+/** Enables condensing of VMREAD instructions, see vmxHCReadToTransient(). */
+#define HMVMX_WITH_CONDENSED_VMREADS
+
+/** Use the function table. */
+#define HMVMX_USE_FUNCTION_TABLE
+
+/** Determine which tagged-TLB flush handler to use. */
+#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID             0
+#define HMVMX_FLUSH_TAGGED_TLB_EPT                  1
+#define HMVMX_FLUSH_TAGGED_TLB_VPID                 2
+#define HMVMX_FLUSH_TAGGED_TLB_NONE                 3
+
+/** Assert that all the given fields have been read from the VMCS. */
+#ifdef VBOX_STRICT
+# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
+        do { \
+            uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
+            Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
+        } while (0)
+#else
+# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
+#endif
+
+/**
+ * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
+ * guest using hardware-assisted VMX.
+ *
+ * This excludes state like GPRs (other than RSP) which are always are
+ * swapped and restored across the world-switch and also registers like EFER,
+ * MSR which cannot be modified by the guest without causing a VM-exit.
+ */
+#define HMVMX_CPUMCTX_EXTRN_ALL      (  CPUMCTX_EXTRN_RIP             \
+                                      | CPUMCTX_EXTRN_RFLAGS          \
+                                      | CPUMCTX_EXTRN_RSP             \
+                                      | CPUMCTX_EXTRN_SREG_MASK       \
+                                      | CPUMCTX_EXTRN_TABLE_MASK      \
+                                      | CPUMCTX_EXTRN_KERNEL_GS_BASE  \
+                                      | CPUMCTX_EXTRN_SYSCALL_MSRS    \
+                                      | CPUMCTX_EXTRN_SYSENTER_MSRS   \
+                                      | CPUMCTX_EXTRN_TSC_AUX         \
+                                      | CPUMCTX_EXTRN_OTHER_MSRS      \
+                                      | CPUMCTX_EXTRN_CR0             \
+                                      | CPUMCTX_EXTRN_CR3             \
+                                      | CPUMCTX_EXTRN_CR4             \
+                                      | CPUMCTX_EXTRN_DR7             \
+                                      | CPUMCTX_EXTRN_HWVIRT          \
+                                      | CPUMCTX_EXTRN_INHIBIT_INT     \
+                                      | CPUMCTX_EXTRN_INHIBIT_NMI)
+
+/**
+ * Exception bitmap mask for real-mode guests (real-on-v86).
+ *
+ * We need to intercept all exceptions manually except:
+ * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
+ *   due to bugs in Intel CPUs.
+ * - \#PF need not be intercepted even in real-mode if we have nested paging
+ * support.
+ */
+#define HMVMX_REAL_MODE_XCPT_MASK    (  RT_BIT(X86_XCPT_DE)  /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI)   \
+                                      | RT_BIT(X86_XCPT_BP)             | RT_BIT(X86_XCPT_OF)    | RT_BIT(X86_XCPT_BR)    \
+                                      | RT_BIT(X86_XCPT_UD)             | RT_BIT(X86_XCPT_NM)    | RT_BIT(X86_XCPT_DF)    \
+                                      | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS)    | RT_BIT(X86_XCPT_NP)    \
+                                      | RT_BIT(X86_XCPT_SS)             | RT_BIT(X86_XCPT_GP)   /* RT_BIT(X86_XCPT_PF) */ \
+                                      | RT_BIT(X86_XCPT_MF)  /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC)    \
+                                      | RT_BIT(X86_XCPT_XF))
+
+/** Maximum VM-instruction error number. */
+#define HMVMX_INSTR_ERROR_MAX        28
+
+/** Profiling macro. */
+#ifdef HM_PROFILE_EXIT_DISPATCH
+# define HMVMX_START_EXIT_DISPATCH_PROF()           STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
+# define HMVMX_STOP_EXIT_DISPATCH_PROF()            STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
+#else
+# define HMVMX_START_EXIT_DISPATCH_PROF()           do { } while (0)
+# define HMVMX_STOP_EXIT_DISPATCH_PROF()            do { } while (0)
+#endif
+
+#ifndef IN_NEM_DARWIN
+/** Assert that preemption is disabled or covered by thread-context hooks. */
+# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu)          Assert(   VMMR0ThreadCtxHookIsEnabled((a_pVCpu))   \
+                                                            || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
+
+/** Assert that we haven't migrated CPUs when thread-context hooks are not
+ *  used. */
+# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu)              AssertMsg(   VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
+                                                               || (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
+                                                               ("Illegal migration! Entered on CPU %u Current %u\n", \
+                                                               (a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()))
+#else
+# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu)          do { } while (0)
+# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu)              do { } while (0)
+#endif
+
+/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
+ *  context. */
+#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz)  AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
+                                                              ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
+                                                              (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
+
+/** Log the VM-exit reason with an easily visible marker to identify it in a
+ *  potential sea of logging data. */
+#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
+    do { \
+        Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
+             HMGetVmxExitName(a_uExitReason))); \
+    } while (0) \
+
+
+/*********************************************************************************************************************************
+*   Structures and Typedefs                                                                                                      *
+*********************************************************************************************************************************/
+/**
+ * Memory operand read or write access.
+ */
+typedef enum VMXMEMACCESS
+{
+    VMXMEMACCESS_READ  = 0,
+    VMXMEMACCESS_WRITE = 1
+} VMXMEMACCESS;
+
+
+/**
+ * VMX VM-exit handler.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+#ifndef HMVMX_USE_FUNCTION_TABLE
+typedef VBOXSTRICTRC               FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
+#else
+typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient));
+/** Pointer to VM-exit handler. */
+typedef FNVMXEXITHANDLER          *PFNVMXEXITHANDLER;
+#endif
+
+/**
+ * VMX VM-exit handler, non-strict status code.
+ *
+ * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
+ *
+ * @returns VBox status code, no informational status code returned.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
+ *          use of that status code will be replaced with VINF_EM_SOMETHING
+ *          later when switching over to IEM.
+ */
+#ifndef HMVMX_USE_FUNCTION_TABLE
+typedef int                        FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
+#else
+typedef FNVMXEXITHANDLER           FNVMXEXITHANDLERNSRC;
+#endif
+
+
+/*********************************************************************************************************************************
+*   Internal Functions                                                                                                           *
+*********************************************************************************************************************************/
+#ifndef HMVMX_USE_FUNCTION_TABLE
+DECLINLINE(VBOXSTRICTRC)           vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
+# define HMVMX_EXIT_DECL           DECLINLINE(VBOXSTRICTRC)
+# define HMVMX_EXIT_NSRC_DECL      DECLINLINE(int)
+#else
+# define HMVMX_EXIT_DECL           static DECLCALLBACK(VBOXSTRICTRC)
+# define HMVMX_EXIT_NSRC_DECL      HMVMX_EXIT_DECL
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+DECLINLINE(VBOXSTRICTRC)           vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
+#endif
+
+static int vmxHCImportGuestStateEx(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
+
+/** @name VM-exit handler prototypes.
+ * @{
+ */
+static FNVMXEXITHANDLER            vmxHCExitXcptOrNmi;
+static FNVMXEXITHANDLER            vmxHCExitExtInt;
+static FNVMXEXITHANDLER            vmxHCExitTripleFault;
+static FNVMXEXITHANDLERNSRC        vmxHCExitIntWindow;
+static FNVMXEXITHANDLERNSRC        vmxHCExitNmiWindow;
+static FNVMXEXITHANDLER            vmxHCExitTaskSwitch;
+static FNVMXEXITHANDLER            vmxHCExitCpuid;
+static FNVMXEXITHANDLER            vmxHCExitGetsec;
+static FNVMXEXITHANDLER            vmxHCExitHlt;
+static FNVMXEXITHANDLERNSRC        vmxHCExitInvd;
+static FNVMXEXITHANDLER            vmxHCExitInvlpg;
+static FNVMXEXITHANDLER            vmxHCExitRdpmc;
+static FNVMXEXITHANDLER            vmxHCExitVmcall;
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+static FNVMXEXITHANDLER            vmxHCExitVmclear;
+static FNVMXEXITHANDLER            vmxHCExitVmlaunch;
+static FNVMXEXITHANDLER            vmxHCExitVmptrld;
+static FNVMXEXITHANDLER            vmxHCExitVmptrst;
+static FNVMXEXITHANDLER            vmxHCExitVmread;
+static FNVMXEXITHANDLER            vmxHCExitVmresume;
+static FNVMXEXITHANDLER            vmxHCExitVmwrite;
+static FNVMXEXITHANDLER            vmxHCExitVmxoff;
+static FNVMXEXITHANDLER            vmxHCExitVmxon;
+static FNVMXEXITHANDLER            vmxHCExitInvvpid;
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+static FNVMXEXITHANDLER            vmxHCExitInvept;
+# endif
+#endif
+static FNVMXEXITHANDLER            vmxHCExitRdtsc;
+static FNVMXEXITHANDLER            vmxHCExitMovCRx;
+static FNVMXEXITHANDLER            vmxHCExitMovDRx;
+static FNVMXEXITHANDLER            vmxHCExitIoInstr;
+static FNVMXEXITHANDLER            vmxHCExitRdmsr;
+static FNVMXEXITHANDLER            vmxHCExitWrmsr;
+static FNVMXEXITHANDLER            vmxHCExitMwait;
+static FNVMXEXITHANDLER            vmxHCExitMtf;
+static FNVMXEXITHANDLER            vmxHCExitMonitor;
+static FNVMXEXITHANDLER            vmxHCExitPause;
+static FNVMXEXITHANDLERNSRC        vmxHCExitTprBelowThreshold;
+static FNVMXEXITHANDLER            vmxHCExitApicAccess;
+static FNVMXEXITHANDLER            vmxHCExitEptViolation;
+static FNVMXEXITHANDLER            vmxHCExitEptMisconfig;
+static FNVMXEXITHANDLER            vmxHCExitRdtscp;
+static FNVMXEXITHANDLER            vmxHCExitPreemptTimer;
+static FNVMXEXITHANDLERNSRC        vmxHCExitWbinvd;
+static FNVMXEXITHANDLER            vmxHCExitXsetbv;
+static FNVMXEXITHANDLER            vmxHCExitInvpcid;
+#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+static FNVMXEXITHANDLERNSRC        vmxHCExitSetPendingXcptUD;
+#endif
+static FNVMXEXITHANDLERNSRC        vmxHCExitErrInvalidGuestState;
+static FNVMXEXITHANDLERNSRC        vmxHCExitErrUnexpected;
+/** @} */
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/** @name Nested-guest VM-exit handler prototypes.
+ * @{
+ */
+static FNVMXEXITHANDLER            vmxHCExitXcptOrNmiNested;
+static FNVMXEXITHANDLER            vmxHCExitTripleFaultNested;
+static FNVMXEXITHANDLERNSRC        vmxHCExitIntWindowNested;
+static FNVMXEXITHANDLERNSRC        vmxHCExitNmiWindowNested;
+static FNVMXEXITHANDLER            vmxHCExitTaskSwitchNested;
+static FNVMXEXITHANDLER            vmxHCExitHltNested;
+static FNVMXEXITHANDLER            vmxHCExitInvlpgNested;
+static FNVMXEXITHANDLER            vmxHCExitRdpmcNested;
+static FNVMXEXITHANDLER            vmxHCExitVmreadVmwriteNested;
+static FNVMXEXITHANDLER            vmxHCExitRdtscNested;
+static FNVMXEXITHANDLER            vmxHCExitMovCRxNested;
+static FNVMXEXITHANDLER            vmxHCExitMovDRxNested;
+static FNVMXEXITHANDLER            vmxHCExitIoInstrNested;
+static FNVMXEXITHANDLER            vmxHCExitRdmsrNested;
+static FNVMXEXITHANDLER            vmxHCExitWrmsrNested;
+static FNVMXEXITHANDLER            vmxHCExitMwaitNested;
+static FNVMXEXITHANDLER            vmxHCExitMtfNested;
+static FNVMXEXITHANDLER            vmxHCExitMonitorNested;
+static FNVMXEXITHANDLER            vmxHCExitPauseNested;
+static FNVMXEXITHANDLERNSRC        vmxHCExitTprBelowThresholdNested;
+static FNVMXEXITHANDLER            vmxHCExitApicAccessNested;
+static FNVMXEXITHANDLER            vmxHCExitApicWriteNested;
+static FNVMXEXITHANDLER            vmxHCExitVirtEoiNested;
+static FNVMXEXITHANDLER            vmxHCExitRdtscpNested;
+static FNVMXEXITHANDLERNSRC        vmxHCExitWbinvdNested;
+static FNVMXEXITHANDLER            vmxHCExitInvpcidNested;
+static FNVMXEXITHANDLERNSRC        vmxHCExitErrInvalidGuestStateNested;
+static FNVMXEXITHANDLER            vmxHCExitInstrNested;
+static FNVMXEXITHANDLER            vmxHCExitInstrWithInfoNested;
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+static FNVMXEXITHANDLER            vmxHCExitEptViolationNested;
+static FNVMXEXITHANDLER            vmxHCExitEptMisconfigNested;
+# endif
+/** @} */
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/*********************************************************************************************************************************
+*   Global Variables                                                                                                             *
+*********************************************************************************************************************************/
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Array of all VMCS fields.
+ * Any fields added to the VT-x spec. should be added here.
+ *
+ * Currently only used to derive shadow VMCS fields for hardware-assisted execution
+ * of nested-guests.
+ */
+static const uint32_t g_aVmcsFields[] =
+{
+    /* 16-bit control fields. */
+    VMX_VMCS16_VPID,
+    VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
+    VMX_VMCS16_EPTP_INDEX,
+    VMX_VMCS16_HLAT_PREFIX_SIZE,
+
+    /* 16-bit guest-state fields. */
+    VMX_VMCS16_GUEST_ES_SEL,
+    VMX_VMCS16_GUEST_CS_SEL,
+    VMX_VMCS16_GUEST_SS_SEL,
+    VMX_VMCS16_GUEST_DS_SEL,
+    VMX_VMCS16_GUEST_FS_SEL,
+    VMX_VMCS16_GUEST_GS_SEL,
+    VMX_VMCS16_GUEST_LDTR_SEL,
+    VMX_VMCS16_GUEST_TR_SEL,
+    VMX_VMCS16_GUEST_INTR_STATUS,
+    VMX_VMCS16_GUEST_PML_INDEX,
+
+    /* 16-bits host-state fields. */
+    VMX_VMCS16_HOST_ES_SEL,
+    VMX_VMCS16_HOST_CS_SEL,
+    VMX_VMCS16_HOST_SS_SEL,
+    VMX_VMCS16_HOST_DS_SEL,
+    VMX_VMCS16_HOST_FS_SEL,
+    VMX_VMCS16_HOST_GS_SEL,
+    VMX_VMCS16_HOST_TR_SEL,
+
+    /* 64-bit control fields. */
+    VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
+    VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
+    VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
+    VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
+    VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
+    VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
+    VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
+    VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
+    VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
+    VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
+    VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
+    VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
+    VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
+    VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
+    VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
+    VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
+    VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
+    VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
+    VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
+    VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
+    VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
+    VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
+    VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
+    VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
+    VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
+    VMX_VMCS64_CTRL_EPTP_FULL,
+    VMX_VMCS64_CTRL_EPTP_HIGH,
+    VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
+    VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
+    VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
+    VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
+    VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
+    VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
+    VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
+    VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
+    VMX_VMCS64_CTRL_EPTP_LIST_FULL,
+    VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
+    VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
+    VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
+    VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL,
+    VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH,
+    VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
+    VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
+    VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_SPPTP_FULL,
+    VMX_VMCS64_CTRL_SPPTP_HIGH,
+    VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
+    VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
+    VMX_VMCS64_CTRL_PROC_EXEC3_FULL,
+    VMX_VMCS64_CTRL_PROC_EXEC3_HIGH,
+    VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL,
+    VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_PCONFIG_EXITING_BITMAP_FULL,
+    VMX_VMCS64_CTRL_PCONFIG_EXITING_BITMAP_HIGH,
+    VMX_VMCS64_CTRL_HLAT_PTR_FULL,
+    VMX_VMCS64_CTRL_HLAT_PTR_HIGH,
+    VMX_VMCS64_CTRL_EXIT2_FULL,
+    VMX_VMCS64_CTRL_EXIT2_HIGH,
+
+    /* 64-bit read-only data fields. */
+    VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
+    VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,
+
+    /* 64-bit guest-state fields. */
+    VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
+    VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
+    VMX_VMCS64_GUEST_DEBUGCTL_FULL,
+    VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
+    VMX_VMCS64_GUEST_PAT_FULL,
+    VMX_VMCS64_GUEST_PAT_HIGH,
+    VMX_VMCS64_GUEST_EFER_FULL,
+    VMX_VMCS64_GUEST_EFER_HIGH,
+    VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
+    VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
+    VMX_VMCS64_GUEST_PDPTE0_FULL,
+    VMX_VMCS64_GUEST_PDPTE0_HIGH,
+    VMX_VMCS64_GUEST_PDPTE1_FULL,
+    VMX_VMCS64_GUEST_PDPTE1_HIGH,
+    VMX_VMCS64_GUEST_PDPTE2_FULL,
+    VMX_VMCS64_GUEST_PDPTE2_HIGH,
+    VMX_VMCS64_GUEST_PDPTE3_FULL,
+    VMX_VMCS64_GUEST_PDPTE3_HIGH,
+    VMX_VMCS64_GUEST_BNDCFGS_FULL,
+    VMX_VMCS64_GUEST_BNDCFGS_HIGH,
+    VMX_VMCS64_GUEST_RTIT_CTL_FULL,
+    VMX_VMCS64_GUEST_RTIT_CTL_HIGH,
+    VMX_VMCS64_GUEST_PKRS_FULL,
+    VMX_VMCS64_GUEST_PKRS_HIGH,
+
+    /* 64-bit host-state fields. */
+    VMX_VMCS64_HOST_PAT_FULL,
+    VMX_VMCS64_HOST_PAT_HIGH,
+    VMX_VMCS64_HOST_EFER_FULL,
+    VMX_VMCS64_HOST_EFER_HIGH,
+    VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
+    VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
+    VMX_VMCS64_HOST_PKRS_FULL,
+    VMX_VMCS64_HOST_PKRS_HIGH,
+
+    /* 32-bit control fields. */
+    VMX_VMCS32_CTRL_PIN_EXEC,
+    VMX_VMCS32_CTRL_PROC_EXEC,
+    VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
+    VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
+    VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
+    VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
+    VMX_VMCS32_CTRL_EXIT,
+    VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
+    VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
+    VMX_VMCS32_CTRL_ENTRY,
+    VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
+    VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
+    VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
+    VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
+    VMX_VMCS32_CTRL_TPR_THRESHOLD,
+    VMX_VMCS32_CTRL_PROC_EXEC2,
+    VMX_VMCS32_CTRL_PLE_GAP,
+    VMX_VMCS32_CTRL_PLE_WINDOW,
+
+    /* 32-bits read-only fields. */
+    VMX_VMCS32_RO_VM_INSTR_ERROR,
+    VMX_VMCS32_RO_EXIT_REASON,
+    VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
+    VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
+    VMX_VMCS32_RO_IDT_VECTORING_INFO,
+    VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
+    VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
+    VMX_VMCS32_RO_EXIT_INSTR_INFO,
+
+    /* 32-bit guest-state fields. */
+    VMX_VMCS32_GUEST_ES_LIMIT,
+    VMX_VMCS32_GUEST_CS_LIMIT,
+    VMX_VMCS32_GUEST_SS_LIMIT,
+    VMX_VMCS32_GUEST_DS_LIMIT,
+    VMX_VMCS32_GUEST_FS_LIMIT,
+    VMX_VMCS32_GUEST_GS_LIMIT,
+    VMX_VMCS32_GUEST_LDTR_LIMIT,
+    VMX_VMCS32_GUEST_TR_LIMIT,
+    VMX_VMCS32_GUEST_GDTR_LIMIT,
+    VMX_VMCS32_GUEST_IDTR_LIMIT,
+    VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
+    VMX_VMCS32_GUEST_INT_STATE,
+    VMX_VMCS32_GUEST_ACTIVITY_STATE,
+    VMX_VMCS32_GUEST_SMBASE,
+    VMX_VMCS32_GUEST_SYSENTER_CS,
+    VMX_VMCS32_PREEMPT_TIMER_VALUE,
+
+    /* 32-bit host-state fields. */
+    VMX_VMCS32_HOST_SYSENTER_CS,
+
+    /* Natural-width control fields. */
+    VMX_VMCS_CTRL_CR0_MASK,
+    VMX_VMCS_CTRL_CR4_MASK,
+    VMX_VMCS_CTRL_CR0_READ_SHADOW,
+    VMX_VMCS_CTRL_CR4_READ_SHADOW,
+    VMX_VMCS_CTRL_CR3_TARGET_VAL0,
+    VMX_VMCS_CTRL_CR3_TARGET_VAL1,
+    VMX_VMCS_CTRL_CR3_TARGET_VAL2,
+    VMX_VMCS_CTRL_CR3_TARGET_VAL3,
+
+    /* Natural-width read-only data fields. */
+    VMX_VMCS_RO_EXIT_QUALIFICATION,
+    VMX_VMCS_RO_IO_RCX,
+    VMX_VMCS_RO_IO_RSI,
+    VMX_VMCS_RO_IO_RDI,
+    VMX_VMCS_RO_IO_RIP,
+    VMX_VMCS_RO_GUEST_LINEAR_ADDR,
+
+    /* Natural-width guest-state field */
+    VMX_VMCS_GUEST_CR0,
+    VMX_VMCS_GUEST_CR3,
+    VMX_VMCS_GUEST_CR4,
+    VMX_VMCS_GUEST_ES_BASE,
+    VMX_VMCS_GUEST_CS_BASE,
+    VMX_VMCS_GUEST_SS_BASE,
+    VMX_VMCS_GUEST_DS_BASE,
+    VMX_VMCS_GUEST_FS_BASE,
+    VMX_VMCS_GUEST_GS_BASE,
+    VMX_VMCS_GUEST_LDTR_BASE,
+    VMX_VMCS_GUEST_TR_BASE,
+    VMX_VMCS_GUEST_GDTR_BASE,
+    VMX_VMCS_GUEST_IDTR_BASE,
+    VMX_VMCS_GUEST_DR7,
+    VMX_VMCS_GUEST_RSP,
+    VMX_VMCS_GUEST_RIP,
+    VMX_VMCS_GUEST_RFLAGS,
+    VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
+    VMX_VMCS_GUEST_SYSENTER_ESP,
+    VMX_VMCS_GUEST_SYSENTER_EIP,
+    VMX_VMCS_GUEST_S_CET,
+    VMX_VMCS_GUEST_SSP,
+    VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR,
+
+    /* Natural-width host-state fields */
+    VMX_VMCS_HOST_CR0,
+    VMX_VMCS_HOST_CR3,
+    VMX_VMCS_HOST_CR4,
+    VMX_VMCS_HOST_FS_BASE,
+    VMX_VMCS_HOST_GS_BASE,
+    VMX_VMCS_HOST_TR_BASE,
+    VMX_VMCS_HOST_GDTR_BASE,
+    VMX_VMCS_HOST_IDTR_BASE,
+    VMX_VMCS_HOST_SYSENTER_ESP,
+    VMX_VMCS_HOST_SYSENTER_EIP,
+    VMX_VMCS_HOST_RSP,
+    VMX_VMCS_HOST_RIP,
+    VMX_VMCS_HOST_S_CET,
+    VMX_VMCS_HOST_SSP,
+    VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR
+};
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+#ifdef HMVMX_USE_FUNCTION_TABLE
+/**
+ * VMX_EXIT dispatch table.
+ */
+static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] =
+{
+    /*  0  VMX_EXIT_XCPT_OR_NMI             */  { vmxHCExitXcptOrNmi },
+    /*  1  VMX_EXIT_EXT_INT                 */  { vmxHCExitExtInt },
+    /*  2  VMX_EXIT_TRIPLE_FAULT            */  { vmxHCExitTripleFault },
+    /*  3  VMX_EXIT_INIT_SIGNAL             */  { vmxHCExitErrUnexpected },
+    /*  4  VMX_EXIT_SIPI                    */  { vmxHCExitErrUnexpected },
+    /*  5  VMX_EXIT_IO_SMI                  */  { vmxHCExitErrUnexpected },
+    /*  6  VMX_EXIT_SMI                     */  { vmxHCExitErrUnexpected },
+    /*  7  VMX_EXIT_INT_WINDOW              */  { vmxHCExitIntWindow },
+    /*  8  VMX_EXIT_NMI_WINDOW              */  { vmxHCExitNmiWindow },
+    /*  9  VMX_EXIT_TASK_SWITCH             */  { vmxHCExitTaskSwitch },
+    /* 10  VMX_EXIT_CPUID                   */  { vmxHCExitCpuid },
+    /* 11  VMX_EXIT_GETSEC                  */  { vmxHCExitGetsec },
+    /* 12  VMX_EXIT_HLT                     */  { vmxHCExitHlt },
+    /* 13  VMX_EXIT_INVD                    */  { vmxHCExitInvd },
+    /* 14  VMX_EXIT_INVLPG                  */  { vmxHCExitInvlpg },
+    /* 15  VMX_EXIT_RDPMC                   */  { vmxHCExitRdpmc },
+    /* 16  VMX_EXIT_RDTSC                   */  { vmxHCExitRdtsc },
+    /* 17  VMX_EXIT_RSM                     */  { vmxHCExitErrUnexpected },
+    /* 18  VMX_EXIT_VMCALL                  */  { vmxHCExitVmcall },
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* 19  VMX_EXIT_VMCLEAR                 */  { vmxHCExitVmclear },
+    /* 20  VMX_EXIT_VMLAUNCH                */  { vmxHCExitVmlaunch },
+    /* 21  VMX_EXIT_VMPTRLD                 */  { vmxHCExitVmptrld },
+    /* 22  VMX_EXIT_VMPTRST                 */  { vmxHCExitVmptrst },
+    /* 23  VMX_EXIT_VMREAD                  */  { vmxHCExitVmread },
+    /* 24  VMX_EXIT_VMRESUME                */  { vmxHCExitVmresume },
+    /* 25  VMX_EXIT_VMWRITE                 */  { vmxHCExitVmwrite },
+    /* 26  VMX_EXIT_VMXOFF                  */  { vmxHCExitVmxoff },
+    /* 27  VMX_EXIT_VMXON                   */  { vmxHCExitVmxon },
+#else
+    /* 19  VMX_EXIT_VMCLEAR                 */  { vmxHCExitSetPendingXcptUD },
+    /* 20  VMX_EXIT_VMLAUNCH                */  { vmxHCExitSetPendingXcptUD },
+    /* 21  VMX_EXIT_VMPTRLD                 */  { vmxHCExitSetPendingXcptUD },
+    /* 22  VMX_EXIT_VMPTRST                 */  { vmxHCExitSetPendingXcptUD },
+    /* 23  VMX_EXIT_VMREAD                  */  { vmxHCExitSetPendingXcptUD },
+    /* 24  VMX_EXIT_VMRESUME                */  { vmxHCExitSetPendingXcptUD },
+    /* 25  VMX_EXIT_VMWRITE                 */  { vmxHCExitSetPendingXcptUD },
+    /* 26  VMX_EXIT_VMXOFF                  */  { vmxHCExitSetPendingXcptUD },
+    /* 27  VMX_EXIT_VMXON                   */  { vmxHCExitSetPendingXcptUD },
+#endif
+    /* 28  VMX_EXIT_MOV_CRX                 */  { vmxHCExitMovCRx },
+    /* 29  VMX_EXIT_MOV_DRX                 */  { vmxHCExitMovDRx },
+    /* 30  VMX_EXIT_IO_INSTR                */  { vmxHCExitIoInstr },
+    /* 31  VMX_EXIT_RDMSR                   */  { vmxHCExitRdmsr },
+    /* 32  VMX_EXIT_WRMSR                   */  { vmxHCExitWrmsr },
+    /* 33  VMX_EXIT_ERR_INVALID_GUEST_STATE */  { vmxHCExitErrInvalidGuestState },
+    /* 34  VMX_EXIT_ERR_MSR_LOAD            */  { vmxHCExitErrUnexpected },
+    /* 35  UNDEFINED                        */  { vmxHCExitErrUnexpected },
+    /* 36  VMX_EXIT_MWAIT                   */  { vmxHCExitMwait },
+    /* 37  VMX_EXIT_MTF                     */  { vmxHCExitMtf },
+    /* 38  UNDEFINED                        */  { vmxHCExitErrUnexpected },
+    /* 39  VMX_EXIT_MONITOR                 */  { vmxHCExitMonitor },
+    /* 40  VMX_EXIT_PAUSE                   */  { vmxHCExitPause },
+    /* 41  VMX_EXIT_ERR_MACHINE_CHECK       */  { vmxHCExitErrUnexpected },
+    /* 42  UNDEFINED                        */  { vmxHCExitErrUnexpected },
+    /* 43  VMX_EXIT_TPR_BELOW_THRESHOLD     */  { vmxHCExitTprBelowThreshold },
+    /* 44  VMX_EXIT_APIC_ACCESS             */  { vmxHCExitApicAccess },
+    /* 45  VMX_EXIT_VIRTUALIZED_EOI         */  { vmxHCExitErrUnexpected },
+    /* 46  VMX_EXIT_GDTR_IDTR_ACCESS        */  { vmxHCExitErrUnexpected },
+    /* 47  VMX_EXIT_LDTR_TR_ACCESS          */  { vmxHCExitErrUnexpected },
+    /* 48  VMX_EXIT_EPT_VIOLATION           */  { vmxHCExitEptViolation },
+    /* 49  VMX_EXIT_EPT_MISCONFIG           */  { vmxHCExitEptMisconfig },
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+    /* 50  VMX_EXIT_INVEPT                  */  { vmxHCExitInvept },
+#else
+    /* 50  VMX_EXIT_INVEPT                  */  { vmxHCExitSetPendingXcptUD },
+#endif
+    /* 51  VMX_EXIT_RDTSCP                  */  { vmxHCExitRdtscp },
+    /* 52  VMX_EXIT_PREEMPT_TIMER           */  { vmxHCExitPreemptTimer },
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /* 53  VMX_EXIT_INVVPID                 */  { vmxHCExitInvvpid },
+#else
+    /* 53  VMX_EXIT_INVVPID                 */  { vmxHCExitSetPendingXcptUD },
+#endif
+    /* 54  VMX_EXIT_WBINVD                  */  { vmxHCExitWbinvd },
+    /* 55  VMX_EXIT_XSETBV                  */  { vmxHCExitXsetbv },
+    /* 56  VMX_EXIT_APIC_WRITE              */  { vmxHCExitErrUnexpected },
+    /* 57  VMX_EXIT_RDRAND                  */  { vmxHCExitErrUnexpected },
+    /* 58  VMX_EXIT_INVPCID                 */  { vmxHCExitInvpcid },
+    /* 59  VMX_EXIT_VMFUNC                  */  { vmxHCExitErrUnexpected },
+    /* 60  VMX_EXIT_ENCLS                   */  { vmxHCExitErrUnexpected },
+    /* 61  VMX_EXIT_RDSEED                  */  { vmxHCExitErrUnexpected },
+    /* 62  VMX_EXIT_PML_FULL                */  { vmxHCExitErrUnexpected },
+    /* 63  VMX_EXIT_XSAVES                  */  { vmxHCExitErrUnexpected },
+    /* 64  VMX_EXIT_XRSTORS                 */  { vmxHCExitErrUnexpected },
+    /* 65  UNDEFINED                        */  { vmxHCExitErrUnexpected },
+    /* 66  VMX_EXIT_SPP_EVENT               */  { vmxHCExitErrUnexpected },
+    /* 67  VMX_EXIT_UMWAIT                  */  { vmxHCExitErrUnexpected },
+    /* 68  VMX_EXIT_TPAUSE                  */  { vmxHCExitErrUnexpected },
+    /* 69  VMX_EXIT_LOADIWKEY               */  { vmxHCExitErrUnexpected },
+};
+#endif /* HMVMX_USE_FUNCTION_TABLE */
+
+#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
+static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
+{
+    /*  0 */ "(Not Used)",
+    /*  1 */ "VMCALL executed in VMX root operation.",
+    /*  2 */ "VMCLEAR with invalid physical address.",
+    /*  3 */ "VMCLEAR with VMXON pointer.",
+    /*  4 */ "VMLAUNCH with non-clear VMCS.",
+    /*  5 */ "VMRESUME with non-launched VMCS.",
+    /*  6 */ "VMRESUME after VMXOFF",
+    /*  7 */ "VM-entry with invalid control fields.",
+    /*  8 */ "VM-entry with invalid host state fields.",
+    /*  9 */ "VMPTRLD with invalid physical address.",
+    /* 10 */ "VMPTRLD with VMXON pointer.",
+    /* 11 */ "VMPTRLD with incorrect revision identifier.",
+    /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
+    /* 13 */ "VMWRITE to read-only VMCS component.",
+    /* 14 */ "(Not Used)",
+    /* 15 */ "VMXON executed in VMX root operation.",
+    /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
+    /* 17 */ "VM-entry with non-launched executing VMCS.",
+    /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
+    /* 19 */ "VMCALL with non-clear VMCS.",
+    /* 20 */ "VMCALL with invalid VM-exit control fields.",
+    /* 21 */ "(Not Used)",
+    /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
+    /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
+    /* 24 */ "VMCALL with invalid SMM-monitor features.",
+    /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
+    /* 26 */ "VM-entry with events blocked by MOV SS.",
+    /* 27 */ "(Not Used)",
+    /* 28 */ "Invalid operand to INVEPT/INVVPID."
+};
+#endif /* VBOX_STRICT && LOG_ENABLED */
+
+
+/**
+ * Gets the CR0 guest/host mask.
+ *
+ * These bits typically does not change through the lifetime of a VM. Any bit set in
+ * this mask is owned by the host/hypervisor and would cause a VM-exit when modified
+ * by the guest.
+ *
+ * @returns The CR0 guest/host mask.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static uint64_t vmxHCGetFixedCr0Mask(PCVMCPUCC pVCpu)
+{
+    /*
+     * Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
+     * to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
+     *
+     * Furthermore, modifications to any bits that are reserved/unspecified currently
+     * by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
+     * when future CPUs specify and use currently reserved/unspecified bits.
+     */
+    /** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
+     *        enmGuestMode to be in-sync with the current mode. See @bugref{6398}
+     *        and @bugref{6944}. */
+    PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    AssertCompile(RT_HI_U32(VMX_EXIT_HOST_CR0_IGNORE_MASK) == UINT32_C(0xffffffff));    /* Paranoia. */
+    return (  X86_CR0_PE
+            | X86_CR0_NE
+            | (VM_IS_VMX_NESTED_PAGING(pVM) ? 0 : X86_CR0_WP)
+            | X86_CR0_PG
+            | VMX_EXIT_HOST_CR0_IGNORE_MASK);
+}
+
+
+/**
+ * Gets the CR4 guest/host mask.
+ *
+ * These bits typically does not change through the lifetime of a VM. Any bit set in
+ * this mask is owned by the host/hypervisor and would cause a VM-exit when modified
+ * by the guest.
+ *
+ * @returns The CR4 guest/host mask.
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static uint64_t vmxHCGetFixedCr4Mask(PCVMCPUCC pVCpu)
+{
+    /*
+     * We construct a mask of all CR4 bits that the guest can modify without causing
+     * a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
+     * a VM-exit when the guest attempts to modify them when executing using
+     * hardware-assisted VMX.
+     *
+     * When a feature is not exposed to the guest (and may be present on the host),
+     * we want to intercept guest modifications to the bit so we can emulate proper
+     * behavior (e.g., #GP).
+     *
+     * Furthermore, only modifications to those bits that don't require immediate
+     * emulation is allowed. For e.g., PCIDE is excluded because the behavior
+     * depends on CR3 which might not always be the guest value while executing
+     * using hardware-assisted VMX.
+     */
+    PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    bool fFsGsBase    = pVM->cpum.ro.GuestFeatures.fFsGsBase;
+#ifdef IN_NEM_DARWIN
+    bool fXSaveRstor  = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
+#endif
+    bool fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;
+
+    /*
+     * Paranoia.
+     * Ensure features exposed to the guest are present on the host.
+     */
+    AssertStmt(!fFsGsBase    || g_CpumHostFeatures.s.fFsGsBase,    fFsGsBase = 0);
+#ifdef IN_NEM_DARWIN
+    AssertStmt(!fXSaveRstor  || g_CpumHostFeatures.s.fXSaveRstor,  fXSaveRstor = 0);
+#endif
+    AssertStmt(!fFxSaveRstor || g_CpumHostFeatures.s.fFxSaveRstor, fFxSaveRstor = 0);
+
+    uint64_t const fGstMask = X86_CR4_PVI
+                            | X86_CR4_TSD
+                            | X86_CR4_DE
+                            | X86_CR4_MCE
+                            | X86_CR4_PCE
+                            | X86_CR4_OSXMMEEXCPT
+                            | (fFsGsBase    ? X86_CR4_FSGSBASE : 0)
+#ifdef IN_NEM_DARWIN /* On native VT-x setting OSXSAVE must exit as we need to load guest XCR0 (see
+                        fLoadSaveGuestXcr0). These exits are not needed on Darwin as that's not our problem. */
+                            | (fXSaveRstor  ? X86_CR4_OSXSAVE  : 0)
+#endif
+                            | (fFxSaveRstor ? X86_CR4_OSFXSR   : 0);
+    return ~fGstMask;
+}
+
+
+/**
+ * Adds one or more exceptions to the exception bitmap and commits it to the current
+ * VMCS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uXcptMask       The exception(s) to add.
+ */
+static void vmxHCAddXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
+{
+    PVMXVMCSINFO pVmcsInfo   = pVmxTransient->pVmcsInfo;
+    uint32_t     uXcptBitmap = pVmcsInfo->u32XcptBitmap;
+    if ((uXcptBitmap & uXcptMask) != uXcptMask)
+    {
+        uXcptBitmap |= uXcptMask;
+        int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
+        AssertRC(rc);
+        pVmcsInfo->u32XcptBitmap = uXcptBitmap;
+    }
+}
+
+
+/**
+ * Adds an exception to the exception bitmap and commits it to the current VMCS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uXcpt           The exception to add.
+ */
+static void vmxHCAddXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
+{
+    Assert(uXcpt <= X86_XCPT_LAST);
+    vmxHCAddXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT_32(uXcpt));
+}
+
+
+/**
+ * Remove one or more exceptions from the exception bitmap and commits it to the
+ * current VMCS.
+ *
+ * This takes care of not removing the exception intercept if a nested-guest
+ * requires the exception to be intercepted.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uXcptMask       The exception(s) to remove.
+ */
+static int vmxHCRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
+{
+    PVMXVMCSINFO pVmcsInfo   = pVmxTransient->pVmcsInfo;
+    uint32_t     uXcptBitmap = pVmcsInfo->u32XcptBitmap;
+    if (uXcptBitmap & uXcptMask)
+    {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        if (!pVmxTransient->fIsNestedGuest)
+        { /* likely */ }
+        else
+            uXcptMask &= ~pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap;
+#endif
+#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
+        uXcptMask &= ~(  RT_BIT(X86_XCPT_BP)
+                       | RT_BIT(X86_XCPT_DE)
+                       | RT_BIT(X86_XCPT_NM)
+                       | RT_BIT(X86_XCPT_TS)
+                       | RT_BIT(X86_XCPT_UD)
+                       | RT_BIT(X86_XCPT_NP)
+                       | RT_BIT(X86_XCPT_SS)
+                       | RT_BIT(X86_XCPT_GP)
+                       | RT_BIT(X86_XCPT_PF)
+                       | RT_BIT(X86_XCPT_MF));
+#elif defined(HMVMX_ALWAYS_TRAP_PF)
+        uXcptMask &= ~RT_BIT(X86_XCPT_PF);
+#endif
+        if (uXcptMask)
+        {
+            /* Validate we are not removing any essential exception intercepts. */
+#ifndef IN_NEM_DARWIN
+            Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging || !(uXcptMask & RT_BIT(X86_XCPT_PF)));
+#else
+            Assert(!(uXcptMask & RT_BIT(X86_XCPT_PF)));
+#endif
+            NOREF(pVCpu);
+            Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
+            Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
+
+            /* Remove it from the exception bitmap. */
+            uXcptBitmap &= ~uXcptMask;
+
+            /* Commit and update the cache if necessary. */
+            if (pVmcsInfo->u32XcptBitmap != uXcptBitmap)
+            {
+                int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
+                AssertRC(rc);
+                pVmcsInfo->u32XcptBitmap = uXcptBitmap;
+            }
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Remove an exceptions from the exception bitmap and commits it to the current
+ * VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uXcpt           The exception to remove.
+ */
+static int vmxHCRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
+{
+    return vmxHCRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
+}
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+
+/**
+ * Loads the shadow VMCS specified by the VMCS info. object.
+ *
+ * @returns VBox status code.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Can be called with interrupts disabled.
+ */
+static int vmxHCLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
+
+    int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
+    if (RT_SUCCESS(rc))
+        pVmcsInfo->fShadowVmcsState |= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
+    return rc;
+}
+
+
+/**
+ * Clears the shadow VMCS specified by the VMCS info. object.
+ *
+ * @returns VBox status code.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Can be called with interrupts disabled.
+ */
+static int vmxHCClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
+
+    int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
+    if (RT_SUCCESS(rc))
+        pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
+    return rc;
+}
+
+
+/**
+ * Switches from and to the specified VMCSes.
+ *
+ * @returns VBox status code.
+ * @param   pVmcsInfoFrom   The VMCS info. object we are switching from.
+ * @param   pVmcsInfoTo     The VMCS info. object we are switching to.
+ *
+ * @remarks Called with interrupts disabled.
+ */
+static int vmxHCSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
+{
+    /*
+     * Clear the VMCS we are switching out if it has not already been cleared.
+     * This will sync any CPU internal data back to the VMCS.
+     */
+    if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
+    {
+        int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
+        if (RT_SUCCESS(rc))
+        {
+            /*
+             * The shadow VMCS, if any, would not be active at this point since we
+             * would have cleared it while importing the virtual hardware-virtualization
+             * state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
+             * clear the shadow VMCS here, just assert for safety.
+             */
+            Assert(!pVmcsInfoFrom->pvShadowVmcs || pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
+        }
+        else
+            return rc;
+    }
+
+    /*
+     * Clear the VMCS we are switching to if it has not already been cleared.
+     * This will initialize the VMCS launch state to "clear" required for loading it.
+     *
+     * See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
+     */
+    if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
+    {
+        int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
+        if (RT_SUCCESS(rc))
+        { /* likely */ }
+        else
+            return rc;
+    }
+
+    /*
+     * Finally, load the VMCS we are switching to.
+     */
+    return hmR0VmxLoadVmcs(pVmcsInfoTo);
+}
+
+
+/**
+ * Switches between the guest VMCS and the nested-guest VMCS as specified by the
+ * caller.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu                   The cross context virtual CPU structure.
+ * @param   fSwitchToNstGstVmcs     Whether to switch to the nested-guest VMCS (pass
+ *                                  true) or guest VMCS (pass false).
+ */
+static int vmxHCSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
+{
+    /* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
+    HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
+
+    PVMXVMCSINFO pVmcsInfoFrom;
+    PVMXVMCSINFO pVmcsInfoTo;
+    if (fSwitchToNstGstVmcs)
+    {
+        pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfo;
+        pVmcsInfoTo   = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
+    }
+    else
+    {
+        pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
+        pVmcsInfoTo   = &pVCpu->hmr0.s.vmx.VmcsInfo;
+    }
+
+    /*
+     * Disable interrupts to prevent being preempted while we switch the current VMCS as the
+     * preemption hook code path acquires the current VMCS.
+     */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    int rc = vmxHCSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
+    if (RT_SUCCESS(rc))
+    {
+        pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs           = fSwitchToNstGstVmcs;
+        pVCpu->hm.s.vmx.fSwitchedToNstGstVmcsCopyForRing3 = fSwitchToNstGstVmcs;
+
+        /*
+         * If we are switching to a VMCS that was executed on a different host CPU or was
+         * never executed before, flag that we need to export the host state before executing
+         * guest/nested-guest code using hardware-assisted VMX.
+         *
+         * This could probably be done in a preemptible context since the preemption hook
+         * will flag the necessary change in host context. However, since preemption is
+         * already disabled and to avoid making assumptions about host specific code in
+         * RTMpCpuId when called with preemption enabled, we'll do this while preemption is
+         * disabled.
+         */
+        if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
+        { /* likely */ }
+        else
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);
+
+        ASMSetFlags(fEFlags);
+
+        /*
+         * We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
+         * flag that we need to update the host MSR values there. Even if we decide in the
+         * future to share the VM-exit MSR-store area page between the guest and nested-guest,
+         * if its content differs, we would have to update the host MSRs anyway.
+         */
+        pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
+    }
+    else
+        ASMSetFlags(fEFlags);
+    return rc;
+}
+
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+#ifdef VBOX_STRICT
+
+/**
+ * Reads the VM-entry interruption-information field from the VMCS into the VMX
+ * transient structure.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) vmxHCReadEntryIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
+    AssertRC(rc);
+}
+
+
+/**
+ * Reads the VM-entry exception error code field from the VMCS into
+ * the VMX transient structure.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) vmxHCReadEntryXcptErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
+    AssertRC(rc);
+}
+
+
+/**
+ * Reads the VM-entry exception error code field from the VMCS into
+ * the VMX transient structure.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(void) vmxHCReadEntryInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
+    AssertRC(rc);
+}
+
+#endif /* VBOX_STRICT */
+
+
+/**
+ * Reads VMCS fields into the VMXTRANSIENT structure, slow path version.
+ *
+ * Don't call directly unless the it's likely that some or all of the fields
+ * given in @a a_fReadMask have already been read.
+ *
+ * @tparam  a_fReadMask     The fields to read.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+template<uint32_t const a_fReadMask>
+static void vmxHCReadToTransientSlow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    AssertCompile((a_fReadMask & ~(  HMVMX_READ_EXIT_QUALIFICATION
+                                   | HMVMX_READ_EXIT_INSTR_LEN
+                                   | HMVMX_READ_EXIT_INSTR_INFO
+                                   | HMVMX_READ_IDT_VECTORING_INFO
+                                   | HMVMX_READ_IDT_VECTORING_ERROR_CODE
+                                   | HMVMX_READ_EXIT_INTERRUPTION_INFO
+                                   | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
+                                   | HMVMX_READ_GUEST_LINEAR_ADDR
+                                   | HMVMX_READ_GUEST_PHYSICAL_ADDR
+                                   | HMVMX_READ_GUEST_PENDING_DBG_XCPTS
+                                   )) == 0);
+
+    if ((pVmxTransient->fVmcsFieldsRead & a_fReadMask) != a_fReadMask)
+    {
+        uint32_t const fVmcsFieldsRead = pVmxTransient->fVmcsFieldsRead;
+
+        if (   (a_fReadMask      & HMVMX_READ_EXIT_QUALIFICATION)
+            && !(fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
+        {
+            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION,          &pVmxTransient->uExitQual);
+            AssertRC(rc);
+        }
+        if (    (a_fReadMask     & HMVMX_READ_EXIT_INSTR_LEN)
+            && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH,         &pVmxTransient->cbExitInstr);
+            AssertRC(rc);
+        }
+        if (   (a_fReadMask      & HMVMX_READ_EXIT_INSTR_INFO)
+            && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO,           &pVmxTransient->ExitInstrInfo.u);
+            AssertRC(rc);
+        }
+        if (   (a_fReadMask      & HMVMX_READ_IDT_VECTORING_INFO)
+            && !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO,        &pVmxTransient->uIdtVectoringInfo);
+            AssertRC(rc);
+        }
+        if (   (a_fReadMask      & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
+            && !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,  &pVmxTransient->uIdtVectoringErrorCode);
+            AssertRC(rc);
+        }
+        if (   (a_fReadMask      & HMVMX_READ_EXIT_INTERRUPTION_INFO)
+            && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,    &pVmxTransient->uExitIntInfo);
+            AssertRC(rc);
+        }
+        if (    (a_fReadMask     & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
+            && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
+            AssertRC(rc);
+        }
+        if (   (a_fReadMask      & HMVMX_READ_GUEST_LINEAR_ADDR)
+            && !(fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
+        {
+            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR,           &pVmxTransient->uGuestLinearAddr);
+            AssertRC(rc);
+        }
+        if (   (a_fReadMask      & HMVMX_READ_GUEST_PHYSICAL_ADDR)
+            && !(fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
+        {
+            int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,      &pVmxTransient->uGuestPhysicalAddr);
+            AssertRC(rc);
+        }
+        if (   (a_fReadMask      & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
+            && !(fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
+        {
+            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,      &pVmxTransient->uGuestPendingDbgXcpts);
+            AssertRC(rc);
+        }
+
+        pVmxTransient->fVmcsFieldsRead |= a_fReadMask;
+    }
+}
+
+
+/**
+ * Reads VMCS fields into the VMXTRANSIENT structure.
+ *
+ * This optimizes for the case where none of @a a_fReadMask has been read yet,
+ * generating an optimized read sequences w/o any conditionals between in
+ * non-strict builds.
+ *
+ * @tparam  a_fReadMask     The fields to read.  One or more of the
+ *                          HMVMX_READ_XXX fields ORed together.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+template<uint32_t const a_fReadMask>
+DECLINLINE(void) vmxHCReadToTransient(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    AssertCompile((a_fReadMask & ~(  HMVMX_READ_EXIT_QUALIFICATION
+                                   | HMVMX_READ_EXIT_INSTR_LEN
+                                   | HMVMX_READ_EXIT_INSTR_INFO
+                                   | HMVMX_READ_IDT_VECTORING_INFO
+                                   | HMVMX_READ_IDT_VECTORING_ERROR_CODE
+                                   | HMVMX_READ_EXIT_INTERRUPTION_INFO
+                                   | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
+                                   | HMVMX_READ_GUEST_LINEAR_ADDR
+                                   | HMVMX_READ_GUEST_PHYSICAL_ADDR
+                                   | HMVMX_READ_GUEST_PENDING_DBG_XCPTS
+                                   )) == 0);
+
+    if (RT_LIKELY(!(pVmxTransient->fVmcsFieldsRead & a_fReadMask)))
+    {
+        if (a_fReadMask & HMVMX_READ_EXIT_QUALIFICATION)
+        {
+            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION,          &pVmxTransient->uExitQual);
+            AssertRC(rc);
+        }
+        if (a_fReadMask & HMVMX_READ_EXIT_INSTR_LEN)
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH,         &pVmxTransient->cbExitInstr);
+            AssertRC(rc);
+        }
+        if (a_fReadMask & HMVMX_READ_EXIT_INSTR_INFO)
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO,           &pVmxTransient->ExitInstrInfo.u);
+            AssertRC(rc);
+        }
+        if (a_fReadMask & HMVMX_READ_IDT_VECTORING_INFO)
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO,        &pVmxTransient->uIdtVectoringInfo);
+            AssertRC(rc);
+        }
+        if (a_fReadMask & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,  &pVmxTransient->uIdtVectoringErrorCode);
+            AssertRC(rc);
+        }
+        if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_INFO)
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,    &pVmxTransient->uExitIntInfo);
+            AssertRC(rc);
+        }
+        if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
+        {
+            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
+            AssertRC(rc);
+        }
+        if (a_fReadMask & HMVMX_READ_GUEST_LINEAR_ADDR)
+        {
+            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR,           &pVmxTransient->uGuestLinearAddr);
+            AssertRC(rc);
+        }
+        if (a_fReadMask & HMVMX_READ_GUEST_PHYSICAL_ADDR)
+        {
+            int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,      &pVmxTransient->uGuestPhysicalAddr);
+            AssertRC(rc);
+        }
+        if (a_fReadMask & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
+        {
+            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,      &pVmxTransient->uGuestPendingDbgXcpts);
+            AssertRC(rc);
+        }
+
+        pVmxTransient->fVmcsFieldsRead |= a_fReadMask;
+    }
+    else
+    {
+        STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatReadToTransientFallback);
+        Log11Func(("a_fReadMask=%#x fVmcsFieldsRead=%#x => %#x - Taking inefficient code path!\n",
+                   a_fReadMask, pVmxTransient->fVmcsFieldsRead, a_fReadMask & pVmxTransient->fVmcsFieldsRead));
+        vmxHCReadToTransientSlow<a_fReadMask>(pVCpu, pVmxTransient);
+    }
+}
+
+
+#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
+/**
+ * Reads all relevant read-only VMCS fields into the VMX transient structure.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+static void vmxHCReadAllRoFieldsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION,             &pVmxTransient->uExitQual);
+    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH,            &pVmxTransient->cbExitInstr);
+    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO,              &pVmxTransient->ExitInstrInfo.u);
+    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO,           &pVmxTransient->uIdtVectoringInfo);
+    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,     &pVmxTransient->uIdtVectoringErrorCode);
+    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,       &pVmxTransient->uExitIntInfo);
+    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
+    rc    |= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR,              &pVmxTransient->uGuestLinearAddr);
+    rc    |= VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,         &pVmxTransient->uGuestPhysicalAddr);
+    AssertRC(rc);
+    pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION
+                                   |  HMVMX_READ_EXIT_INSTR_LEN
+                                   |  HMVMX_READ_EXIT_INSTR_INFO
+                                   |  HMVMX_READ_IDT_VECTORING_INFO
+                                   |  HMVMX_READ_IDT_VECTORING_ERROR_CODE
+                                   |  HMVMX_READ_EXIT_INTERRUPTION_INFO
+                                   |  HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
+                                   |  HMVMX_READ_GUEST_LINEAR_ADDR
+                                   |  HMVMX_READ_GUEST_PHYSICAL_ADDR;
+}
+#endif
+
+/**
+ * Verifies that our cached values of the VMCS fields are all consistent with
+ * what's actually present in the VMCS.
+ *
+ * @returns VBox status code.
+ * @retval  VINF_SUCCESS if all our caches match their respective VMCS fields.
+ * @retval  VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
+ *                                            VMCS content. HMCPU error-field is
+ *                                            updated, see VMX_VCI_XXX.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS info. object.
+ * @param   fIsNstGstVmcs   Whether this is a nested-guest VMCS.
+ */
+static int vmxHCCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
+{
+    const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";
+
+    uint32_t u32Val;
+    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
+                        ("%s entry controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
+                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_ENTRY,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXIT, &u32Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
+                        ("%s exit controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
+                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_EXIT,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
+                        ("%s pin controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
+                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PIN_EXEC,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
+                        ("%s proc controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
+                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
+    {
+        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
+        AssertRC(rc);
+        AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
+                            ("%s proc2 controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
+                            VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
+                            VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+    }
+
+    uint64_t u64Val;
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS)
+    {
+        rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_PROC_EXEC3_FULL, &u64Val);
+        AssertRC(rc);
+        AssertMsgReturnStmt(pVmcsInfo->u64ProcCtls3 == u64Val,
+                            ("%s proc3 controls mismatch: Cache=%#RX32 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64ProcCtls3, u64Val),
+                            VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC3,
+                            VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+    }
+
+    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
+                        ("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
+                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
+    AssertRC(rc);
+    AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
+                        ("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
+                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
+                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);
+
+    NOREF(pcszVmcs);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exports the guest state with appropriate VM-entry and VM-exit controls in the
+ * VMCS.
+ *
+ * This is typically required when the guest changes paging mode.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks Requires EFER.
+ * @remarks No-long-jump zone!!!
+ */
+static int vmxHCExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        PVMXVMCSINFO pVmcsInfo      = pVmxTransient->pVmcsInfo;
+
+        /*
+         * VM-entry controls.
+         */
+        {
+            uint32_t       fVal = g_HmMsrs.u.vmx.EntryCtls.n.allowed0;    /* Bits set here must be set in the VMCS. */
+            uint32_t const fZap = g_HmMsrs.u.vmx.EntryCtls.n.allowed1;    /* Bits cleared here must be cleared in the VMCS. */
+
+            /*
+             * Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
+             * The first VT-x capable CPUs only supported the 1-setting of this bit.
+             *
+             * For nested-guests, this is a mandatory VM-entry control. It's also
+             * required because we do not want to leak host bits to the nested-guest.
+             */
+            fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
+
+            /*
+             * Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
+             *
+             * For nested-guests, the "IA-32e mode guest" control we initialize with what is
+             * required to get the nested-guest working with hardware-assisted VMX execution.
+             * It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
+             * can skip intercepting changes to the EFER MSR. This is why it needs to be done
+             * here rather than while merging the guest VMCS controls.
+             */
+            if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
+            {
+                Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
+                fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
+            }
+            else
+                Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
+
+            /*
+             * If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
+             *
+             * For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
+             * regardless of whether the nested-guest VMCS specifies it because we are free to
+             * load whatever MSRs we require and we do not need to modify the guest visible copy
+             * of the VM-entry MSR load area.
+             */
+            if (   g_fHmVmxSupportsVmcsEfer
+#ifndef IN_NEM_DARWIN
+                && hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient)
+#endif
+                )
+                fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
+            else
+                Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
+
+            /*
+             * The following should -not- be set (since we're not in SMM mode):
+             * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
+             * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
+             */
+
+            /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
+             *        VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
+
+            if ((fVal & fZap) == fVal)
+            { /* likely */ }
+            else
+            {
+                Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
+                          g_HmMsrs.u.vmx.EntryCtls.n.allowed0, fVal, fZap));
+                VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_ENTRY;
+                return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+            }
+
+            /* Commit it to the VMCS. */
+            if (pVmcsInfo->u32EntryCtls != fVal)
+            {
+                int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, fVal);
+                AssertRC(rc);
+                pVmcsInfo->u32EntryCtls = fVal;
+            }
+        }
+
+        /*
+         * VM-exit controls.
+         */
+        {
+            uint32_t       fVal = g_HmMsrs.u.vmx.ExitCtls.n.allowed0;     /* Bits set here must be set in the VMCS. */
+            uint32_t const fZap = g_HmMsrs.u.vmx.ExitCtls.n.allowed1;     /* Bits cleared here must be cleared in the VMCS. */
+
+            /*
+             * Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
+             * supported the 1-setting of this bit.
+             *
+             * For nested-guests, we set the "save debug controls" as the converse
+             * "load debug controls" is mandatory for nested-guests anyway.
+             */
+            fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
+
+            /*
+             * Set the host long mode active (EFER.LMA) bit (which Intel calls
+             * "Host address-space size") if necessary. On VM-exit, VT-x sets both the
+             * host EFER.LMA and EFER.LME bit to this value. See assertion in
+             * vmxHCExportHostMsrs().
+             *
+             * For nested-guests, we always set this bit as we do not support 32-bit
+             * hosts.
+             */
+            fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
+
+#ifndef IN_NEM_DARWIN
+            /*
+             * If the VMCS EFER MSR fields are supported by the hardware, we use it.
+             *
+             * For nested-guests, we should use the "save IA32_EFER" control if we also
+             * used the "load IA32_EFER" control while exporting VM-entry controls.
+             */
+            if (   g_fHmVmxSupportsVmcsEfer
+                && hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
+            {
+                fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
+                     |  VMX_EXIT_CTLS_LOAD_EFER_MSR;
+            }
+#endif
+
+            /*
+             * Enable saving of the VMX-preemption timer value on VM-exit.
+             * For nested-guests, currently not exposed/used.
+             */
+            /** @todo r=bird: Measure performance hit because of this vs. always rewriting
+             *        the timer value. */
+            if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
+            {
+                Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
+                fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
+            }
+
+            /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
+            Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
+
+            /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
+             *        VMX_EXIT_CTLS_SAVE_PAT_MSR,
+             *        VMX_EXIT_CTLS_LOAD_PAT_MSR. */
+
+            if ((fVal & fZap) == fVal)
+            { /* likely */ }
+            else
+            {
+                Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
+                          g_HmMsrs.u.vmx.ExitCtls.n.allowed0, fVal, fZap));
+                VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_EXIT;
+                return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
+            }
+
+            /* Commit it to the VMCS. */
+            if (pVmcsInfo->u32ExitCtls != fVal)
+            {
+                int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT, fVal);
+                AssertRC(rc);
+                pVmcsInfo->u32ExitCtls = fVal;
+            }
+        }
+
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Sets the TPR threshold in the VMCS.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   pVmcsInfo           The VMCS info. object.
+ * @param   u32TprThreshold     The TPR threshold (task-priority class only).
+ */
+DECLINLINE(void) vmxHCApicSetTprThreshold(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
+{
+    Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK));         /* Bits 31:4 MBZ. */
+    Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+    RT_NOREF(pVmcsInfo);
+    int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
+    AssertRC(rc);
+}
+
+
+/**
+ * Exports the guest APIC TPR state into the VMCS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void vmxHCExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
+
+        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+        if (!pVmxTransient->fIsNestedGuest)
+        {
+            if (   PDMHasApic(pVCpu->CTX_SUFF(pVM))
+                && APICIsEnabled(pVCpu))
+            {
+                /*
+                 * Setup TPR shadowing.
+                 */
+                if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+                {
+                    bool    fPendingIntr  = false;
+                    uint8_t u8Tpr         = 0;
+                    uint8_t u8PendingIntr = 0;
+                    int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
+                    AssertRC(rc);
+
+                    /*
+                     * If there are interrupts pending but masked by the TPR, instruct VT-x to
+                     * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
+                     * priority of the pending interrupt so we can deliver the interrupt. If there
+                     * are no interrupts pending, set threshold to 0 to not cause any
+                     * TPR-below-threshold VM-exits.
+                     */
+                    uint32_t u32TprThreshold = 0;
+                    if (fPendingIntr)
+                    {
+                        /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
+                           (which is the Task-Priority Class). */
+                        const uint8_t u8PendingPriority = u8PendingIntr >> 4;
+                        const uint8_t u8TprPriority     = u8Tpr >> 4;
+                        if (u8PendingPriority <= u8TprPriority)
+                            u32TprThreshold = u8PendingPriority;
+                    }
+
+                    vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
+                }
+            }
+        }
+        /* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
+    }
+}
+
+
+/**
+ * Gets the guest interruptibility-state and updates related force-flags.
+ *
+ * @returns Guest's interruptibility-state.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static uint32_t vmxHCGetGuestIntrStateAndUpdateFFs(PVMCPUCC pVCpu)
+{
+    uint32_t fIntrState;
+
+    /*
+     * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
+     */
+    if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx))
+        fIntrState = 0;
+    else
+    {
+        /* If inhibition is active, RIP should've been imported from the VMCS already. */
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
+
+        if (CPUMIsInInterruptShadowAfterSs(&pVCpu->cpum.GstCtx))
+            fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
+        else
+        {
+            fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
+
+            /* Block-by-STI must not be set when interrupts are disabled. */
+            AssertStmt(pVCpu->cpum.GstCtx.eflags.Bits.u1IF, fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
+        }
+    }
+
+    /*
+     * Check if we should inhibit NMI delivery.
+     */
+    if (!CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx))
+    { /* likely */ }
+    else
+        fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
+
+    /*
+     * Validate.
+     */
+    /* We don't support block-by-SMI yet.*/
+    Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));
+
+    return fIntrState;
+}
+
+
+/**
+ * Exports the exception intercepts required for guest execution in the VMCS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void vmxHCExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
+    {
+        /* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
+        if (   !pVmxTransient->fIsNestedGuest
+            &&  VCPU_2_VMXSTATE(pVCpu).fGIMTrapXcptUD)
+            vmxHCAddXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
+        else
+            vmxHCRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
+
+        /* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
+    }
+}
+
+
+/**
+ * Exports the guest's RIP into the guest-state area in the VMCS.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void vmxHCExportGuestRip(PVMCPUCC pVCpu)
+{
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RIP)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
+
+        int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
+        AssertRC(rc);
+
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RIP);
+        Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
+    }
+}
+
+
+/**
+ * Exports the guest's RFLAGS into the guest-state area in the VMCS.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void vmxHCExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
+
+        /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits
+           of RFLAGS are reserved (MBZ).  We use bits 63:24 for internal purposes, so no need
+           to assert this, the CPUMX86EFLAGS/CPUMX86RFLAGS union masks these off for us.
+           Use 32-bit VMWRITE. */
+        uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
+        Assert((fEFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK);
+        AssertMsg(!(fEFlags & ~(X86_EFL_LIVE_MASK | X86_EFL_RA1_MASK)), ("%#x\n", fEFlags));
+
+#ifndef IN_NEM_DARWIN
+        /*
+         * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
+         * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
+         * can run the real-mode guest code under Virtual 8086 mode.
+         */
+        PVMXVMCSINFOSHARED pVmcsInfo = pVmxTransient->pVmcsInfo->pShared;
+        if (pVmcsInfo->RealMode.fRealOnV86Active)
+        {
+            Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
+            Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
+            Assert(!pVmxTransient->fIsNestedGuest);
+            pVmcsInfo->RealMode.Eflags.u32 = fEFlags;        /* Save the original eflags of the real-mode guest. */
+            fEFlags |= X86_EFL_VM;                           /* Set the Virtual 8086 mode bit. */
+            fEFlags &= ~X86_EFL_IOPL;                        /* Change IOPL to 0, otherwise certain instructions won't fault. */
+        }
+#else
+        RT_NOREF(pVmxTransient);
+#endif
+
+        int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, fEFlags);
+        AssertRC(rc);
+
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
+        Log4Func(("eflags=%#RX32\n", fEFlags));
+    }
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Copies the nested-guest VMCS to the shadow VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int vmxHCCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    PVMCC      const pVM         = pVCpu->CTX_SUFF(pVM);
+    PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    /*
+     * Disable interrupts so we don't get preempted while the shadow VMCS is the
+     * current VMCS, as we may try saving guest lazy MSRs.
+     *
+     * Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
+     * calling the import VMCS code which is currently performing the guest MSR reads
+     * (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
+     * and the rest of the VMX leave session machinery.
+     */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+
+    int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Copy all guest read/write VMCS fields.
+         *
+         * We don't check for VMWRITE failures here for performance reasons and
+         * because they are not expected to fail, barring irrecoverable conditions
+         * like hardware errors.
+         */
+        uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
+        for (uint32_t i = 0; i < cShadowVmcsFields; i++)
+        {
+            uint64_t       u64Val;
+            uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
+            IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
+            VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
+        }
+
+        /*
+         * If the host CPU supports writing all VMCS fields, copy the guest read-only
+         * VMCS fields, so the guest can VMREAD them without causing a VM-exit.
+         */
+        if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
+        {
+            uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
+            for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
+            {
+                uint64_t       u64Val;
+                uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsRoFields[i];
+                IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
+                VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
+            }
+        }
+
+        rc  = vmxHCClearShadowVmcs(pVmcsInfo);
+        rc |= hmR0VmxLoadVmcs(pVmcsInfo);
+    }
+
+    ASMSetFlags(fEFlags);
+    return rc;
+}
+
+
+/**
+ * Copies the shadow VMCS to the nested-guest VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Called with interrupts disabled.
+ */
+static int vmxHCCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
+    PVMCC const     pVM         = pVCpu->CTX_SUFF(pVM);
+    PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+
+    int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
+    if (RT_SUCCESS(rc))
+    {
+        /*
+         * Copy guest read/write fields from the shadow VMCS.
+         * Guest read-only fields cannot be modified, so no need to copy them.
+         *
+         * We don't check for VMREAD failures here for performance reasons and
+         * because they are not expected to fail, barring irrecoverable conditions
+         * like hardware errors.
+         */
+        uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
+        for (uint32_t i = 0; i < cShadowVmcsFields; i++)
+        {
+            uint64_t       u64Val;
+            uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
+            VMX_VMCS_READ_64(pVCpu, uVmcsField, &u64Val);
+            IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
+        }
+
+        rc  = vmxHCClearShadowVmcs(pVmcsInfo);
+        rc |= hmR0VmxLoadVmcs(pVmcsInfo);
+    }
+    return rc;
+}
+
+
+/**
+ * Enables VMCS shadowing for the given VMCS info. object.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void vmxHCEnableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
+    if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
+    {
+        Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
+        uProcCtls2 |= VMX_PROC_CTLS2_VMCS_SHADOWING;
+        int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);                            AssertRC(rc);
+        rc     = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);  AssertRC(rc);
+        pVmcsInfo->u32ProcCtls2   = uProcCtls2;
+        pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
+        Log4Func(("Enabled\n"));
+    }
+}
+
+
+/**
+ * Disables VMCS shadowing for the given VMCS info. object.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static void vmxHCDisableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    /*
+     * We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
+     * VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
+     * to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
+     * Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
+     *
+     * See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
+     * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
+     */
+    uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
+    if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+    {
+        uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
+        int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);                AssertRC(rc);
+        rc     = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);     AssertRC(rc);
+        pVmcsInfo->u32ProcCtls2   = uProcCtls2;
+        pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
+        Log4Func(("Disabled\n"));
+    }
+}
+#endif
+
+
+/**
+ * Exports the guest CR0 control register into the guest-state area in the VMCS.
+ *
+ * The guest FPU state is always pre-loaded hence we don't need to bother about
+ * sharing FPU related CR0 bits between the guest and host.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int vmxHCExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR0)
+    {
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+
+        uint64_t       fSetCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed0;
+        uint64_t const fZapCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed1;
+        if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
+            fSetCr0 &= ~(uint64_t)(X86_CR0_PE | X86_CR0_PG);
+        else
+            Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
+
+        if (!pVmxTransient->fIsNestedGuest)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+            uint64_t       u64GuestCr0  = pVCpu->cpum.GstCtx.cr0;
+            uint64_t const u64ShadowCr0 = u64GuestCr0;
+            Assert(!RT_HI_U32(u64GuestCr0));
+
+            /*
+             * Setup VT-x's view of the guest CR0.
+             */
+            uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
+            if (VM_IS_VMX_NESTED_PAGING(pVM))
+            {
+#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
+                if (CPUMIsGuestPagingEnabled(pVCpu))
+                {
+                    /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
+                    uProcCtls &= ~(  VMX_PROC_CTLS_CR3_LOAD_EXIT
+                                   | VMX_PROC_CTLS_CR3_STORE_EXIT);
+                }
+                else
+                {
+                    /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
+                    uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
+                              |  VMX_PROC_CTLS_CR3_STORE_EXIT;
+                }
+
+                /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
+                if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
+                    uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
+#endif
+            }
+            else
+            {
+                /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
+                u64GuestCr0 |= X86_CR0_WP;
+            }
+
+            /*
+             * Guest FPU bits.
+             *
+             * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
+             * using CR0.TS.
+             *
+             * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
+             * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
+             */
+            u64GuestCr0 |= X86_CR0_NE;
+
+            /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
+            bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
+
+            /*
+             * Update exception intercepts.
+             */
+            uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
+#ifndef IN_NEM_DARWIN
+            if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
+            {
+                Assert(PDMVmmDevHeapIsEnabled(pVM));
+                Assert(pVM->hm.s.vmx.pRealModeTSS);
+                uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
+            }
+            else
+#endif
+            {
+                /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
+                uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
+                if (fInterceptMF)
+                    uXcptBitmap |= RT_BIT(X86_XCPT_MF);
+            }
+
+            /* Additional intercepts for debugging, define these yourself explicitly. */
+#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
+            uXcptBitmap |= 0
+                        |  RT_BIT(X86_XCPT_BP)
+                        |  RT_BIT(X86_XCPT_DE)
+                        |  RT_BIT(X86_XCPT_NM)
+                        |  RT_BIT(X86_XCPT_TS)
+                        |  RT_BIT(X86_XCPT_UD)
+                        |  RT_BIT(X86_XCPT_NP)
+                        |  RT_BIT(X86_XCPT_SS)
+                        |  RT_BIT(X86_XCPT_GP)
+                        |  RT_BIT(X86_XCPT_PF)
+                        |  RT_BIT(X86_XCPT_MF)
+                        ;
+#elif defined(HMVMX_ALWAYS_TRAP_PF)
+            uXcptBitmap |= RT_BIT(X86_XCPT_PF);
+#endif
+            if (VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv)
+                uXcptBitmap |= RT_BIT(X86_XCPT_GP);
+            if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE)
+                uXcptBitmap |= RT_BIT(X86_XCPT_DE);
+            Assert(VM_IS_VMX_NESTED_PAGING(pVM) || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
+
+            /* Apply the hardware specified CR0 fixed bits and enable caching. */
+            u64GuestCr0 |= fSetCr0;
+            u64GuestCr0 &= fZapCr0;
+            u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD | X86_CR0_NW);
+
+            Assert(!RT_HI_U32(u64GuestCr0));
+            Assert(u64GuestCr0 & X86_CR0_NE);
+
+            /* Commit the CR0 and related fields to the guest VMCS. */
+            int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0);               AssertRC(rc);
+            rc     = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);   AssertRC(rc);
+            if (uProcCtls != pVmcsInfo->u32ProcCtls)
+            {
+                rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
+                AssertRC(rc);
+            }
+            if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
+            {
+                rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
+                AssertRC(rc);
+            }
+
+            /* Update our caches. */
+            pVmcsInfo->u32ProcCtls   = uProcCtls;
+            pVmcsInfo->u32XcptBitmap = uXcptBitmap;
+
+            Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
+        }
+        else
+        {
+            /*
+             * With nested-guests, we may have extended the guest/host mask here since we
+             * merged in the outer guest's mask. Thus, the merged mask can include more bits
+             * (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
+             * originally supplied. We must copy those bits from the nested-guest CR0 into
+             * the nested-guest CR0 read-shadow.
+             */
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+            uint64_t       u64GuestCr0  = pVCpu->cpum.GstCtx.cr0;
+            uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);
+
+            /* Apply the hardware specified CR0 fixed bits and enable caching. */
+            u64GuestCr0 |= fSetCr0;
+            u64GuestCr0 &= fZapCr0;
+            u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD | X86_CR0_NW);
+
+            Assert(!RT_HI_U32(u64GuestCr0));
+            Assert(u64GuestCr0 & X86_CR0_NE);
+
+            /* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
+            int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0);               AssertRC(rc);
+            rc     = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);   AssertRC(rc);
+
+            Log4Func(("cr0=%#RX64 shadow=%#RX64 vmcs_read_shw=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0,
+                      pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0ReadShadow.u, fSetCr0, fZapCr0));
+        }
+
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR0);
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exports the guest control registers (CR3, CR4) into the guest-state area
+ * in the VMCS.
+ *
+ * @returns VBox strict status code.
+ * @retval  VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
+ *          without unrestricted guest access and the VMMDev is not presently
+ *          mapped (e.g. EFI32).
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static VBOXSTRICTRC vmxHCExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    int rc  = VINF_SUCCESS;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Guest CR2.
+     * It's always loaded in the assembler code. Nothing to do here.
+     */
+
+    /*
+     * Guest CR3.
+     */
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR3)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
+
+        if (VM_IS_VMX_NESTED_PAGING(pVM))
+        {
+#ifndef IN_NEM_DARWIN
+            PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+            pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
+
+            /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
+            Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
+            Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
+            Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
+
+            /* VMX_EPT_MEMTYPE_WB support is already checked in vmxHCSetupTaggedTlb(). */
+            pVmcsInfo->HCPhysEPTP |= RT_BF_MAKE(VMX_BF_EPTP_MEMTYPE,          VMX_EPTP_MEMTYPE_WB)
+                                  |  RT_BF_MAKE(VMX_BF_EPTP_PAGE_WALK_LENGTH, VMX_EPTP_PAGE_WALK_LENGTH_4);
+
+            /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
+            AssertMsg(   ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3      /* Bits 3:5 (EPT page walk length - 1) must be 3. */
+                      && ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0,     /* Bits 7:11 MBZ. */
+                         ("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
+            AssertMsg(  !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01)           /* Bit 6 (EPT accessed & dirty bit). */
+                      || (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY),
+                         ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
+
+            rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
+            AssertRC(rc);
+#endif
+
+            PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+            uint64_t  u64GuestCr3 = pCtx->cr3;
+            if (   VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
+                || CPUMIsGuestPagingEnabledEx(pCtx))
+            {
+                /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
+                if (CPUMIsGuestInPAEModeEx(pCtx))
+                {
+                    rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, pCtx->aPaePdpes[0].u);     AssertRC(rc);
+                    rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, pCtx->aPaePdpes[1].u);     AssertRC(rc);
+                    rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, pCtx->aPaePdpes[2].u);     AssertRC(rc);
+                    rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, pCtx->aPaePdpes[3].u);     AssertRC(rc);
+                }
+
+                /*
+                 * The guest's view of its CR3 is unblemished with nested paging when the
+                 * guest is using paging or we have unrestricted guest execution to handle
+                 * the guest when it's not using paging.
+                 */
+            }
+#ifndef IN_NEM_DARWIN
+            else
+            {
+                /*
+                 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
+                 * thinks it accesses physical memory directly, we use our identity-mapped
+                 * page table to map guest-linear to guest-physical addresses. EPT takes care
+                 * of translating it to host-physical addresses.
+                 */
+                RTGCPHYS GCPhys;
+                Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
+
+                /* We obtain it here every time as the guest could have relocated this PCI region. */
+                rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
+                if (RT_SUCCESS(rc))
+                { /* likely */ }
+                else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
+                {
+                    Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
+                    return VINF_EM_RESCHEDULE_REM;  /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
+                }
+                else
+                    AssertMsgFailedReturn(("%Rrc\n",  rc), rc);
+
+                u64GuestCr3 = GCPhys;
+            }
+#endif
+
+            Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
+            rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, u64GuestCr3);
+            AssertRC(rc);
+        }
+        else
+        {
+            Assert(!pVmxTransient->fIsNestedGuest);
+            /* Non-nested paging case, just use the hypervisor's CR3. */
+            RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);
+
+            Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
+            rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
+            AssertRC(rc);
+        }
+
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR3);
+    }
+
+    /*
+     * Guest CR4.
+     * ASSUMES this is done everytime we get in from ring-3! (XCR0)
+     */
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR4)
+    {
+        PCPUMCTX     pCtx      = &pVCpu->cpum.GstCtx;
+        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+
+        uint64_t const fSetCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed0;
+        uint64_t const fZapCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed1;
+
+        /*
+         * With nested-guests, we may have extended the guest/host mask here (since we
+         * merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
+         * mask can include more bits (to read from the nested-guest CR4 read-shadow) than
+         * the nested hypervisor originally supplied. Thus, we should, in essence, copy
+         * those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
+         */
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+        uint64_t       u64GuestCr4  = pCtx->cr4;
+        uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
+                                    ? pCtx->cr4
+                                    : CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
+        Assert(!RT_HI_U32(u64GuestCr4));
+
+#ifndef IN_NEM_DARWIN
+        /*
+         * Setup VT-x's view of the guest CR4.
+         *
+         * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
+         * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
+         * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
+         *
+         * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
+         */
+        if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
+        {
+            Assert(pVM->hm.s.vmx.pRealModeTSS);
+            Assert(PDMVmmDevHeapIsEnabled(pVM));
+            u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
+        }
+#endif
+
+        if (VM_IS_VMX_NESTED_PAGING(pVM))
+        {
+            if (   !CPUMIsGuestPagingEnabledEx(pCtx)
+                && !VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
+            {
+                /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
+                u64GuestCr4 |= X86_CR4_PSE;
+                /* Our identity mapping is a 32-bit page directory. */
+                u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
+            }
+            /* else use guest CR4.*/
+        }
+        else
+        {
+            Assert(!pVmxTransient->fIsNestedGuest);
+
+            /*
+             * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
+             * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
+             */
+            switch (VCPU_2_VMXSTATE(pVCpu).enmShadowMode)
+            {
+                case PGMMODE_REAL:              /* Real-mode. */
+                case PGMMODE_PROTECTED:         /* Protected mode without paging. */
+                case PGMMODE_32_BIT:            /* 32-bit paging. */
+                {
+                    u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
+                    break;
+                }
+
+                case PGMMODE_PAE:               /* PAE paging. */
+                case PGMMODE_PAE_NX:            /* PAE paging with NX. */
+                {
+                    u64GuestCr4 |= X86_CR4_PAE;
+                    break;
+                }
+
+                case PGMMODE_AMD64:             /* 64-bit AMD paging (long mode). */
+                case PGMMODE_AMD64_NX:          /* 64-bit AMD paging (long mode) with NX enabled. */
+                {
+#ifdef VBOX_WITH_64_BITS_GUESTS
+                    /* For our assumption in vmxHCShouldSwapEferMsr. */
+                    Assert(u64GuestCr4 & X86_CR4_PAE);
+                    break;
+#endif
+                }
+                default:
+                    AssertFailed();
+                    return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
+            }
+        }
+
+        /* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
+        u64GuestCr4 |= fSetCr4;
+        u64GuestCr4 &= fZapCr4;
+
+        Assert(!RT_HI_U32(u64GuestCr4));
+        Assert(u64GuestCr4 & X86_CR4_VMXE);
+
+        /* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
+        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR4, u64GuestCr4);               AssertRC(rc);
+        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);   AssertRC(rc);
+
+#ifndef IN_NEM_DARWIN
+        /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
+        bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
+        if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
+        {
+            pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
+            hmR0VmxUpdateStartVmFunction(pVCpu);
+        }
+#endif
+
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR4);
+
+        Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
+    }
+    return rc;
+}
+
+
+#ifdef VBOX_STRICT
+/**
+ * Strict function to validate segment registers.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Will import guest CR0 on strict builds during validation of
+ *          segments.
+ */
+static void vmxHCValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    /*
+     * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
+     *
+     * The reason we check for attribute value 0 in this function and not just the unusable bit is
+     * because vmxHCExportGuestSegReg() only updates the VMCS' copy of the value with the
+     * unusable bit and doesn't change the guest-context value.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
+    if (   !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
+        && (   !CPUMIsGuestInRealModeEx(pCtx)
+            && !CPUMIsGuestInV86ModeEx(pCtx)))
+    {
+        /* Protected mode checks */
+        /* CS */
+        Assert(pCtx->cs.Attr.n.u1Present);
+        Assert(!(pCtx->cs.Attr.u & 0xf00));
+        Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
+        Assert(   (pCtx->cs.u32Limit & 0xfff) == 0xfff
+               || !(pCtx->cs.Attr.n.u1Granularity));
+        Assert(   !(pCtx->cs.u32Limit & 0xfff00000)
+               || (pCtx->cs.Attr.n.u1Granularity));
+        /* CS cannot be loaded with NULL in protected mode. */
+        Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
+        if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
+            Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
+        else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
+            Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
+        else
+            AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
+        /* SS */
+        Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
+        Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
+        if (   !(pCtx->cr0 & X86_CR0_PE)
+            || pCtx->cs.Attr.n.u4Type == 3)
+        {
+            Assert(!pCtx->ss.Attr.n.u2Dpl);
+        }
+        if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
+            Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
+            Assert(pCtx->ss.Attr.n.u1Present);
+            Assert(!(pCtx->ss.Attr.u & 0xf00));
+            Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
+            Assert(   (pCtx->ss.u32Limit & 0xfff) == 0xfff
+                   || !(pCtx->ss.Attr.n.u1Granularity));
+            Assert(   !(pCtx->ss.u32Limit & 0xfff00000)
+                   || (pCtx->ss.Attr.n.u1Granularity));
+        }
+        /* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSegReg(). */
+        if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
+            Assert(pCtx->ds.Attr.n.u1Present);
+            Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
+            Assert(!(pCtx->ds.Attr.u & 0xf00));
+            Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
+            Assert(   (pCtx->ds.u32Limit & 0xfff) == 0xfff
+                   || !(pCtx->ds.Attr.n.u1Granularity));
+            Assert(   !(pCtx->ds.u32Limit & 0xfff00000)
+                   || (pCtx->ds.Attr.n.u1Granularity));
+            Assert(   !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                   || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
+        }
+        if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
+            Assert(pCtx->es.Attr.n.u1Present);
+            Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
+            Assert(!(pCtx->es.Attr.u & 0xf00));
+            Assert(!(pCtx->es.Attr.u & 0xfffe0000));
+            Assert(   (pCtx->es.u32Limit & 0xfff) == 0xfff
+                   || !(pCtx->es.Attr.n.u1Granularity));
+            Assert(   !(pCtx->es.u32Limit & 0xfff00000)
+                   || (pCtx->es.Attr.n.u1Granularity));
+            Assert(   !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                   || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
+        }
+        if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
+            Assert(pCtx->fs.Attr.n.u1Present);
+            Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
+            Assert(!(pCtx->fs.Attr.u & 0xf00));
+            Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
+            Assert(   (pCtx->fs.u32Limit & 0xfff) == 0xfff
+                   || !(pCtx->fs.Attr.n.u1Granularity));
+            Assert(   !(pCtx->fs.u32Limit & 0xfff00000)
+                   || (pCtx->fs.Attr.n.u1Granularity));
+            Assert(   !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                   || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
+        }
+        if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
+        {
+            Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
+            Assert(pCtx->gs.Attr.n.u1Present);
+            Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
+            Assert(!(pCtx->gs.Attr.u & 0xf00));
+            Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
+            Assert(   (pCtx->gs.u32Limit & 0xfff) == 0xfff
+                   || !(pCtx->gs.Attr.n.u1Granularity));
+            Assert(   !(pCtx->gs.u32Limit & 0xfff00000)
+                   || (pCtx->gs.Attr.n.u1Granularity));
+            Assert(   !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                   || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
+        }
+        /* 64-bit capable CPUs. */
+        Assert(!RT_HI_U32(pCtx->cs.u64Base));
+        Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
+        Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
+        Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
+    }
+    else if (   CPUMIsGuestInV86ModeEx(pCtx)
+             || (   CPUMIsGuestInRealModeEx(pCtx)
+                 && !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)))
+    {
+        /* Real and v86 mode checks. */
+        /* vmxHCExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
+        uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
+#ifndef IN_NEM_DARWIN
+        if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
+        {
+            u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
+            u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
+        }
+        else
+#endif
+        {
+            u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
+            u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
+        }
+
+        /* CS */
+        AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
+        Assert(pCtx->cs.u32Limit == 0xffff);
+        AssertMsg(u32CSAttr == 0xf3, ("cs=%#x %#x ", pCtx->cs.Sel, u32CSAttr));
+        /* SS */
+        Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
+        Assert(pCtx->ss.u32Limit == 0xffff);
+        Assert(u32SSAttr == 0xf3);
+        /* DS */
+        Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
+        Assert(pCtx->ds.u32Limit == 0xffff);
+        Assert(u32DSAttr == 0xf3);
+        /* ES */
+        Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
+        Assert(pCtx->es.u32Limit == 0xffff);
+        Assert(u32ESAttr == 0xf3);
+        /* FS */
+        Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
+        Assert(pCtx->fs.u32Limit == 0xffff);
+        Assert(u32FSAttr == 0xf3);
+        /* GS */
+        Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
+        Assert(pCtx->gs.u32Limit == 0xffff);
+        Assert(u32GSAttr == 0xf3);
+        /* 64-bit capable CPUs. */
+        Assert(!RT_HI_U32(pCtx->cs.u64Base));
+        Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
+        Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
+        Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
+    }
+}
+#endif /* VBOX_STRICT */
+
+
+/**
+ * Exports a guest segment register into the guest-state area in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ * @param   iSegReg     The segment register number (X86_SREG_XXX).
+ * @param   pSelReg     Pointer to the segment selector.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int vmxHCExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg)
+{
+    Assert(iSegReg < X86_SREG_COUNT);
+
+    uint32_t u32Access = pSelReg->Attr.u;
+#ifndef IN_NEM_DARWIN
+    if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
+#endif
+    {
+        /*
+         * The way to differentiate between whether this is really a null selector or was just
+         * a selector loaded with 0 in real-mode is using the segment attributes. A selector
+         * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
+         * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
+         * NULL selectors loaded in protected-mode have their attribute as 0.
+         */
+        if (u32Access)
+        { }
+        else
+            u32Access = X86DESCATTR_UNUSABLE;
+    }
+#ifndef IN_NEM_DARWIN
+    else
+    {
+        /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
+        u32Access = 0xf3;
+        Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
+        Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
+        RT_NOREF_PV(pVCpu);
+    }
+#else
+    RT_NOREF(pVmcsInfo);
+#endif
+
+    /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
+    AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
+              ("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u));
+
+    /*
+     * Commit it to the VMCS.
+     */
+    int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg),           pSelReg->Sel);      AssertRC(rc);
+    rc     = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg),         pSelReg->u32Limit); AssertRC(rc);
+    rc     = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg),            pSelReg->u64Base);  AssertRC(rc);
+    rc     = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access);         AssertRC(rc);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
+ * area in the VMCS.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks Will import guest CR0 on strict builds during validation of
+ *          segments.
+ * @remarks No-long-jump zone!!!
+ */
+static int vmxHCExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
+{
+    int                 rc              = VERR_INTERNAL_ERROR_5;
+#ifndef IN_NEM_DARWIN
+    PVMCC               pVM             = pVCpu->CTX_SUFF(pVM);
+#endif
+    PCCPUMCTX           pCtx            = &pVCpu->cpum.GstCtx;
+    PVMXVMCSINFO        pVmcsInfo       = pVmxTransient->pVmcsInfo;
+#ifndef IN_NEM_DARWIN
+    PVMXVMCSINFOSHARED  pVmcsInfoShared = pVmcsInfo->pShared;
+#endif
+
+    /*
+     * Guest Segment registers: CS, SS, DS, ES, FS, GS.
+     */
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
+    {
+        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CS)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
+#ifndef IN_NEM_DARWIN
+            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
+                pVmcsInfoShared->RealMode.AttrCS.u = pCtx->cs.Attr.u;
+#endif
+            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CS);
+        }
+
+        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SS)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
+#ifndef IN_NEM_DARWIN
+            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
+                pVmcsInfoShared->RealMode.AttrSS.u = pCtx->ss.Attr.u;
+#endif
+            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SS);
+        }
+
+        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_DS)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
+#ifndef IN_NEM_DARWIN
+            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
+                pVmcsInfoShared->RealMode.AttrDS.u = pCtx->ds.Attr.u;
+#endif
+            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_DS);
+        }
+
+        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_ES)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
+#ifndef IN_NEM_DARWIN
+            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
+                pVmcsInfoShared->RealMode.AttrES.u = pCtx->es.Attr.u;
+#endif
+            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_ES);
+        }
+
+        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_FS)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
+#ifndef IN_NEM_DARWIN
+            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
+                pVmcsInfoShared->RealMode.AttrFS.u = pCtx->fs.Attr.u;
+#endif
+            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_FS);
+        }
+
+        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GS)
+        {
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
+#ifndef IN_NEM_DARWIN
+            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
+                pVmcsInfoShared->RealMode.AttrGS.u = pCtx->gs.Attr.u;
+#endif
+            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
+            AssertRC(rc);
+            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GS);
+        }
+
+#ifdef VBOX_STRICT
+        vmxHCValidateSegmentRegs(pVCpu, pVmcsInfo);
+#endif
+        Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
+                  pCtx->cs.Attr.u));
+    }
+
+    /*
+     * Guest TR.
+     */
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_TR)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
+
+        /*
+         * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
+         * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
+         * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
+         */
+        uint16_t u16Sel;
+        uint32_t u32Limit;
+        uint64_t u64Base;
+        uint32_t u32AccessRights;
+#ifndef IN_NEM_DARWIN
+        if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
+#endif
+        {
+            u16Sel          = pCtx->tr.Sel;
+            u32Limit        = pCtx->tr.u32Limit;
+            u64Base         = pCtx->tr.u64Base;
+            u32AccessRights = pCtx->tr.Attr.u;
+        }
+#ifndef IN_NEM_DARWIN
+        else
+        {
+            Assert(!pVmxTransient->fIsNestedGuest);
+            Assert(pVM->hm.s.vmx.pRealModeTSS);
+            Assert(PDMVmmDevHeapIsEnabled(pVM));    /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
+
+            /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
+            RTGCPHYS GCPhys;
+            rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
+            AssertRCReturn(rc, rc);
+
+            X86DESCATTR DescAttr;
+            DescAttr.u           = 0;
+            DescAttr.n.u1Present = 1;
+            DescAttr.n.u4Type    = X86_SEL_TYPE_SYS_386_TSS_BUSY;
+
+            u16Sel          = 0;
+            u32Limit        = HM_VTX_TSS_SIZE;
+            u64Base         = GCPhys;
+            u32AccessRights = DescAttr.u;
+        }
+#endif
+
+        /* Validate. */
+        Assert(!(u16Sel & RT_BIT(2)));
+        AssertMsg(   (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
+                  || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
+        AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
+        Assert(!(u32AccessRights & RT_BIT(4)));                 /* System MBZ.*/
+        Assert(u32AccessRights & RT_BIT(7));                    /* Present MB1.*/
+        Assert(!(u32AccessRights & 0xf00));                     /* 11:8 MBZ. */
+        Assert(!(u32AccessRights & 0xfffe0000));                /* 31:17 MBZ. */
+        Assert(   (u32Limit & 0xfff) == 0xfff
+               || !(u32AccessRights & RT_BIT(15)));             /* Granularity MBZ. */
+        Assert(   !(pCtx->tr.u32Limit & 0xfff00000)
+               || (u32AccessRights & RT_BIT(15)));              /* Granularity MB1. */
+
+        rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL,           u16Sel);             AssertRC(rc);
+        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT,         u32Limit);           AssertRC(rc);
+        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);    AssertRC(rc);
+        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE,            u64Base);            AssertRC(rc);
+
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_TR);
+        Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
+    }
+
+    /*
+     * Guest GDTR.
+     */
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GDTR)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
+
+        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);     AssertRC(rc);
+        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE,  pCtx->gdtr.pGdt);        AssertRC(rc);
+
+        /* Validate. */
+        Assert(!(pCtx->gdtr.cbGdt & 0xffff0000));          /* Bits 31:16 MBZ. */
+
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
+        Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
+    }
+
+    /*
+     * Guest LDTR.
+     */
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_LDTR)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
+
+        /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
+        uint32_t u32Access;
+        if (   !pVmxTransient->fIsNestedGuest
+            && !pCtx->ldtr.Attr.u)
+            u32Access = X86DESCATTR_UNUSABLE;
+        else
+            u32Access = pCtx->ldtr.Attr.u;
+
+        rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL,           pCtx->ldtr.Sel);       AssertRC(rc);
+        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT,         pCtx->ldtr.u32Limit);  AssertRC(rc);
+        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);            AssertRC(rc);
+        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE,            pCtx->ldtr.u64Base);   AssertRC(rc);
+
+        /* Validate. */
+        if (!(u32Access & X86DESCATTR_UNUSABLE))
+        {
+            Assert(!(pCtx->ldtr.Sel & RT_BIT(2)));              /* TI MBZ. */
+            Assert(pCtx->ldtr.Attr.n.u4Type == 2);              /* Type MB2 (LDT). */
+            Assert(!pCtx->ldtr.Attr.n.u1DescType);              /* System MBZ. */
+            Assert(pCtx->ldtr.Attr.n.u1Present == 1);           /* Present MB1. */
+            Assert(!pCtx->ldtr.Attr.n.u4LimitHigh);             /* 11:8 MBZ. */
+            Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000));          /* 31:17 MBZ. */
+            Assert(   (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
+                   || !pCtx->ldtr.Attr.n.u1Granularity);        /* Granularity MBZ. */
+            Assert(   !(pCtx->ldtr.u32Limit & 0xfff00000)
+                   || pCtx->ldtr.Attr.n.u1Granularity);         /* Granularity MB1. */
+        }
+
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
+        Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
+    }
+
+    /*
+     * Guest IDTR.
+     */
+    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_IDTR)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
+
+        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);     AssertRC(rc);
+        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE,  pCtx->idtr.pIdt);        AssertRC(rc);
+
+        /* Validate. */
+        Assert(!(pCtx->idtr.cbIdt & 0xffff0000));          /* Bits 31:16 MBZ. */
+
+        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
+        Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Gets the IEM exception flags for the specified vector and IDT vectoring /
+ * VM-exit interruption info type.
+ *
+ * @returns The IEM exception flags.
+ * @param   uVector         The event vector.
+ * @param   uVmxEventType   The VMX event type.
+ *
+ * @remarks This function currently only constructs flags required for
+ *          IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
+ *          and CR2 aspects of an exception are not included).
+ */
+static uint32_t vmxHCGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
+{
+    uint32_t fIemXcptFlags;
+    switch (uVmxEventType)
+    {
+        case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
+        case VMX_IDT_VECTORING_INFO_TYPE_NMI:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
+            break;
+
+        case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
+        {
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
+            if (uVector == X86_XCPT_BP)
+                fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
+            else if (uVector == X86_XCPT_OF)
+                fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
+            else
+            {
+                fIemXcptFlags = 0;
+                AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
+            }
+            break;
+        }
+
+        case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
+            fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
+            break;
+
+        default:
+            fIemXcptFlags = 0;
+            AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
+            break;
+    }
+    return fIemXcptFlags;
+}
+
+
+/**
+ * Sets an event as a pending event to be injected into the guest.
+ *
+ * @param   pVCpu               The cross context virtual CPU structure.
+ * @param   u32IntInfo          The VM-entry interruption-information field.
+ * @param   cbInstr             The VM-entry instruction length in bytes (for
+ *                              software interrupts, exceptions and privileged
+ *                              software exceptions).
+ * @param   u32ErrCode          The VM-entry exception error code.
+ * @param   GCPtrFaultAddress   The fault-address (CR2) in case it's a
+ *                              page-fault.
+ */
+DECLINLINE(void) vmxHCSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
+                                      RTGCUINTPTR GCPtrFaultAddress)
+{
+    Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
+    VCPU_2_VMXSTATE(pVCpu).Event.fPending          = true;
+    VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo        = u32IntInfo;
+    VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode        = u32ErrCode;
+    VCPU_2_VMXSTATE(pVCpu).Event.cbInstr           = cbInstr;
+    VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress = GCPtrFaultAddress;
+}
+
+
+/**
+ * Sets an external interrupt as pending-for-injection into the VM.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   u8Interrupt     The external interrupt vector.
+ */
+DECLINLINE(void) vmxHCSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR,          u8Interrupt)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) vmxHCSetPendingXcptNmi(PVMCPUCC pVCpu)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_NMI)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_ENTRY_INT_INFO_TYPE_NMI)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) vmxHCSetPendingXcptDF(PVMCPUCC pVCpu)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_DF)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) vmxHCSetPendingXcptUD(PVMCPUCC pVCpu)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_UD)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets a debug (\#DB) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+DECLINLINE(void) vmxHCSetPendingXcptDB(PVMCPUCC pVCpu)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_DB)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u32ErrCode  The error code for the general-protection exception.
+ */
+DECLINLINE(void) vmxHCSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_GP)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
+}
+
+
+/**
+ * Sets a stack (\#SS) exception as pending-for-injection into the VM.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   u32ErrCode  The error code for the stack exception.
+ */
+DECLINLINE(void) vmxHCSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
+{
+    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_SS)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
+                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
+    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * Fixes up attributes for the specified segment register.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pSelReg     The segment register that needs fixing.
+ * @param   pszRegName  The register name (for logging and assertions).
+ */
+static void vmxHCFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName)
+{
+    Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
+
+    /*
+     * If VT-x marks the segment as unusable, most other bits remain undefined:
+     *   - For CS the L, D and G bits have meaning.
+     *   - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
+     *   - For the remaining data segments no bits are defined.
+     *
+     * The present bit and the unusable bit has been observed to be set at the
+     * same time (the selector was supposed to be invalid as we started executing
+     * a V8086 interrupt in ring-0).
+     *
+     * What should be important for the rest of the VBox code, is that the P bit is
+     * cleared.  Some of the other VBox code recognizes the unusable bit, but
+     * AMD-V certainly don't, and REM doesn't really either.  So, to be on the
+     * safe side here, we'll strip off P and other bits we don't care about.  If
+     * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
+     *
+     * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
+     */
+#ifdef VBOX_STRICT
+    uint32_t const uAttr = pSelReg->Attr.u;
+#endif
+
+    /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
+    pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L    | X86DESCATTR_D  | X86DESCATTR_G
+                     | X86DESCATTR_DPL      | X86DESCATTR_TYPE | X86DESCATTR_DT;
+
+#ifdef VBOX_STRICT
+# ifndef IN_NEM_DARWIN
+    VMMRZCallRing3Disable(pVCpu);
+# endif
+    Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u));
+# ifdef DEBUG_bird
+    AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
+              ("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
+               pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
+# endif
+# ifndef IN_NEM_DARWIN
+    VMMRZCallRing3Enable(pVCpu);
+# endif
+    NOREF(uAttr);
+#endif
+    RT_NOREF2(pVCpu, pszRegName);
+}
+
+
+/**
+ * Imports a guest segment register from the current VMCS into the guest-CPU
+ * context.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @tparam  a_iSegReg   The segment register number (X86_SREG_XXX).
+ *
+ * @remarks Called with interrupts and/or preemption disabled.
+ */
+template<uint32_t const a_iSegReg>
+DECLINLINE(void) vmxHCImportGuestSegReg(PVMCPUCC pVCpu)
+{
+    AssertCompile(a_iSegReg < X86_SREG_COUNT);
+    /* Check that the macros we depend upon here and in the export parenter function works: */
+#define MY_SEG_VMCS_FIELD(a_FieldPrefix, a_FieldSuff) \
+        (  a_iSegReg == X86_SREG_ES ? a_FieldPrefix ## ES ## a_FieldSuff \
+         : a_iSegReg == X86_SREG_CS ? a_FieldPrefix ## CS ## a_FieldSuff \
+         : a_iSegReg == X86_SREG_SS ? a_FieldPrefix ## SS ## a_FieldSuff \
+         : a_iSegReg == X86_SREG_DS ? a_FieldPrefix ## DS ## a_FieldSuff \
+         : a_iSegReg == X86_SREG_FS ? a_FieldPrefix ## FS ## a_FieldSuff \
+         : a_iSegReg == X86_SREG_GS ? a_FieldPrefix ## GS ## a_FieldSuff : 0)
+    AssertCompile(VMX_VMCS_GUEST_SEG_BASE(a_iSegReg)             == MY_SEG_VMCS_FIELD(VMX_VMCS_GUEST_,_BASE));
+    AssertCompile(VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg)            == MY_SEG_VMCS_FIELD(VMX_VMCS16_GUEST_,_SEL));
+    AssertCompile(VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg)          == MY_SEG_VMCS_FIELD(VMX_VMCS32_GUEST_,_LIMIT));
+    AssertCompile(VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg)  == MY_SEG_VMCS_FIELD(VMX_VMCS32_GUEST_,_ACCESS_RIGHTS));
+
+    PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[a_iSegReg];
+
+    uint16_t u16Sel;
+    int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg), &u16Sel);   AssertRC(rc);
+    pSelReg->Sel      = u16Sel;
+    pSelReg->ValidSel = u16Sel;
+
+    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg), &pSelReg->u32Limit); AssertRC(rc);
+    rc     = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(a_iSegReg), &pSelReg->u64Base);     AssertRC(rc);
+
+    uint32_t u32Attr;
+    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg), &u32Attr);   AssertRC(rc);
+    pSelReg->Attr.u   = u32Attr;
+    if (u32Attr & X86DESCATTR_UNUSABLE)
+        vmxHCFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + a_iSegReg * 3);
+
+    pSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
+}
+
+
+/**
+ * Imports the guest LDTR from the current VMCS into the guest-CPU context.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks Called with interrupts and/or preemption disabled.
+ */
+DECLINLINE(void) vmxHCImportGuestLdtr(PVMCPUCC pVCpu)
+{
+    uint16_t u16Sel;
+    uint64_t u64Base;
+    uint32_t u32Limit, u32Attr;
+    int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL,           &u16Sel);       AssertRC(rc);
+    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT,         &u32Limit);     AssertRC(rc);
+    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);      AssertRC(rc);
+    rc     = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE,            &u64Base);      AssertRC(rc);
+
+    pVCpu->cpum.GstCtx.ldtr.Sel      = u16Sel;
+    pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
+    pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
+    pVCpu->cpum.GstCtx.ldtr.u64Base  = u64Base;
+    pVCpu->cpum.GstCtx.ldtr.Attr.u   = u32Attr;
+    if (u32Attr & X86DESCATTR_UNUSABLE)
+        vmxHCFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR");
+}
+
+
+/**
+ * Imports the guest TR from the current VMCS into the guest-CPU context.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks Called with interrupts and/or preemption disabled.
+ */
+DECLINLINE(void) vmxHCImportGuestTr(PVMCPUCC pVCpu)
+{
+    uint16_t u16Sel;
+    uint64_t u64Base;
+    uint32_t u32Limit, u32Attr;
+    int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL,           &u16Sel);     AssertRC(rc);
+    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT,         &u32Limit);   AssertRC(rc);
+    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);    AssertRC(rc);
+    rc     = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE,            &u64Base);    AssertRC(rc);
+
+    pVCpu->cpum.GstCtx.tr.Sel      = u16Sel;
+    pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
+    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
+    pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
+    pVCpu->cpum.GstCtx.tr.u64Base  = u64Base;
+    pVCpu->cpum.GstCtx.tr.Attr.u   = u32Attr;
+    /* TR is the only selector that can never be unusable. */
+    Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
+}
+
+
+/**
+ * Core: Imports the guest RIP from the VMCS back into the guest-CPU context.
+ *
+ * @returns The RIP value.
+ * @param   pVCpu               The cross context virtual CPU structure.
+ *
+ * @remarks Called with interrupts and/or preemption disabled, should not assert!
+ * @remarks Do -not- call this function directly!
+ */
+DECL_FORCE_INLINE(uint64_t) vmxHCImportGuestCoreRip(PVMCPUCC pVCpu)
+{
+    uint64_t u64Val;
+    int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
+    AssertRC(rc);
+
+    pVCpu->cpum.GstCtx.rip = u64Val;
+
+    return u64Val;
+}
+
+
+/**
+ * Imports the guest RIP from the VMCS back into the guest-CPU context.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ *
+ * @remarks Called with interrupts and/or preemption disabled, should not assert!
+ * @remarks Do -not- call this function directly, use vmxHCImportGuestState()
+ *          instead!!!
+ */
+DECLINLINE(void) vmxHCImportGuestRip(PVMCPUCC pVCpu)
+{
+    if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP)
+    {
+        EMHistoryUpdatePC(pVCpu, vmxHCImportGuestCoreRip(pVCpu), false);
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
+    }
+}
+
+
+/**
+ * Core: Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Called with interrupts and/or preemption disabled, should not assert!
+ * @remarks Do -not- call this function directly!
+ */
+DECL_FORCE_INLINE(void) vmxHCImportGuestCoreRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+    uint64_t fRFlags;
+    int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &fRFlags);
+    AssertRC(rc);
+
+    Assert((fRFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK);
+    Assert((fRFlags & ~(uint64_t)(X86_EFL_1 | X86_EFL_LIVE_MASK)) == 0);
+
+    pVCpu->cpum.GstCtx.rflags.u = fRFlags;
+#ifndef IN_NEM_DARWIN
+    PCVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
+    if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
+    { /* mostly likely */ }
+    else
+    {
+        pVCpu->cpum.GstCtx.eflags.Bits.u1VM   = 0;
+        pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL = pVmcsInfoShared->RealMode.Eflags.Bits.u2IOPL;
+    }
+#else
+    RT_NOREF(pVmcsInfo);
+#endif
+}
+
+
+/**
+ * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Called with interrupts and/or preemption disabled, should not assert!
+ * @remarks Do -not- call this function directly, use vmxHCImportGuestState()
+ *          instead!!!
+ */
+DECLINLINE(void) vmxHCImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+    if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RFLAGS)
+    {
+        vmxHCImportGuestCoreRFlags(pVCpu, pVmcsInfo);
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
+    }
+}
+
+
+/**
+ * Worker for vmxHCImportGuestIntrState that handles the case where any of the
+ * relevant VMX_VMCS32_GUEST_INT_STATE bits are set.
+ */
+DECL_NO_INLINE(static,void) vmxHCImportGuestIntrStateSlow(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t fGstIntState)
+{
+    /*
+     * We must import RIP here to set our EM interrupt-inhibited state.
+     * We also import RFLAGS as our code that evaluates pending interrupts
+     * before VM-entry requires it.
+     */
+    vmxHCImportGuestRip(pVCpu);
+    vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
+
+    CPUMUpdateInterruptShadowSsStiEx(&pVCpu->cpum.GstCtx,
+                                     RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
+                                     RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
+                                     pVCpu->cpum.GstCtx.rip);
+    CPUMUpdateInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx, RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
+}
+
+
+/**
+ * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
+ * context.
+ *
+ * @note    May import RIP and RFLAGS if interrupt or NMI are blocked.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks Called with interrupts and/or preemption disabled, try not to assert and
+ *          do not log!
+ * @remarks Do -not- call this function directly, use vmxHCImportGuestState()
+ *          instead!!!
+ */
+DECLINLINE(void) vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+    uint32_t u32Val;
+    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32Val);    AssertRC(rc);
+    if (!u32Val)
+    {
+        CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
+        CPUMClearInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
+    }
+    else
+        vmxHCImportGuestIntrStateSlow(pVCpu, pVmcsInfo, u32Val);
+}
+
+
+/**
+ * Worker for VMXR0ImportStateOnDemand.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ * @param   fWhat       What to import, CPUMCTX_EXTRN_XXX.
+ */
+static int vmxHCImportGuestStateEx(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
+{
+    int      rc   = VINF_SUCCESS;
+    PVMCC    pVM  = pVCpu->CTX_SUFF(pVM);
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    uint32_t u32Val;
+
+    /*
+     * Note! This is hack to workaround a mysterious BSOD observed with release builds
+     *       on Windows 10 64-bit hosts. Profile and debug builds are not affected and
+     *       neither are other host platforms.
+     *
+     *       Committing this temporarily as it prevents BSOD.
+     *
+     * Update: This is very likely a compiler optimization bug, see @bugref{9180}.
+     */
+#ifdef RT_OS_WINDOWS
+    if (pVM == 0 || pVM == (void *)(uintptr_t)-1)
+        return VERR_HM_IPE_1;
+#endif
+
+    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
+
+#ifndef IN_NEM_DARWIN
+    /*
+     * We disable interrupts to make the updating of the state and in particular
+     * the fExtrn modification atomic wrt to preemption hooks.
+     */
+    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
+#endif
+
+    fWhat &= pCtx->fExtrn;
+    if (fWhat)
+    {
+        do
+        {
+            if (fWhat & CPUMCTX_EXTRN_RIP)
+                vmxHCImportGuestRip(pVCpu);
+
+            if (fWhat & CPUMCTX_EXTRN_RFLAGS)
+                vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);
+
+            /* Note! vmxHCImportGuestIntrState may also include RIP and RFLAGS and update fExtrn. */
+            if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
+                vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
+
+            if (fWhat & CPUMCTX_EXTRN_RSP)
+            {
+                rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pCtx->rsp);
+                AssertRC(rc);
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
+            {
+                PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
+#ifndef IN_NEM_DARWIN
+                bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
+#else
+                bool const fRealOnV86Active = false; /* HV supports only unrestricted guest execution. */
+#endif
+                if (fWhat & CPUMCTX_EXTRN_CS)
+                {
+                    vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
+                    vmxHCImportGuestRip(pVCpu); /** @todo WTF? */
+                    if (fRealOnV86Active)
+                        pCtx->cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
+                    EMHistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
+                }
+                if (fWhat & CPUMCTX_EXTRN_SS)
+                {
+                    vmxHCImportGuestSegReg<X86_SREG_SS>(pVCpu);
+                    if (fRealOnV86Active)
+                        pCtx->ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
+                }
+                if (fWhat & CPUMCTX_EXTRN_DS)
+                {
+                    vmxHCImportGuestSegReg<X86_SREG_DS>(pVCpu);
+                    if (fRealOnV86Active)
+                        pCtx->ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
+                }
+                if (fWhat & CPUMCTX_EXTRN_ES)
+                {
+                    vmxHCImportGuestSegReg<X86_SREG_ES>(pVCpu);
+                    if (fRealOnV86Active)
+                        pCtx->es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
+                }
+                if (fWhat & CPUMCTX_EXTRN_FS)
+                {
+                    vmxHCImportGuestSegReg<X86_SREG_FS>(pVCpu);
+                    if (fRealOnV86Active)
+                        pCtx->fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
+                }
+                if (fWhat & CPUMCTX_EXTRN_GS)
+                {
+                    vmxHCImportGuestSegReg<X86_SREG_GS>(pVCpu);
+                    if (fRealOnV86Active)
+                        pCtx->gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
+                }
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
+            {
+                if (fWhat & CPUMCTX_EXTRN_LDTR)
+                    vmxHCImportGuestLdtr(pVCpu);
+
+                if (fWhat & CPUMCTX_EXTRN_GDTR)
+                {
+                    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE,    &pCtx->gdtr.pGdt);  AssertRC(rc);
+                    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);           AssertRC(rc);
+                    pCtx->gdtr.cbGdt = u32Val;
+                }
+
+                /* Guest IDTR. */
+                if (fWhat & CPUMCTX_EXTRN_IDTR)
+                {
+                    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE,    &pCtx->idtr.pIdt);  AssertRC(rc);
+                    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);           AssertRC(rc);
+                    pCtx->idtr.cbIdt = u32Val;
+                }
+
+                /* Guest TR. */
+                if (fWhat & CPUMCTX_EXTRN_TR)
+                {
+#ifndef IN_NEM_DARWIN
+                    /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
+                       don't need to import that one. */
+                    if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
+#endif
+                        vmxHCImportGuestTr(pVCpu);
+                }
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_DR7)
+            {
+#ifndef IN_NEM_DARWIN
+                if (!pVCpu->hmr0.s.fUsingHyperDR7)
+#endif
+                {
+                    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pCtx->dr[7]);
+                    AssertRC(rc);
+                }
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+            {
+                rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP,  &pCtx->SysEnter.eip);  AssertRC(rc);
+                rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP,  &pCtx->SysEnter.esp);  AssertRC(rc);
+                rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);              AssertRC(rc);
+                pCtx->SysEnter.cs = u32Val;
+            }
+
+#ifndef IN_NEM_DARWIN
+            if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+            {
+                if (   pVM->hmr0.s.fAllow64BitGuests
+                    && (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
+                    pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
+            }
+
+            if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+            {
+                if (   pVM->hmr0.s.fAllow64BitGuests
+                    && (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
+                {
+                    pCtx->msrLSTAR  = ASMRdMsr(MSR_K8_LSTAR);
+                    pCtx->msrSTAR   = ASMRdMsr(MSR_K6_STAR);
+                    pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
+                }
+            }
+
+            if (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
+            {
+                PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
+                PCVMXAUTOMSR       pMsrs           = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
+                uint32_t const     cMsrs           = pVmcsInfo->cExitMsrStore;
+                Assert(pMsrs);
+                Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
+                Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE);
+                for (uint32_t i = 0; i < cMsrs; i++)
+                {
+                    uint32_t const idMsr = pMsrs[i].u32Msr;
+                    switch (idMsr)
+                    {
+                        case MSR_K8_TSC_AUX:        CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value);     break;
+                        case MSR_IA32_SPEC_CTRL:    CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value);   break;
+                        case MSR_K6_EFER:           /* Can't be changed without causing a VM-exit */  break;
+                        default:
+                        {
+                            uint32_t idxLbrMsr;
+                            if (VM_IS_VMX_LBR(pVM))
+                            {
+                                if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
+                                {
+                                    Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
+                                    pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
+                                    break;
+                                }
+                                if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
+                                {
+                                    Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
+                                    pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
+                                    break;
+                                }
+                                if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
+                                {
+                                    pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
+                                    break;
+                                }
+                                /* Fallthru (no break) */
+                            }
+                            pCtx->fExtrn = 0;
+                            VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr;
+                            ASMSetFlags(fEFlags);
+                            AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
+                            return VERR_HM_UNEXPECTED_LD_ST_MSR;
+                        }
+                    }
+                }
+            }
+#endif
+
+            if (fWhat & CPUMCTX_EXTRN_CR_MASK)
+            {
+                if (fWhat & CPUMCTX_EXTRN_CR0)
+                {
+                    uint64_t u64Cr0;
+                    uint64_t u64Shadow;
+                    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0,            &u64Cr0);       AssertRC(rc);
+                    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);    AssertRC(rc);
+#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
+                    u64Cr0 = (u64Cr0    & ~pVmcsInfo->u64Cr0Mask)
+                           | (u64Shadow &  pVmcsInfo->u64Cr0Mask);
+#else
+                    if (!CPUMIsGuestInVmxNonRootMode(pCtx))
+                    {
+                        u64Cr0 = (u64Cr0    & ~pVmcsInfo->u64Cr0Mask)
+                               | (u64Shadow &  pVmcsInfo->u64Cr0Mask);
+                    }
+                    else
+                    {
+                        /*
+                         * We've merged the guest and nested-guest's CR0 guest/host mask while executing
+                         * the nested-guest using hardware-assisted VMX. Accordingly we need to
+                         * re-construct CR0. See @bugref{9180#c95} for details.
+                         */
+                        PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
+                        PVMXVVMCS const     pVmcsNstGst  = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+                        u64Cr0 = (u64Cr0                     & ~(pVmcsInfoGst->u64Cr0Mask & pVmcsNstGst->u64Cr0Mask.u))
+                               | (pVmcsNstGst->u64GuestCr0.u &   pVmcsNstGst->u64Cr0Mask.u)
+                               | (u64Shadow                  &  (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
+                        Assert(u64Cr0 & X86_CR0_NE);
+                    }
+#endif
+#ifndef IN_NEM_DARWIN
+                    VMMRZCallRing3Disable(pVCpu);   /* May call into PGM which has Log statements. */
+#endif
+                    CPUMSetGuestCR0(pVCpu, u64Cr0);
+#ifndef IN_NEM_DARWIN
+                    VMMRZCallRing3Enable(pVCpu);
+#endif
+                }
+
+                if (fWhat & CPUMCTX_EXTRN_CR4)
+                {
+                    uint64_t u64Cr4;
+                    uint64_t u64Shadow;
+                    rc  = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4,            &u64Cr4);      AssertRC(rc);
+                    rc |= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);   AssertRC(rc);
+#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
+                    u64Cr4 = (u64Cr4    & ~pVmcsInfo->u64Cr4Mask)
+                           | (u64Shadow &  pVmcsInfo->u64Cr4Mask);
+#else
+                    if (!CPUMIsGuestInVmxNonRootMode(pCtx))
+                    {
+                        u64Cr4 = (u64Cr4    & ~pVmcsInfo->u64Cr4Mask)
+                               | (u64Shadow &  pVmcsInfo->u64Cr4Mask);
+                    }
+                    else
+                    {
+                        /*
+                         * We've merged the guest and nested-guest's CR4 guest/host mask while executing
+                         * the nested-guest using hardware-assisted VMX. Accordingly we need to
+                         * re-construct CR4. See @bugref{9180#c95} for details.
+                         */
+                        PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
+                        PVMXVVMCS const     pVmcsNstGst  = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+                        u64Cr4 = (u64Cr4                     & ~(pVmcsInfo->u64Cr4Mask & pVmcsNstGst->u64Cr4Mask.u))
+                               | (pVmcsNstGst->u64GuestCr4.u &   pVmcsNstGst->u64Cr4Mask.u)
+                               | (u64Shadow                  &  (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
+                        Assert(u64Cr4 & X86_CR4_VMXE);
+                    }
+#endif
+                    pCtx->cr4 = u64Cr4;
+                }
+
+                if (fWhat & CPUMCTX_EXTRN_CR3)
+                {
+                    /* CR0.PG bit changes are always intercepted, so it's up to date. */
+                    if (   VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
+                        || (   VM_IS_VMX_NESTED_PAGING(pVM)
+                            && CPUMIsGuestPagingEnabledEx(pCtx)))
+                    {
+                        uint64_t u64Cr3;
+                        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3);  AssertRC(rc);
+                        if (pCtx->cr3 != u64Cr3)
+                        {
+                            pCtx->cr3 = u64Cr3;
+                            VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
+                        }
+
+                        /*
+                         * If the guest is in PAE mode, sync back the PDPE's into the guest state.
+                         * CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
+                         */
+                        if (CPUMIsGuestInPAEModeEx(pCtx))
+                        {
+                            X86PDPE aPaePdpes[4];
+                            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u);     AssertRC(rc);
+                            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u);     AssertRC(rc);
+                            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u);     AssertRC(rc);
+                            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u);     AssertRC(rc);
+                            if (memcmp(&aPaePdpes[0], &pCtx->aPaePdpes[0], sizeof(aPaePdpes)))
+                            {
+                                memcpy(&pCtx->aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
+                                /* PGM now updates PAE PDPTEs while updating CR3. */
+                                VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
+                            }
+                        }
+                    }
+                }
+            }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            if (fWhat & CPUMCTX_EXTRN_HWVIRT)
+            {
+                if (   (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+                    && !CPUMIsGuestInVmxNonRootMode(pCtx))
+                {
+                    Assert(CPUMIsGuestInVmxRootMode(pCtx));
+                    rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
+                    if (RT_SUCCESS(rc))
+                    { /* likely */ }
+                    else
+                        break;
+                }
+            }
+#endif
+        } while (0);
+
+        if (RT_SUCCESS(rc))
+        {
+            /* Update fExtrn. */
+            pCtx->fExtrn &= ~fWhat;
+
+            /* If everything has been imported, clear the HM keeper bit. */
+            if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
+            {
+#ifndef IN_NEM_DARWIN
+                pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
+#else
+                pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
+#endif
+                Assert(!pCtx->fExtrn);
+            }
+        }
+    }
+#ifndef IN_NEM_DARWIN
+    else
+        AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
+
+    /*
+     * Restore interrupts.
+     */
+    ASMSetFlags(fEFlags);
+#endif
+
+    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
+
+    if (RT_SUCCESS(rc))
+    { /* likely */ }
+    else
+        return rc;
+
+    /*
+     * Honor any pending CR3 updates.
+     *
+     * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
+     * -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
+     * -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
+     *
+     * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
+     * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
+     * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
+     * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
+     *
+     * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
+     *
+     * The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
+     */
+    if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)
+#ifndef IN_NEM_DARWIN
+        && VMMRZCallRing3IsEnabled(pVCpu)
+#endif
+        )
+    {
+        Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
+        PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
+        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal state fetcher, inner version where we fetch all of a_fWhat.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ * @param   fEFlags     Saved EFLAGS for restoring the interrupt flag. Ignored
+ *                      in NEM/darwin context.
+ * @tparam  a_fWhat     What to import, zero or more bits from
+ *                      HMVMX_CPUMCTX_EXTRN_ALL.
+ */
+template<uint64_t const a_fWhat>
+static int vmxHCImportGuestStateInner(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t fEFlags)
+{
+    Assert(a_fWhat != 0); /* No AssertCompile as the assertion probably kicks in before the compiler (clang) discards it. */
+    AssertCompile(!(a_fWhat & ~HMVMX_CPUMCTX_EXTRN_ALL));
+    Assert(   (pVCpu->cpum.GstCtx.fExtrn & a_fWhat) == a_fWhat
+           || (pVCpu->cpum.GstCtx.fExtrn & a_fWhat) == (a_fWhat & ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)));
+
+    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
+
+    PVMCC const pVM  = pVCpu->CTX_SUFF(pVM);
+
+    /* RIP and RFLAGS may have been imported already by the post exit code
+       together with the CPUMCTX_EXTRN_INHIBIT_INT/NMI state, so this part
+       of the code is skipping this part of the code. */
+    if (   (a_fWhat & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS))
+        && pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS))
+    {
+        if (a_fWhat & CPUMCTX_EXTRN_RFLAGS)
+            vmxHCImportGuestCoreRFlags(pVCpu, pVmcsInfo);
+
+        if (a_fWhat & CPUMCTX_EXTRN_RIP)
+        {
+            if (!(a_fWhat & CPUMCTX_EXTRN_CS))
+                EMHistoryUpdatePC(pVCpu, vmxHCImportGuestCoreRip(pVCpu), false);
+            else
+                vmxHCImportGuestCoreRip(pVCpu);
+        }
+    }
+
+    /* Note! vmxHCImportGuestIntrState may also include RIP and RFLAGS and update fExtrn. */
+    if (a_fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
+        vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);
+
+    if (a_fWhat & (CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TR))
+    {
+        if (a_fWhat & CPUMCTX_EXTRN_CS)
+        {
+            vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
+            /** @todo try get rid of this carp, it smells and is probably never ever
+             *        used: */
+            if (   !(a_fWhat & CPUMCTX_EXTRN_RIP)
+                && (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP))
+            {
+                vmxHCImportGuestCoreRip(pVCpu);
+                pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
+            }
+            EMHistoryUpdatePC(pVCpu, pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, true /* fFlattened */);
+        }
+        if (a_fWhat & CPUMCTX_EXTRN_SS)
+            vmxHCImportGuestSegReg<X86_SREG_SS>(pVCpu);
+        if (a_fWhat & CPUMCTX_EXTRN_DS)
+            vmxHCImportGuestSegReg<X86_SREG_DS>(pVCpu);
+        if (a_fWhat & CPUMCTX_EXTRN_ES)
+            vmxHCImportGuestSegReg<X86_SREG_ES>(pVCpu);
+        if (a_fWhat & CPUMCTX_EXTRN_FS)
+            vmxHCImportGuestSegReg<X86_SREG_FS>(pVCpu);
+        if (a_fWhat & CPUMCTX_EXTRN_GS)
+            vmxHCImportGuestSegReg<X86_SREG_GS>(pVCpu);
+
+        /* Guest TR.
+           Real-mode emulation using virtual-8086 mode has the fake TSS
+           (pRealModeTSS) in TR, don't need to import that one. */
+#ifndef IN_NEM_DARWIN
+        PVMXVMCSINFOSHARED const pVmcsInfoShared  = pVmcsInfo->pShared;
+        bool const               fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
+        if ((a_fWhat & CPUMCTX_EXTRN_TR) && !fRealOnV86Active)
+#else
+        if (a_fWhat & CPUMCTX_EXTRN_TR)
+#endif
+            vmxHCImportGuestTr(pVCpu);
+
+#ifndef IN_NEM_DARWIN /* NEM/Darwin: HV supports only unrestricted guest execution. */
+        if (fRealOnV86Active)
+        {
+            if (a_fWhat & CPUMCTX_EXTRN_CS)
+                pVCpu->cpum.GstCtx.cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
+            if (a_fWhat & CPUMCTX_EXTRN_SS)
+                pVCpu->cpum.GstCtx.ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
+            if (a_fWhat & CPUMCTX_EXTRN_DS)
+                pVCpu->cpum.GstCtx.ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
+            if (a_fWhat & CPUMCTX_EXTRN_ES)
+                pVCpu->cpum.GstCtx.es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
+            if (a_fWhat & CPUMCTX_EXTRN_FS)
+                pVCpu->cpum.GstCtx.fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
+            if (a_fWhat & CPUMCTX_EXTRN_GS)
+                pVCpu->cpum.GstCtx.gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
+        }
+#endif
+    }
+
+    if (a_fWhat & CPUMCTX_EXTRN_RSP)
+    {
+        int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pVCpu->cpum.GstCtx.rsp);
+        AssertRC(rc);
+    }
+
+    if (a_fWhat & CPUMCTX_EXTRN_LDTR)
+        vmxHCImportGuestLdtr(pVCpu);
+
+    if (a_fWhat & CPUMCTX_EXTRN_GDTR)
+    {
+        int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE,    &pVCpu->cpum.GstCtx.gdtr.pGdt); AssertRC(rc1);
+        uint32_t u32Val;
+        int const rc2 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc2);
+        pVCpu->cpum.GstCtx.gdtr.cbGdt = (uint16_t)u32Val;
+    }
+
+    /* Guest IDTR. */
+    if (a_fWhat & CPUMCTX_EXTRN_IDTR)
+    {
+        int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE,    &pVCpu->cpum.GstCtx.idtr.pIdt); AssertRC(rc1);
+        uint32_t u32Val;
+        int const rc2 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc2);
+        pVCpu->cpum.GstCtx.idtr.cbIdt = (uint64_t)u32Val;
+    }
+
+    if (a_fWhat & CPUMCTX_EXTRN_DR7)
+    {
+#ifndef IN_NEM_DARWIN
+        if (!pVCpu->hmr0.s.fUsingHyperDR7)
+#endif
+        {
+            int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pVCpu->cpum.GstCtx.dr[7]);
+            AssertRC(rc);
+        }
+    }
+
+    if (a_fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
+    {
+        int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP,  &pVCpu->cpum.GstCtx.SysEnter.eip); AssertRC(rc1);
+        int const rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP,  &pVCpu->cpum.GstCtx.SysEnter.esp); AssertRC(rc2);
+        uint32_t u32Val;
+        int const rc3 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc3);
+        pVCpu->cpum.GstCtx.SysEnter.cs = u32Val;
+    }
+
+#ifndef IN_NEM_DARWIN
+    if (a_fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
+    {
+        if (   (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
+            && pVM->hmr0.s.fAllow64BitGuests)
+            pVCpu->cpum.GstCtx.msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
+    }
+
+    if (a_fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
+    {
+        if (   (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
+            && pVM->hmr0.s.fAllow64BitGuests)
+        {
+            pVCpu->cpum.GstCtx.msrLSTAR  = ASMRdMsr(MSR_K8_LSTAR);
+            pVCpu->cpum.GstCtx.msrSTAR   = ASMRdMsr(MSR_K6_STAR);
+            pVCpu->cpum.GstCtx.msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
+        }
+    }
+
+    if (a_fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
+    {
+        PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
+        PCVMXAUTOMSR       pMsrs           = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
+        uint32_t const     cMsrs           = pVmcsInfo->cExitMsrStore;
+        Assert(pMsrs);
+        Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
+        Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE);
+        for (uint32_t i = 0; i < cMsrs; i++)
+        {
+            uint32_t const idMsr = pMsrs[i].u32Msr;
+            switch (idMsr)
+            {
+                case MSR_K8_TSC_AUX:        CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value);     break;
+                case MSR_IA32_SPEC_CTRL:    CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value);   break;
+                case MSR_K6_EFER:           /* Can't be changed without causing a VM-exit */  break;
+                default:
+                {
+                    uint32_t idxLbrMsr;
+                    if (VM_IS_VMX_LBR(pVM))
+                    {
+                        if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
+                        {
+                            Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
+                            pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
+                            break;
+                        }
+                        if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
+                        {
+                            Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
+                            pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
+                            break;
+                        }
+                        if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
+                        {
+                            pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
+                            break;
+                        }
+                    }
+                    pVCpu->cpum.GstCtx.fExtrn = 0;
+                    VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr;
+                    ASMSetFlags(fEFlags);
+                    AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
+                    return VERR_HM_UNEXPECTED_LD_ST_MSR;
+                }
+            }
+        }
+    }
+#endif
+
+    if (a_fWhat & CPUMCTX_EXTRN_CR0)
+    {
+        uint64_t u64Cr0;
+        uint64_t u64Shadow;
+        int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0,            &u64Cr0);    AssertRC(rc1);
+        int const rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc2);
+#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
+        u64Cr0 = (u64Cr0    & ~pVmcsInfo->u64Cr0Mask)
+               | (u64Shadow &  pVmcsInfo->u64Cr0Mask);
+#else
+        if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+            u64Cr0 = (u64Cr0    & ~pVmcsInfo->u64Cr0Mask)
+                   | (u64Shadow &  pVmcsInfo->u64Cr0Mask);
+        else
+        {
+            /*
+             * We've merged the guest and nested-guest's CR0 guest/host mask while executing
+             * the nested-guest using hardware-assisted VMX. Accordingly we need to
+             * re-construct CR0. See @bugref{9180#c95} for details.
+             */
+            PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
+            PVMXVVMCS const     pVmcsNstGst  = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+            u64Cr0 = (u64Cr0                     & ~(pVmcsInfoGst->u64Cr0Mask & pVmcsNstGst->u64Cr0Mask.u))
+                   | (pVmcsNstGst->u64GuestCr0.u &   pVmcsNstGst->u64Cr0Mask.u)
+                   | (u64Shadow                  &  (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
+            Assert(u64Cr0 & X86_CR0_NE);
+        }
+#endif
+#ifndef IN_NEM_DARWIN
+        VMMRZCallRing3Disable(pVCpu);   /* May call into PGM which has Log statements. */
+#endif
+        CPUMSetGuestCR0(pVCpu, u64Cr0);
+#ifndef IN_NEM_DARWIN
+        VMMRZCallRing3Enable(pVCpu);
+#endif
+    }
+
+    if (a_fWhat & CPUMCTX_EXTRN_CR4)
+    {
+        uint64_t u64Cr4;
+        uint64_t u64Shadow;
+        int rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4,            &u64Cr4);    AssertRC(rc1);
+        int rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc2);
+#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
+        u64Cr4 = (u64Cr4    & ~pVmcsInfo->u64Cr4Mask)
+               | (u64Shadow &  pVmcsInfo->u64Cr4Mask);
+#else
+        if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+            u64Cr4 = (u64Cr4    & ~pVmcsInfo->u64Cr4Mask)
+                   | (u64Shadow &  pVmcsInfo->u64Cr4Mask);
+        else
+        {
+            /*
+             * We've merged the guest and nested-guest's CR4 guest/host mask while executing
+             * the nested-guest using hardware-assisted VMX. Accordingly we need to
+             * re-construct CR4. See @bugref{9180#c95} for details.
+             */
+            PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
+            PVMXVVMCS const     pVmcsNstGst  = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+            u64Cr4 = (u64Cr4                     & ~(pVmcsInfo->u64Cr4Mask & pVmcsNstGst->u64Cr4Mask.u))
+                   | (pVmcsNstGst->u64GuestCr4.u &   pVmcsNstGst->u64Cr4Mask.u)
+                   | (u64Shadow                  &  (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
+            Assert(u64Cr4 & X86_CR4_VMXE);
+        }
+#endif
+        pVCpu->cpum.GstCtx.cr4 = u64Cr4;
+    }
+
+    if (a_fWhat & CPUMCTX_EXTRN_CR3)
+    {
+        /* CR0.PG bit changes are always intercepted, so it's up to date. */
+        if (   VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
+            || (   VM_IS_VMX_NESTED_PAGING(pVM)
+                && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)))
+        {
+            uint64_t u64Cr3;
+            int const rc0 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3);  AssertRC(rc0);
+            if (pVCpu->cpum.GstCtx.cr3 != u64Cr3)
+            {
+                pVCpu->cpum.GstCtx.cr3 = u64Cr3;
+                VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
+            }
+
+            /*
+             * If the guest is in PAE mode, sync back the PDPE's into the guest state.
+             * CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
+             */
+            if (CPUMIsGuestInPAEModeEx(&pVCpu->cpum.GstCtx))
+            {
+                X86PDPE aPaePdpes[4];
+                int const rc1 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc1);
+                int const rc2 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc2);
+                int const rc3 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc3);
+                int const rc4 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc4);
+                if (memcmp(&aPaePdpes[0], &pVCpu->cpum.GstCtx.aPaePdpes[0], sizeof(aPaePdpes)))
+                {
+                    memcpy(&pVCpu->cpum.GstCtx.aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
+                    /* PGM now updates PAE PDPTEs while updating CR3. */
+                    VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
+                }
+            }
+        }
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    if (a_fWhat & CPUMCTX_EXTRN_HWVIRT)
+    {
+        if (   (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
+            && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+        {
+            Assert(CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx));
+            int const rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
+            AssertRCReturn(rc, rc);
+        }
+    }
+#endif
+
+    /* Update fExtrn. */
+    pVCpu->cpum.GstCtx.fExtrn &= ~a_fWhat;
+
+    /* If everything has been imported, clear the HM keeper bit. */
+    if (!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
+    {
+#ifndef IN_NEM_DARWIN
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
+#else
+        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
+#endif
+        Assert(!pVCpu->cpum.GstCtx.fExtrn);
+    }
+
+    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);
+
+    /*
+     * Honor any pending CR3 updates.
+     *
+     * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
+     * -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
+     * -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
+     *
+     * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
+     * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
+     * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
+     * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
+     *
+     * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
+     *
+     * The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
+     */
+#ifndef IN_NEM_DARWIN
+    if (!(a_fWhat & CPUMCTX_EXTRN_CR3)
+        ? RT_LIKELY(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) || !VMMRZCallRing3IsEnabled(pVCpu))
+        :           !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) || !VMMRZCallRing3IsEnabled(pVCpu) )
+        return VINF_SUCCESS;
+    ASMSetFlags(fEFlags);
+#else
+    if (!(a_fWhat & CPUMCTX_EXTRN_CR3)
+        ? RT_LIKELY(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
+        :           !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) )
+        return VINF_SUCCESS;
+    RT_NOREF_PV(fEFlags);
+#endif
+
+    Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
+    PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
+    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Internal state fetcher.
+ *
+ * @returns VBox status code.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS info. object.
+ * @param   pszCaller       For logging.
+ * @tparam  a_fWhat         What needs to be imported, CPUMCTX_EXTRN_XXX.
+ * @tparam  a_fDoneLocal    What's ASSUMED to have been retrieved locally
+ *                          already.  This is ORed together with @a a_fWhat when
+ *                          calculating what needs fetching (just for safety).
+ * @tparam  a_fDonePostExit What's ASSUMED to been been retrieved by
+ *                          hmR0VmxPostRunGuest()/nemR3DarwinHandleExitCommon()
+ *                          already.  This is ORed together with @a a_fWhat when
+ *                          calculating what needs fetching (just for safety).
+ */
+template<uint64_t const a_fWhat,
+    uint64_t const a_fDoneLocal = 0,
+    uint64_t const a_fDonePostExit = 0
+#ifndef IN_NEM_DARWIN
+                                   | CPUMCTX_EXTRN_INHIBIT_INT
+                                   | CPUMCTX_EXTRN_INHIBIT_NMI
+# if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
+                                   | HMVMX_CPUMCTX_EXTRN_ALL
+# elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
+                                   | CPUMCTX_EXTRN_RFLAGS
+# endif
+#else  /* IN_NEM_DARWIN */
+                                   | CPUMCTX_EXTRN_ALL /** @todo optimize */
+#endif /* IN_NEM_DARWIN */
+>
+DECLINLINE(int) vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, const char *pszCaller)
+{
+    RT_NOREF_PV(pszCaller);
+    if ((a_fWhat | a_fDoneLocal | a_fDonePostExit) & HMVMX_CPUMCTX_EXTRN_ALL)
+    {
+#ifndef IN_NEM_DARWIN
+        /*
+         * We disable interrupts to make the updating of the state and in particular
+         * the fExtrn modification atomic wrt to preemption hooks.
+         */
+        RTCCUINTREG const fEFlags   = ASMIntDisableFlags();
+#else
+        RTCCUINTREG const fEFlags   = 0;
+#endif
+
+        /*
+         * We combine all three parameters and take the (probably) inlined optimized
+         * code path for the new things specified in a_fWhat.
+         *
+         * As a tweak to deal with exits that have INHIBIT_INT/NMI active, causing
+         * vmxHCImportGuestIntrState to automatically fetch both RIP & RFLAGS, we
+         * also take the streamlined path when both of these are cleared in fExtrn
+         * already. vmxHCImportGuestStateInner checks fExtrn before fetching.  This
+         * helps with MWAIT and HLT exits that always inhibit IRQs on many platforms.
+         */
+        uint64_t const    fWhatToDo = pVCpu->cpum.GstCtx.fExtrn
+                                    & ((a_fWhat | a_fDoneLocal | a_fDonePostExit) & HMVMX_CPUMCTX_EXTRN_ALL);
+        if (RT_LIKELY(   (   fWhatToDo ==   (a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal | a_fDonePostExit))
+                          || fWhatToDo == (  a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal | a_fDonePostExit)
+                                           & ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)) /* fetch with INHIBIT_INT/NMI */))
+            && (a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal | a_fDonePostExit)) != 0 /* just in case */)
+        {
+            int const rc = vmxHCImportGuestStateInner<  a_fWhat
+                                                      & HMVMX_CPUMCTX_EXTRN_ALL
+                                                      & ~(a_fDoneLocal | a_fDonePostExit)>(pVCpu, pVmcsInfo, fEFlags);
+#ifndef IN_NEM_DARWIN
+            ASMSetFlags(fEFlags);
+#endif
+            return rc;
+        }
+
+#ifndef IN_NEM_DARWIN
+        ASMSetFlags(fEFlags);
+#endif
+
+        /*
+         * We shouldn't normally get here, but it may happen when executing
+         * in the debug run-loops.  Typically, everything should already have
+         * been fetched then.  Otherwise call the fallback state import function.
+         */
+        if (fWhatToDo == 0)
+        { /* hope the cause was the debug loop or something similar */ }
+        else
+        {
+            STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestStateFallback);
+            Log11Func(("a_fWhat=%#RX64/%#RX64/%#RX64 fExtrn=%#RX64 => %#RX64 - Taking inefficient code path from %s!\n",
+                       a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL, a_fDoneLocal & HMVMX_CPUMCTX_EXTRN_ALL,
+                       a_fDonePostExit & HMVMX_CPUMCTX_EXTRN_ALL, pVCpu->cpum.GstCtx.fExtrn, fWhatToDo, pszCaller));
+            return vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, a_fWhat | a_fDoneLocal | a_fDonePostExit);
+        }
+    }
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Check per-VM and per-VCPU force flag actions that require us to go back to
+ * ring-3 for one reason or another.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too)
+ * @retval VINF_SUCCESS if we don't have any actions that require going back to
+ *         ring-3.
+ * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
+ * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
+ *         interrupts)
+ * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
+ *         all EMTs to be in ring-3.
+ * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
+ * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
+ *         to the EM loop.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   fIsNestedGuest  Flag whether this is for a for a pending nested guest event.
+ * @param   fStepping       Whether we are single-stepping the guest using the
+ *                          hypervisor debugger.
+ *
+ * @remarks This might cause nested-guest VM-exits, caller must check if the guest
+ *          is no longer in VMX non-root mode.
+ */
+static VBOXSTRICTRC vmxHCCheckForceFlags(PVMCPUCC pVCpu, bool fIsNestedGuest, bool fStepping)
+{
+#ifndef IN_NEM_DARWIN
+    Assert(VMMRZCallRing3IsEnabled(pVCpu));
+#endif
+
+    /*
+     * Update pending interrupts into the APIC's IRR.
+     */
+    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
+        APICUpdatePendingInterrupts(pVCpu);
+
+    /*
+     * Anything pending?  Should be more likely than not if we're doing a good job.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (  !fStepping
+        ?    !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
+          && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
+        :    !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
+          && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
+        return VINF_SUCCESS;
+
+    /* Pending PGM C3 sync. */
+    if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
+    {
+        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
+        VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
+                                           VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
+        if (rcStrict != VINF_SUCCESS)
+        {
+            AssertRC(VBOXSTRICTRC_VAL(rcStrict));
+            Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+    }
+
+    /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
+    if (   VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
+        || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
+    {
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHmToR3FF);
+        int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
+        Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d (fVM=%#RX64 fCpu=%#RX64)\n",
+                  rc, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
+        return rc;
+    }
+
+    /* Pending VM request packets, such as hardware interrupts. */
+    if (   VM_FF_IS_SET(pVM, VM_FF_REQUEST)
+        || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
+    {
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchVmReq);
+        Log4Func(("Pending VM request forcing us back to ring-3\n"));
+        return VINF_EM_PENDING_REQUEST;
+    }
+
+    /* Pending PGM pool flushes. */
+    if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
+    {
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchPgmPoolFlush);
+        Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
+        return VINF_PGM_POOL_FLUSH_PENDING;
+    }
+
+    /* Pending DMA requests. */
+    if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
+    {
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchDma);
+        Log4Func(("Pending DMA request forcing us back to ring-3\n"));
+        return VINF_EM_RAW_TO_R3;
+    }
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+    /*
+     * Pending nested-guest events.
+     *
+     * Please note the priority of these events are specified and important.
+     * See Intel spec. 29.4.3.2 "APIC-Write Emulation".
+     * See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
+     */
+    if (fIsNestedGuest)
+    {
+        /* Pending nested-guest APIC-write. */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
+        {
+            Log4Func(("Pending nested-guest APIC-write\n"));
+            VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
+            Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+            return rcStrict;
+        }
+
+        /* Pending nested-guest monitor-trap flag (MTF). */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
+        {
+            Log4Func(("Pending nested-guest MTF\n"));
+            VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
+            Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+            return rcStrict;
+        }
+
+        /* Pending nested-guest VMX-preemption timer expired. */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
+        {
+            Log4Func(("Pending nested-guest preempt timer\n"));
+            VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
+            Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+            return rcStrict;
+        }
+    }
+#else
+    NOREF(fIsNestedGuest);
+#endif
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Converts any TRPM trap into a pending HM event. This is typically used when
+ * entering from ring-3 (not longjmp returns).
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void vmxHCTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
+{
+    Assert(TRPMHasTrap(pVCpu));
+    Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
+
+    uint8_t     uVector;
+    TRPMEVENT   enmTrpmEvent;
+    uint32_t    uErrCode;
+    RTGCUINTPTR GCPtrFaultAddress;
+    uint8_t     cbInstr;
+    bool        fIcebp;
+
+    int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
+    AssertRC(rc);
+
+    uint32_t u32IntInfo;
+    u32IntInfo  = uVector | VMX_IDT_VECTORING_INFO_VALID;
+    u32IntInfo |= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);
+
+    rc = TRPMResetTrap(pVCpu);
+    AssertRC(rc);
+    Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
+          u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
+
+    vmxHCSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
+}
+
+
+/**
+ * Converts the pending HM event into a TRPM trap.
+ *
+ * @param   pVCpu   The cross context virtual CPU structure.
+ */
+static void vmxHCPendingEventToTrpmTrap(PVMCPUCC pVCpu)
+{
+    Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
+
+    /* If a trap was already pending, we did something wrong! */
+    Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
+
+    uint32_t const  u32IntInfo  = VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo;
+    uint32_t const  uVector     = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
+    TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);
+
+    Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
+
+    int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
+    AssertRC(rc);
+
+    if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
+        TRPMSetErrorCode(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode);
+
+    if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
+        TRPMSetFaultAddress(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress);
+    else
+    {
+        uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
+        switch (uVectorType)
+        {
+            case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
+                TRPMSetTrapDueToIcebp(pVCpu);
+                RT_FALL_THRU();
+            case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
+            case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
+            {
+                AssertMsg(   uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
+                          || (   uVector == X86_XCPT_BP /* INT3 */
+                              || uVector == X86_XCPT_OF /* INTO */
+                              || uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
+                          ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
+                TRPMSetInstrLength(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.cbInstr);
+                break;
+            }
+        }
+    }
+
+    /* We're now done converting the pending event. */
+    VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
+}
+
+
+/**
+ * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
+ * cause a VM-exit as soon as the guest is in a state to receive interrupts.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static void vmxHCSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
+    {
+        if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
+        {
+            pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
+            int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+            AssertRC(rc);
+        }
+    } /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
+}
+
+
+/**
+ * Clears the interrupt-window exiting control in the VMCS.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+DECLINLINE(void) vmxHCClearIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
+    {
+        pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
+        int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+        AssertRC(rc);
+    }
+}
+
+
+/**
+ * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
+ * cause a VM-exit as soon as the guest is in a state to receive NMIs.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+static void vmxHCSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
+    {
+        if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
+        {
+            pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
+            int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+            AssertRC(rc);
+            Log4Func(("Setup NMI-window exiting\n"));
+        }
+    } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
+}
+
+
+/**
+ * Clears the NMI-window exiting control in the VMCS.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ */
+DECLINLINE(void) vmxHCClearNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
+{
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
+    {
+        pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
+        int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+        AssertRC(rc);
+    }
+}
+
+
+/**
+ * Injects an event into the guest upon VM-entry by updating the relevant fields
+ * in the VM-entry area in the VMCS.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS if the event is successfully injected into the VMCS.
+ * @retval  VINF_EM_RESET if event injection resulted in a triple-fault.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS info object.
+ * @param   fIsNestedGuest  Flag whether this is for a for a pending nested guest event.
+ * @param   pEvent          The event being injected.
+ * @param   pfIntrState     Pointer to the VT-x guest-interruptibility-state. This
+ *                          will be updated if necessary. This cannot not be NULL.
+ * @param   fStepping       Whether we're single-stepping guest execution and should
+ *                          return VINF_EM_DBG_STEPPED if the event is injected
+ *                          directly (registers modified by us, not by hardware on
+ *                          VM-entry).
+ */
+static VBOXSTRICTRC vmxHCInjectEventVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, PCHMEVENT pEvent,
+                                         bool fStepping, uint32_t *pfIntrState)
+{
+    /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
+    AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
+    Assert(pfIntrState);
+
+#ifdef IN_NEM_DARWIN
+    RT_NOREF(fIsNestedGuest, fStepping, pfIntrState);
+#endif
+
+    PCPUMCTX          pCtx       = &pVCpu->cpum.GstCtx;
+    uint32_t          u32IntInfo = pEvent->u64IntInfo;
+    uint32_t const    u32ErrCode = pEvent->u32ErrCode;
+    uint32_t const    cbInstr    = pEvent->cbInstr;
+    RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
+    uint8_t const     uVector    = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
+    uint32_t const    uIntType   = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
+
+#ifdef VBOX_STRICT
+    /*
+     * Validate the error-code-valid bit for hardware exceptions.
+     * No error codes for exceptions in real-mode.
+     *
+     * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
+     */
+    if (   uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
+        && !CPUMIsGuestInRealModeEx(pCtx))
+    {
+        switch (uVector)
+        {
+            case X86_XCPT_PF:
+            case X86_XCPT_DF:
+            case X86_XCPT_TS:
+            case X86_XCPT_NP:
+            case X86_XCPT_SS:
+            case X86_XCPT_GP:
+            case X86_XCPT_AC:
+                AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
+                          ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
+                RT_FALL_THRU();
+            default:
+                break;
+        }
+    }
+
+    /* Cannot inject an NMI when block-by-MOV SS is in effect. */
+    Assert(   uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
+           || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
+#endif
+
+    RT_NOREF(uVector);
+    if (   uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
+        || uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
+        || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
+        || uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
+    {
+        Assert(uVector <= X86_XCPT_LAST);
+        Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI          || uVector == X86_XCPT_NMI);
+        Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT || uVector == X86_XCPT_DB);
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedXcpts[uVector]);
+    }
+    else
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedIrqs[uVector & MASK_INJECT_IRQ_STAT]);
+
+    /*
+     * Hardware interrupts & exceptions cannot be delivered through the software interrupt
+     * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
+     * interrupt handler in the (real-mode) guest.
+     *
+     * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
+     * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
+     */
+    if (CPUMIsGuestInRealModeEx(pCtx))     /* CR0.PE bit changes are always intercepted, so it's up to date. */
+    {
+#ifndef IN_NEM_DARWIN
+        if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest)
+#endif
+        {
+            /*
+             * For CPUs with unrestricted guest execution enabled and with the guest
+             * in real-mode, we must not set the deliver-error-code bit.
+             *
+             * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
+             */
+            u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
+        }
+#ifndef IN_NEM_DARWIN
+        else
+        {
+            PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+            Assert(PDMVmmDevHeapIsEnabled(pVM));
+            Assert(pVM->hm.s.vmx.pRealModeTSS);
+            Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
+
+            /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
+            int rc2 = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK
+                                                                | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
+            AssertRCReturn(rc2, rc2);
+
+            /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
+            size_t const cbIdtEntry = sizeof(X86IDTR16);
+            if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
+            {
+                /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
+                if (uVector == X86_XCPT_DF)
+                    return VINF_EM_RESET;
+
+                /* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
+                   No error codes for exceptions in real-mode. */
+                if (uVector == X86_XCPT_GP)
+                {
+                    static HMEVENT const s_EventXcptDf
+                        = HMEVENT_INIT_ONLY_INT_INFO(  RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
+                                                     | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,   VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
+                                                     | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                                                     | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1));
+                    return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptDf, fStepping, pfIntrState);
+                }
+
+                /*
+                 * If we're injecting an event with no valid IDT entry, inject a #GP.
+                 * No error codes for exceptions in real-mode.
+                 *
+                 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
+                 */
+                static HMEVENT const s_EventXcptGp
+                    = HMEVENT_INIT_ONLY_INT_INFO(  RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
+                                                 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,   VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
+                                                 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
+                                                 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1));
+                return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptGp, fStepping, pfIntrState);
+            }
+
+            /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
+            uint16_t uGuestIp = pCtx->ip;
+            if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
+            {
+                Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
+                /* #BP and #OF are both benign traps, we need to resume the next instruction. */
+                uGuestIp = pCtx->ip + (uint16_t)cbInstr;
+            }
+            else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
+                uGuestIp = pCtx->ip + (uint16_t)cbInstr;
+
+            /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
+            X86IDTR16 IdtEntry;
+            RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
+            rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
+            AssertRCReturn(rc2, rc2);
+
+            /* Construct the stack frame for the interrupt/exception handler. */
+            VBOXSTRICTRC rcStrict;
+            rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, (uint16_t)pCtx->eflags.u);
+            if (rcStrict == VINF_SUCCESS)
+            {
+                rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
+                if (rcStrict == VINF_SUCCESS)
+                    rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
+            }
+
+            /* Clear the required eflag bits and jump to the interrupt/exception handler. */
+            if (rcStrict == VINF_SUCCESS)
+            {
+                pCtx->eflags.u   &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
+                pCtx->rip         = IdtEntry.offSel;
+                pCtx->cs.Sel      = IdtEntry.uSel;
+                pCtx->cs.ValidSel = IdtEntry.uSel;
+                pCtx->cs.u64Base  = IdtEntry.uSel << cbIdtEntry;
+                if (   uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
+                    && uVector  == X86_XCPT_PF)
+                    pCtx->cr2 = GCPtrFault;
+
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CS  | HM_CHANGED_GUEST_CR2
+                                                         | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
+                                                         | HM_CHANGED_GUEST_RSP);
+
+                /*
+                 * If we delivered a hardware exception (other than an NMI) and if there was
+                 * block-by-STI in effect, we should clear it.
+                 */
+                if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+                {
+                    Assert(   uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
+                           && uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
+                    Log4Func(("Clearing inhibition due to STI\n"));
+                    *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
+                }
+
+                Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
+                      u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
+
+                /*
+                 * The event has been truly dispatched to the guest. Mark it as no longer pending so
+                 * we don't attempt to undo it if we are returning to ring-3 before executing guest code.
+                 */
+                VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
+
+                /*
+                 * If we eventually support nested-guest execution without unrestricted guest execution,
+                 * we should set fInterceptEvents here.
+                 */
+                Assert(!fIsNestedGuest);
+
+                /* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
+                if (fStepping)
+                    rcStrict = VINF_EM_DBG_STEPPED;
+            }
+            AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
+                      ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            return rcStrict;
+        }
+#else
+        RT_NOREF(pVmcsInfo);
+#endif
+    }
+
+    /*
+     * Validate.
+     */
+    Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo));                     /* Bit 31 (Valid bit) must be set by caller. */
+    Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK));       /* Bits 30:12 MBZ. */
+
+    /*
+     * Inject the event into the VMCS.
+     */
+    int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
+    if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
+        rc |= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
+    rc |= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
+    AssertRC(rc);
+
+    /*
+     * Update guest CR2 if this is a page-fault.
+     */
+    if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
+        pCtx->cr2 = GCPtrFault;
+
+    Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Evaluates the event to be delivered to the guest and sets it as the pending
+ * event.
+ *
+ * Toggling of interrupt force-flags here is safe since we update TRPM on premature
+ * exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must
+ * NOT restore these force-flags.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS information structure.
+ * @param   fIsNestedGuest  Flag whether the evaluation happens for a nested guest.
+ * @param   pfIntrState     Where to store the VT-x guest-interruptibility state.
+ */
+static VBOXSTRICTRC vmxHCEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, uint32_t *pfIntrState)
+{
+    Assert(pfIntrState);
+    Assert(!TRPMHasTrap(pVCpu));
+
+    /*
+     * Compute/update guest-interruptibility state related FFs.
+     * The FFs will be used below while evaluating events to be injected.
+     */
+    *pfIntrState = vmxHCGetGuestIntrStateAndUpdateFFs(pVCpu);
+
+    /*
+     * Evaluate if a new event needs to be injected.
+     * An event that's already pending has already performed all necessary checks.
+     */
+    if (   !VCPU_2_VMXSTATE(pVCpu).Event.fPending
+        && !CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx))
+    {
+        /** @todo SMI. SMIs take priority over NMIs. */
+
+        /*
+         * NMIs.
+         * NMIs take priority over external interrupts.
+         */
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+#endif
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
+        {
+            /*
+             * For a guest, the FF always indicates the guest's ability to receive an NMI.
+             *
+             * For a nested-guest, the FF always indicates the outer guest's ability to
+             * receive an NMI while the guest-interruptibility state bit depends on whether
+             * the nested-hypervisor is using virtual-NMIs.
+             */
+            if (!CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
+            {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                if (   fIsNestedGuest
+                    && CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
+                    return IEMExecVmxVmexitXcptNmi(pVCpu);
+#endif
+                vmxHCSetPendingXcptNmi(pVCpu);
+                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
+                Log4Func(("NMI pending injection\n"));
+
+                /* We've injected the NMI, bail. */
+                return VINF_SUCCESS;
+            }
+            if (!fIsNestedGuest)
+                vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
+        }
+
+        /*
+         * External interrupts (PIC/APIC).
+         * Once PDMGetInterrupt() returns a valid interrupt we -must- deliver it.
+         * We cannot re-request the interrupt from the controller again.
+         */
+        if (    VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
+            && !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
+        {
+            Assert(!DBGFIsStepping(pVCpu));
+            int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
+            AssertRC(rc);
+
+            /*
+             * We must not check EFLAGS directly when executing a nested-guest, use
+             * CPUMIsGuestPhysIntrEnabled() instead as EFLAGS.IF does not control the blocking of
+             * external interrupts when "External interrupt exiting" is set. This fixes a nasty
+             * SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued by
+             * other VM-exits (like a preemption timer), see @bugref{9562#c18}.
+             *
+             * See Intel spec. 25.4.1 "Event Blocking".
+             */
+            if (CPUMIsGuestPhysIntrEnabled(pVCpu))
+            {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                if (    fIsNestedGuest
+                    &&  CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
+                {
+                    VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector */, true /* fIntPending */);
+                    if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
+                        return rcStrict;
+                }
+#endif
+                uint8_t u8Interrupt;
+                rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
+                if (RT_SUCCESS(rc))
+                {
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+                    if (   fIsNestedGuest
+                        && CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
+                    {
+                        VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
+                        Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
+                        return rcStrict;
+                    }
+#endif
+                    vmxHCSetPendingExtInt(pVCpu, u8Interrupt);
+                    Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
+                }
+                else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
+                {
+                    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchTprMaskedIrq);
+
+                    if (   !fIsNestedGuest
+                        && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
+                        vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
+                    /* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */
+
+                    /*
+                     * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
+                     * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
+                     * need to re-set this force-flag here.
+                     */
+                }
+                else
+                    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchGuestIrq);
+
+                /* We've injected the interrupt or taken necessary action, bail. */
+                return VINF_SUCCESS;
+            }
+            if (!fIsNestedGuest)
+                vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
+        }
+    }
+    else if (!fIsNestedGuest)
+    {
+        /*
+         * An event is being injected or we are in an interrupt shadow. Check if another event is
+         * pending. If so, instruct VT-x to cause a VM-exit as soon as the guest is ready to accept
+         * the pending event.
+         */
+        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
+            vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
+        else if (   VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
+                 && !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
+            vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
+    }
+    /* else: for nested-guests, NMI/interrupt-window exiting will be picked up when merging VMCS controls. */
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Injects any pending events into the guest if the guest is in a state to
+ * receive them.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmcsInfo       The VMCS information structure.
+ * @param   fIsNestedGuest  Flag whether the event injection happens for a nested guest.
+ * @param   fIntrState      The VT-x guest-interruptibility state.
+ * @param   fStepping       Whether we are single-stepping the guest using the
+ *                          hypervisor debugger and should return
+ *                          VINF_EM_DBG_STEPPED if the event was dispatched
+ *                          directly.
+ */
+static VBOXSTRICTRC vmxHCInjectPendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest,
+                                            uint32_t fIntrState, bool fStepping)
+{
+    HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
+#ifndef IN_NEM_DARWIN
+    Assert(VMMRZCallRing3IsEnabled(pVCpu));
+#endif
+
+#ifdef VBOX_STRICT
+    /*
+     * Verify guest-interruptibility state.
+     *
+     * We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
+     * since injecting an event may modify the interruptibility state and we must thus always
+     * use fIntrState.
+     */
+    {
+        bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
+        bool const fBlockSti   = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
+        Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
+        Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF);     /* Cannot set block-by-STI when interrupts are disabled. */
+        Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));    /* We don't support block-by-SMI yet.*/
+        Assert(!TRPMHasTrap(pVCpu));
+        NOREF(fBlockMovSS); NOREF(fBlockSti);
+    }
+#endif
+
+    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
+    if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
+    {
+        /*
+         * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
+         * pending even while injecting an event and in this case, we want a VM-exit as soon as
+         * the guest is ready for the next interrupt, see @bugref{6208#c45}.
+         *
+         * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
+         */
+        uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo);
+#ifdef VBOX_STRICT
+        if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
+        {
+            Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
+            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
+            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
+        }
+        else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
+        {
+            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
+            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
+            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
+        }
+#endif
+        Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
+              uIntType));
+
+        /*
+         * Inject the event and get any changes to the guest-interruptibility state.
+         *
+         * The guest-interruptibility state may need to be updated if we inject the event
+         * into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
+         */
+        rcStrict = vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &VCPU_2_VMXSTATE(pVCpu).Event, fStepping, &fIntrState);
+        AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
+
+        if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterrupt);
+        else
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectXcpt);
+    }
+
+    /*
+     * Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
+     * is an interrupt shadow (block-by-STI or block-by-MOV SS).
+     */
+    if (   (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
+        && !fIsNestedGuest)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
+
+        if (!VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
+        {
+            /*
+             * Set or clear the BS bit depending on whether the trap flag is active or not. We need
+             * to do both since we clear the BS bit from the VMCS while exiting to ring-3.
+             */
+            Assert(!DBGFIsStepping(pVCpu));
+            uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_TF);
+            int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
+                                       fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
+            AssertRC(rc);
+        }
+        else
+        {
+            /*
+             * We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
+             * hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
+             * we take care of this case in vmxHCExportSharedDebugState and also the case if
+             * we use MTF, so just make sure it's called before executing guest-code.
+             */
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
+        }
+    }
+    /* else: for nested-guest currently handling while merging controls. */
+
+    /*
+     * Finally, update the guest-interruptibility state.
+     *
+     * This is required for the real-on-v86 software interrupt injection, for
+     * pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
+     */
+    int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
+    AssertRC(rc);
+
+    /*
+     * There's no need to clear the VM-entry interruption-information field here if we're not
+     * injecting anything. VT-x clears the valid bit on every VM-exit.
+     *
+     * See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
+     */
+
+    Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
+    return rcStrict;
+}
+
+
+/**
+ * Tries to determine what part of the guest-state VT-x has deemed as invalid
+ * and update error record fields accordingly.
+ *
+ * @returns VMX_IGS_* error codes.
+ * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
+ *         wrong with the guest state.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   pVmcsInfo   The VMCS info. object.
+ *
+ * @remarks This function assumes our cache of the VMCS controls
+ *          are valid, i.e. vmxHCCheckCachedVmcsCtls() succeeded.
+ */
+static uint32_t vmxHCCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
+{
+#define HMVMX_ERROR_BREAK(err)              { uError = (err); break; }
+#define HMVMX_CHECK_BREAK(expr, err)        if (!(expr)) { uError = (err); break; } else do { } while (0)
+
+    PCPUMCTX pCtx   = &pVCpu->cpum.GstCtx;
+    uint32_t uError = VMX_IGS_ERROR;
+    uint32_t u32IntrState = 0;
+#ifndef IN_NEM_DARWIN
+    PVMCC    pVM    = pVCpu->CTX_SUFF(pVM);
+    bool const fUnrestrictedGuest = VM_IS_VMX_UNRESTRICTED_GUEST(pVM);
+#else
+    bool const fUnrestrictedGuest = true;
+#endif
+    do
+    {
+        int rc;
+
+        /*
+         * Guest-interruptibility state.
+         *
+         * Read this first so that any check that fails prior to those that actually
+         * require the guest-interruptibility state would still reflect the correct
+         * VMCS value and avoids causing further confusion.
+         */
+        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
+        AssertRC(rc);
+
+        uint32_t u32Val;
+        uint64_t u64Val;
+
+        /*
+         * CR0.
+         */
+        /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
+        uint64_t       fSetCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 & g_HmMsrs.u.vmx.u64Cr0Fixed1);
+        uint64_t const fZapCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 | g_HmMsrs.u.vmx.u64Cr0Fixed1);
+        /* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
+           See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
+        if (fUnrestrictedGuest)
+            fSetCr0 &= ~(uint64_t)(X86_CR0_PE | X86_CR0_PG);
+
+        uint64_t u64GuestCr0;
+        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64GuestCr0);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
+        HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
+        if (   !fUnrestrictedGuest
+            &&  (u64GuestCr0 & X86_CR0_PG)
+            && !(u64GuestCr0 & X86_CR0_PE))
+            HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
+
+        /*
+         * CR4.
+         */
+        /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
+        uint64_t const fSetCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 & g_HmMsrs.u.vmx.u64Cr4Fixed1);
+        uint64_t const fZapCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 | g_HmMsrs.u.vmx.u64Cr4Fixed1);
+
+        uint64_t u64GuestCr4;
+        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64GuestCr4);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
+        HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
+
+        /*
+         * IA32_DEBUGCTL MSR.
+         */
+        rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
+        AssertRC(rc);
+        if (   (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+            && (u64Val & 0xfffffe3c))                           /* Bits 31:9, bits 5:2 MBZ. */
+        {
+            HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
+        }
+        uint64_t u64DebugCtlMsr = u64Val;
+
+#ifdef VBOX_STRICT
+        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
+        AssertRC(rc);
+        Assert(u32Val == pVmcsInfo->u32EntryCtls);
+#endif
+        bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
+
+        /*
+         * RIP and RFLAGS.
+         */
+        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
+        AssertRC(rc);
+        /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
+        if (   !fLongModeGuest
+            || !pCtx->cs.Attr.n.u1Long)
+        {
+            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
+        }
+        /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
+         *        must be identical if the "IA-32e mode guest" VM-entry
+         *        control is 1 and CS.L is 1. No check applies if the
+         *        CPU supports 64 linear-address bits. */
+
+        /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
+        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)),                     /* Bit 63:22, Bit 15, 5, 3 MBZ. */
+                          VMX_IGS_RFLAGS_RESERVED);
+        HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1);       /* Bit 1 MB1. */
+        uint32_t const u32Eflags = u64Val;
+
+        if (   fLongModeGuest
+            || (   fUnrestrictedGuest
+                && !(u64GuestCr0 & X86_CR0_PE)))
+        {
+            HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
+        }
+
+        uint32_t u32EntryInfo;
+        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
+        AssertRC(rc);
+        if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
+        {
+            HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
+        }
+
+        /*
+         * 64-bit checks.
+         */
+        if (fLongModeGuest)
+        {
+            HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG,  VMX_IGS_CR0_PG_LONGMODE);
+            HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
+        }
+
+        if (   !fLongModeGuest
+            && (u64GuestCr4 & X86_CR4_PCIDE))
+            HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
+
+        /** @todo CR3 field must be such that bits 63:52 and bits in the range
+         *        51:32 beyond the processor's physical-address width are 0. */
+
+        if (   (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
+            && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
+            HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
+
+#ifndef IN_NEM_DARWIN
+        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
+
+        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
+#endif
+
+        /*
+         * PERF_GLOBAL MSR.
+         */
+        if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
+        {
+            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
+                              VMX_IGS_PERF_GLOBAL_MSR_RESERVED);        /* Bits 63:35, bits 31:2 MBZ. */
+        }
+
+        /*
+         * PAT MSR.
+         */
+        if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
+        {
+            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
+            for (unsigned i = 0; i < 8; i++)
+            {
+                uint8_t u8Val = (u64Val & 0xff);
+                if (   u8Val != 0 /* UC */
+                    && u8Val != 1 /* WC */
+                    && u8Val != 4 /* WT */
+                    && u8Val != 5 /* WP */
+                    && u8Val != 6 /* WB */
+                    && u8Val != 7 /* UC- */)
+                    HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
+                u64Val >>= 8;
+            }
+        }
+
+        /*
+         * EFER MSR.
+         */
+        if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
+        {
+            Assert(g_fHmVmxSupportsVmcsEfer);
+            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
+                              VMX_IGS_EFER_MSR_RESERVED);               /* Bits 63:12, bit 9, bits 7:1 MBZ. */
+            HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(  pVmcsInfo->u32EntryCtls
+                                                                           & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
+                              VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
+            /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
+             *        iemVmxVmentryCheckGuestState(). */
+            HMVMX_CHECK_BREAK(   fUnrestrictedGuest
+                              || !(u64GuestCr0 & X86_CR0_PG)
+                              || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
+                              VMX_IGS_EFER_LMA_LME_MISMATCH);
+        }
+
+        /*
+         * Segment registers.
+         */
+        HMVMX_CHECK_BREAK(   (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
+                          || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
+        if (!(u32Eflags & X86_EFL_VM))
+        {
+            /* CS */
+            HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
+            HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
+            HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
+            HMVMX_CHECK_BREAK(   (pCtx->cs.u32Limit & 0xfff) == 0xfff
+                              || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
+            HMVMX_CHECK_BREAK(   !(pCtx->cs.u32Limit & 0xfff00000)
+                              || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
+            /* CS cannot be loaded with NULL in protected mode. */
+            HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
+            HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
+            if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
+                HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
+            else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
+                HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
+            else if (fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
+                HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
+            else
+                HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
+
+            /* SS */
+            HMVMX_CHECK_BREAK(   fUnrestrictedGuest
+                              || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
+            HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
+            if (   !(pCtx->cr0 & X86_CR0_PE)
+                || pCtx->cs.Attr.n.u4Type == 3)
+            {
+                HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
+            }
+
+            if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
+            {
+                HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
+                HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
+                HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(   (pCtx->ss.u32Limit & 0xfff) == 0xfff
+                                  || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->ss.u32Limit & 0xfff00000)
+                                  || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
+            }
+
+            /* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSReg(). */
+            if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
+            {
+                HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
+                HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
+                HMVMX_CHECK_BREAK(   fUnrestrictedGuest
+                                  || pCtx->ds.Attr.n.u4Type > 11
+                                  || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
+                HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(   (pCtx->ds.u32Limit & 0xfff) == 0xfff
+                                  || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->ds.u32Limit & 0xfff00000)
+                                  || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                                  || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
+            }
+            if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
+            {
+                HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
+                HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
+                HMVMX_CHECK_BREAK(   fUnrestrictedGuest
+                                  || pCtx->es.Attr.n.u4Type > 11
+                                  || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
+                HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(   (pCtx->es.u32Limit & 0xfff) == 0xfff
+                                  || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->es.u32Limit & 0xfff00000)
+                                  || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                                  || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
+            }
+            if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
+            {
+                HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
+                HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
+                HMVMX_CHECK_BREAK(   fUnrestrictedGuest
+                                  || pCtx->fs.Attr.n.u4Type > 11
+                                  || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
+                HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(   (pCtx->fs.u32Limit & 0xfff) == 0xfff
+                                  || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->fs.u32Limit & 0xfff00000)
+                                  || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                                  || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
+            }
+            if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
+            {
+                HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
+                HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
+                HMVMX_CHECK_BREAK(   fUnrestrictedGuest
+                                  || pCtx->gs.Attr.n.u4Type > 11
+                                  || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
+                HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
+                HMVMX_CHECK_BREAK(   (pCtx->gs.u32Limit & 0xfff) == 0xfff
+                                  || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->gs.u32Limit & 0xfff00000)
+                                  || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
+                HMVMX_CHECK_BREAK(   !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
+                                  || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
+            }
+            /* 64-bit capable CPUs. */
+            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(   (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
+                              || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
+                              VMX_IGS_LONGMODE_SS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
+                              VMX_IGS_LONGMODE_DS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
+                              VMX_IGS_LONGMODE_ES_BASE_INVALID);
+        }
+        else
+        {
+            /* V86 mode checks. */
+            uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
+            if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
+            {
+                u32CSAttr = 0xf3;   u32SSAttr = 0xf3;
+                u32DSAttr = 0xf3;   u32ESAttr = 0xf3;
+                u32FSAttr = 0xf3;   u32GSAttr = 0xf3;
+            }
+            else
+            {
+                u32CSAttr = pCtx->cs.Attr.u;   u32SSAttr = pCtx->ss.Attr.u;
+                u32DSAttr = pCtx->ds.Attr.u;   u32ESAttr = pCtx->es.Attr.u;
+                u32FSAttr = pCtx->fs.Attr.u;   u32GSAttr = pCtx->gs.Attr.u;
+            }
+
+            /* CS */
+            HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
+            /* SS */
+            HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
+            /* DS */
+            HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
+            /* ES */
+            HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
+            /* FS */
+            HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
+            /* GS */
+            HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
+            HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
+            HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
+            /* 64-bit capable CPUs. */
+            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(   (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
+                              || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
+            HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
+                              VMX_IGS_LONGMODE_SS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
+                              VMX_IGS_LONGMODE_DS_BASE_INVALID);
+            HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
+                              VMX_IGS_LONGMODE_ES_BASE_INVALID);
+        }
+
+        /*
+         * TR.
+         */
+        HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
+        /* 64-bit capable CPUs. */
+        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
+        if (fLongModeGuest)
+            HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11,           /* 64-bit busy TSS. */
+                              VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
+        else
+            HMVMX_CHECK_BREAK(   pCtx->tr.Attr.n.u4Type == 3          /* 16-bit busy TSS. */
+                              || pCtx->tr.Attr.n.u4Type == 11,        /* 32-bit busy TSS.*/
+                              VMX_IGS_TR_ATTR_TYPE_INVALID);
+        HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
+        HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
+        HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED);   /* Bits 11:8 MBZ. */
+        HMVMX_CHECK_BREAK(   (pCtx->tr.u32Limit & 0xfff) == 0xfff
+                          || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
+        HMVMX_CHECK_BREAK(   !(pCtx->tr.u32Limit & 0xfff00000)
+                          || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
+        HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
+
+        /*
+         * GDTR and IDTR (64-bit capable checks).
+         */
+        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
+
+        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
+
+        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID);      /* Bits 31:16 MBZ. */
+
+        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID);      /* Bits 31:16 MBZ. */
+
+        /*
+         * Guest Non-Register State.
+         */
+        /* Activity State. */
+        uint32_t u32ActivityState;
+        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
+        AssertRC(rc);
+        HMVMX_CHECK_BREAK(   !u32ActivityState
+                          || (u32ActivityState & RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
+                             VMX_IGS_ACTIVITY_STATE_INVALID);
+        HMVMX_CHECK_BREAK(   !(pCtx->ss.Attr.n.u2Dpl)
+                          || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
+
+        if (   u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
+            || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+        {
+            HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
+        }
+
+        /** @todo Activity state and injecting interrupts. Left as a todo since we
+         *        currently don't use activity states but ACTIVE. */
+
+        HMVMX_CHECK_BREAK(   !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
+                          || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
+
+        /* Guest interruptibility-state. */
+        HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
+        HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
+                                       != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
+                          VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
+        HMVMX_CHECK_BREAK(   (u32Eflags & X86_EFL_IF)
+                          || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
+                          VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
+        if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
+        {
+            HMVMX_CHECK_BREAK(   !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+                              && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
+                              VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
+        }
+        else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
+        {
+            HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
+                              VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
+            HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
+                              VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
+        }
+        /** @todo Assumes the processor is not in SMM. */
+        HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
+                          VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
+        HMVMX_CHECK_BREAK(   !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
+                          || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
+                             VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
+        if (   (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+            && VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
+        {
+            HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
+        }
+
+        /* Pending debug exceptions. */
+        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
+        AssertRC(rc);
+        /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
+        HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
+        u32Val = u64Val;    /* For pending debug exceptions checks below. */
+
+        if (   (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+            || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
+            || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
+        {
+            if (   (u32Eflags & X86_EFL_TF)
+                && !(u64DebugCtlMsr & RT_BIT_64(1)))    /* Bit 1 is IA32_DEBUGCTL.BTF. */
+            {
+                /* Bit 14 is PendingDebug.BS. */
+                HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
+            }
+            if (   !(u32Eflags & X86_EFL_TF)
+                || (u64DebugCtlMsr & RT_BIT_64(1)))     /* Bit 1 is IA32_DEBUGCTL.BTF. */
+            {
+                /* Bit 14 is PendingDebug.BS. */
+                HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
+            }
+        }
+
+#ifndef IN_NEM_DARWIN
+        /* VMCS link pointer. */
+        rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
+        AssertRC(rc);
+        if (u64Val != UINT64_C(0xffffffffffffffff))
+        {
+            HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
+            /** @todo Bits beyond the processor's physical-address width MBZ. */
+            /** @todo SMM checks. */
+            Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
+            Assert(pVmcsInfo->pvShadowVmcs);
+            VMXVMCSREVID VmcsRevId;
+            VmcsRevId.u = *(uint32_t *)pVmcsInfo->pvShadowVmcs;
+            HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID),
+                              VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
+            HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
+                              VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
+        }
+
+        /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
+         *        not using nested paging? */
+        if (   VM_IS_VMX_NESTED_PAGING(pVM)
+            && !fLongModeGuest
+            && CPUMIsGuestInPAEModeEx(pCtx))
+        {
+            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
+
+            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
+
+            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
+
+            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
+            AssertRC(rc);
+            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
+        }
+#endif
+
+        /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
+        if (uError == VMX_IGS_ERROR)
+            uError = VMX_IGS_REASON_NOT_FOUND;
+    } while (0);
+
+    VCPU_2_VMXSTATE(pVCpu).u32HMError = uError;
+    VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32GuestIntrState = u32IntrState;
+    return uError;
+
+#undef HMVMX_ERROR_BREAK
+#undef HMVMX_CHECK_BREAK
+}
+
+
+#ifndef HMVMX_USE_FUNCTION_TABLE
+/**
+ * Handles a guest VM-exit from hardware-assisted VMX execution.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+#ifdef DEBUG_ramshankar
+# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
+       do { \
+            if (a_fSave != 0) \
+                vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); \
+            VBOXSTRICTRC rcStrict = a_CallExpr; \
+            if (a_fSave != 0) \
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); \
+            return rcStrict; \
+        } while (0)
+#else
+# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
+#endif
+    uint32_t const uExitReason = pVmxTransient->uExitReason;
+    switch (uExitReason)
+    {
+        case VMX_EXIT_EPT_MISCONFIG:           VMEXIT_CALL_RET(0, vmxHCExitEptMisconfig(pVCpu, pVmxTransient));
+        case VMX_EXIT_EPT_VIOLATION:           VMEXIT_CALL_RET(0, vmxHCExitEptViolation(pVCpu, pVmxTransient));
+        case VMX_EXIT_IO_INSTR:                VMEXIT_CALL_RET(0, vmxHCExitIoInstr(pVCpu, pVmxTransient));
+        case VMX_EXIT_CPUID:                   VMEXIT_CALL_RET(0, vmxHCExitCpuid(pVCpu, pVmxTransient));
+        case VMX_EXIT_RDTSC:                   VMEXIT_CALL_RET(0, vmxHCExitRdtsc(pVCpu, pVmxTransient));
+        case VMX_EXIT_RDTSCP:                  VMEXIT_CALL_RET(0, vmxHCExitRdtscp(pVCpu, pVmxTransient));
+        case VMX_EXIT_APIC_ACCESS:             VMEXIT_CALL_RET(0, vmxHCExitApicAccess(pVCpu, pVmxTransient));
+        case VMX_EXIT_XCPT_OR_NMI:             VMEXIT_CALL_RET(0, vmxHCExitXcptOrNmi(pVCpu, pVmxTransient));
+        case VMX_EXIT_MOV_CRX:                 VMEXIT_CALL_RET(0, vmxHCExitMovCRx(pVCpu, pVmxTransient));
+        case VMX_EXIT_EXT_INT:                 VMEXIT_CALL_RET(0, vmxHCExitExtInt(pVCpu, pVmxTransient));
+        case VMX_EXIT_INT_WINDOW:              VMEXIT_CALL_RET(0, vmxHCExitIntWindow(pVCpu, pVmxTransient));
+        case VMX_EXIT_TPR_BELOW_THRESHOLD:     VMEXIT_CALL_RET(0, vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient));
+        case VMX_EXIT_MWAIT:                   VMEXIT_CALL_RET(0, vmxHCExitMwait(pVCpu, pVmxTransient));
+        case VMX_EXIT_MONITOR:                 VMEXIT_CALL_RET(0, vmxHCExitMonitor(pVCpu, pVmxTransient));
+        case VMX_EXIT_TASK_SWITCH:             VMEXIT_CALL_RET(0, vmxHCExitTaskSwitch(pVCpu, pVmxTransient));
+        case VMX_EXIT_PREEMPT_TIMER:           VMEXIT_CALL_RET(0, vmxHCExitPreemptTimer(pVCpu, pVmxTransient));
+        case VMX_EXIT_RDMSR:                   VMEXIT_CALL_RET(0, vmxHCExitRdmsr(pVCpu, pVmxTransient));
+        case VMX_EXIT_WRMSR:                   VMEXIT_CALL_RET(0, vmxHCExitWrmsr(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMCALL:                  VMEXIT_CALL_RET(0, vmxHCExitVmcall(pVCpu, pVmxTransient));
+        case VMX_EXIT_MOV_DRX:                 VMEXIT_CALL_RET(0, vmxHCExitMovDRx(pVCpu, pVmxTransient));
+        case VMX_EXIT_HLT:                     VMEXIT_CALL_RET(0, vmxHCExitHlt(pVCpu, pVmxTransient));
+        case VMX_EXIT_INVD:                    VMEXIT_CALL_RET(0, vmxHCExitInvd(pVCpu, pVmxTransient));
+        case VMX_EXIT_INVLPG:                  VMEXIT_CALL_RET(0, vmxHCExitInvlpg(pVCpu, pVmxTransient));
+        case VMX_EXIT_MTF:                     VMEXIT_CALL_RET(0, vmxHCExitMtf(pVCpu, pVmxTransient));
+        case VMX_EXIT_PAUSE:                   VMEXIT_CALL_RET(0, vmxHCExitPause(pVCpu, pVmxTransient));
+        case VMX_EXIT_WBINVD:                  VMEXIT_CALL_RET(0, vmxHCExitWbinvd(pVCpu, pVmxTransient));
+        case VMX_EXIT_XSETBV:                  VMEXIT_CALL_RET(0, vmxHCExitXsetbv(pVCpu, pVmxTransient));
+        case VMX_EXIT_INVPCID:                 VMEXIT_CALL_RET(0, vmxHCExitInvpcid(pVCpu, pVmxTransient));
+        case VMX_EXIT_GETSEC:                  VMEXIT_CALL_RET(0, vmxHCExitGetsec(pVCpu, pVmxTransient));
+        case VMX_EXIT_RDPMC:                   VMEXIT_CALL_RET(0, vmxHCExitRdpmc(pVCpu, pVmxTransient));
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+        case VMX_EXIT_VMCLEAR:                 VMEXIT_CALL_RET(0, vmxHCExitVmclear(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMLAUNCH:                VMEXIT_CALL_RET(0, vmxHCExitVmlaunch(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMPTRLD:                 VMEXIT_CALL_RET(0, vmxHCExitVmptrld(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMPTRST:                 VMEXIT_CALL_RET(0, vmxHCExitVmptrst(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMREAD:                  VMEXIT_CALL_RET(0, vmxHCExitVmread(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMRESUME:                VMEXIT_CALL_RET(0, vmxHCExitVmwrite(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMWRITE:                 VMEXIT_CALL_RET(0, vmxHCExitVmresume(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMXOFF:                  VMEXIT_CALL_RET(0, vmxHCExitVmxoff(pVCpu, pVmxTransient));
+        case VMX_EXIT_VMXON:                   VMEXIT_CALL_RET(0, vmxHCExitVmxon(pVCpu, pVmxTransient));
+        case VMX_EXIT_INVVPID:                 VMEXIT_CALL_RET(0, vmxHCExitInvvpid(pVCpu, pVmxTransient));
+#else
+        case VMX_EXIT_VMCLEAR:
+        case VMX_EXIT_VMLAUNCH:
+        case VMX_EXIT_VMPTRLD:
+        case VMX_EXIT_VMPTRST:
+        case VMX_EXIT_VMREAD:
+        case VMX_EXIT_VMRESUME:
+        case VMX_EXIT_VMWRITE:
+        case VMX_EXIT_VMXOFF:
+        case VMX_EXIT_VMXON:
+        case VMX_EXIT_INVVPID:
+            return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
+#endif
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case VMX_EXIT_INVEPT:                  VMEXIT_CALL_RET(0, vmxHCExitInvept(pVCpu, pVmxTransient));
+#else
+        case VMX_EXIT_INVEPT:                  return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
+#endif
+
+        case VMX_EXIT_TRIPLE_FAULT:            return vmxHCExitTripleFault(pVCpu, pVmxTransient);
+        case VMX_EXIT_NMI_WINDOW:              return vmxHCExitNmiWindow(pVCpu, pVmxTransient);
+        case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_INIT_SIGNAL:
+        case VMX_EXIT_SIPI:
+        case VMX_EXIT_IO_SMI:
+        case VMX_EXIT_SMI:
+        case VMX_EXIT_ERR_MSR_LOAD:
+        case VMX_EXIT_ERR_MACHINE_CHECK:
+        case VMX_EXIT_PML_FULL:
+        case VMX_EXIT_VIRTUALIZED_EOI:
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+        case VMX_EXIT_LDTR_TR_ACCESS:
+        case VMX_EXIT_APIC_WRITE:
+        case VMX_EXIT_RDRAND:
+        case VMX_EXIT_RSM:
+        case VMX_EXIT_VMFUNC:
+        case VMX_EXIT_ENCLS:
+        case VMX_EXIT_RDSEED:
+        case VMX_EXIT_XSAVES:
+        case VMX_EXIT_XRSTORS:
+        case VMX_EXIT_UMWAIT:
+        case VMX_EXIT_TPAUSE:
+        case VMX_EXIT_LOADIWKEY:
+        default:
+            return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
+    }
+#undef VMEXIT_CALL_RET
+}
+#endif /* !HMVMX_USE_FUNCTION_TABLE */
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Handles a nested-guest VM-exit from hardware-assisted VMX execution.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ */
+DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    uint32_t const uExitReason = pVmxTransient->uExitReason;
+    switch (uExitReason)
+    {
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+        case VMX_EXIT_EPT_MISCONFIG:            return vmxHCExitEptMisconfigNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_EPT_VIOLATION:            return vmxHCExitEptViolationNested(pVCpu, pVmxTransient);
+# else
+        case VMX_EXIT_EPT_MISCONFIG:            return vmxHCExitEptMisconfig(pVCpu, pVmxTransient);
+        case VMX_EXIT_EPT_VIOLATION:            return vmxHCExitEptViolation(pVCpu, pVmxTransient);
+# endif
+        case VMX_EXIT_XCPT_OR_NMI:              return vmxHCExitXcptOrNmiNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_IO_INSTR:                 return vmxHCExitIoInstrNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_HLT:                      return vmxHCExitHltNested(pVCpu, pVmxTransient);
+
+        /*
+         * We shouldn't direct host physical interrupts to the nested-guest.
+         */
+        case VMX_EXIT_EXT_INT:
+            return vmxHCExitExtInt(pVCpu, pVmxTransient);
+
+        /*
+         * Instructions that cause VM-exits unconditionally or the condition is
+         * always taken solely from the nested hypervisor (meaning if the VM-exit
+         * happens, it's guaranteed to be a nested-guest VM-exit).
+         *
+         *   - Provides VM-exit instruction length ONLY.
+         */
+        case VMX_EXIT_CPUID:              /* Unconditional. */
+        case VMX_EXIT_VMCALL:
+        case VMX_EXIT_GETSEC:
+        case VMX_EXIT_INVD:
+        case VMX_EXIT_XSETBV:
+        case VMX_EXIT_VMLAUNCH:
+        case VMX_EXIT_VMRESUME:
+        case VMX_EXIT_VMXOFF:
+        case VMX_EXIT_ENCLS:              /* Condition specified solely by nested hypervisor. */
+        case VMX_EXIT_VMFUNC:
+            return vmxHCExitInstrNested(pVCpu, pVmxTransient);
+
+        /*
+         * Instructions that cause VM-exits unconditionally or the condition is
+         * always taken solely from the nested hypervisor (meaning if the VM-exit
+         * happens, it's guaranteed to be a nested-guest VM-exit).
+         *
+         *   - Provides VM-exit instruction length.
+         *   - Provides VM-exit information.
+         *   - Optionally provides Exit qualification.
+         *
+         * Since Exit qualification is 0 for all VM-exits where it is not
+         * applicable, reading and passing it to the guest should produce
+         * defined behavior.
+         *
+         * See Intel spec. 27.2.1 "Basic VM-Exit Information".
+         */
+        case VMX_EXIT_INVEPT:             /* Unconditional. */
+        case VMX_EXIT_INVVPID:
+        case VMX_EXIT_VMCLEAR:
+        case VMX_EXIT_VMPTRLD:
+        case VMX_EXIT_VMPTRST:
+        case VMX_EXIT_VMXON:
+        case VMX_EXIT_GDTR_IDTR_ACCESS:   /* Condition specified solely by nested hypervisor. */
+        case VMX_EXIT_LDTR_TR_ACCESS:
+        case VMX_EXIT_RDRAND:
+        case VMX_EXIT_RDSEED:
+        case VMX_EXIT_XSAVES:
+        case VMX_EXIT_XRSTORS:
+        case VMX_EXIT_UMWAIT:
+        case VMX_EXIT_TPAUSE:
+            return vmxHCExitInstrWithInfoNested(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_RDTSC:                    return vmxHCExitRdtscNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_RDTSCP:                   return vmxHCExitRdtscpNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_RDMSR:                    return vmxHCExitRdmsrNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_WRMSR:                    return vmxHCExitWrmsrNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_INVLPG:                   return vmxHCExitInvlpgNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_INVPCID:                  return vmxHCExitInvpcidNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_TASK_SWITCH:              return vmxHCExitTaskSwitchNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_WBINVD:                   return vmxHCExitWbinvdNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_MTF:                      return vmxHCExitMtfNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_APIC_ACCESS:              return vmxHCExitApicAccessNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_APIC_WRITE:               return vmxHCExitApicWriteNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_VIRTUALIZED_EOI:          return vmxHCExitVirtEoiNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_MOV_CRX:                  return vmxHCExitMovCRxNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_INT_WINDOW:               return vmxHCExitIntWindowNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_NMI_WINDOW:               return vmxHCExitNmiWindowNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_TPR_BELOW_THRESHOLD:      return vmxHCExitTprBelowThresholdNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_MWAIT:                    return vmxHCExitMwaitNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_MONITOR:                  return vmxHCExitMonitorNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_PAUSE:                    return vmxHCExitPauseNested(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_PREEMPT_TIMER:
+        {
+            /** @todo NSTVMX: Preempt timer. */
+            return vmxHCExitPreemptTimer(pVCpu, pVmxTransient);
+        }
+
+        case VMX_EXIT_MOV_DRX:                  return vmxHCExitMovDRxNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_RDPMC:                    return vmxHCExitRdpmcNested(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_VMREAD:
+        case VMX_EXIT_VMWRITE:                  return vmxHCExitVmreadVmwriteNested(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_TRIPLE_FAULT:             return vmxHCExitTripleFaultNested(pVCpu, pVmxTransient);
+        case VMX_EXIT_ERR_INVALID_GUEST_STATE:  return vmxHCExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_INIT_SIGNAL:
+        case VMX_EXIT_SIPI:
+        case VMX_EXIT_IO_SMI:
+        case VMX_EXIT_SMI:
+        case VMX_EXIT_ERR_MSR_LOAD:
+        case VMX_EXIT_ERR_MACHINE_CHECK:
+        case VMX_EXIT_PML_FULL:
+        case VMX_EXIT_RSM:
+        default:
+            return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
+    }
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/** @name VM-exit helpers.
+ * @{
+ */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+
+/** Macro for VM-exits called unexpectedly. */
+#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
+    do { \
+        VCPU_2_VMXSTATE((a_pVCpu)).u32HMError = (a_HmError); \
+        return VERR_VMX_UNEXPECTED_EXIT; \
+    } while (0)
+
+#ifdef VBOX_STRICT
+# ifndef IN_NEM_DARWIN
+/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
+# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
+    RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
+
+# define HMVMX_ASSERT_PREEMPT_CPUID() \
+    do { \
+         RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
+         AssertMsg(idAssertCpu == idAssertCpuNow,  ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
+    } while (0)
+
+# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { \
+        AssertPtr((a_pVCpu)); \
+        AssertPtr((a_pVmxTransient)); \
+        Assert(   (a_pVmxTransient)->fVMEntryFailed == false \
+               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE \
+               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MSR_LOAD \
+               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MACHINE_CHECK); \
+        Assert((a_pVmxTransient)->pVmcsInfo); \
+        Assert(ASMIntAreEnabled()); \
+        HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
+        HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
+        Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
+        HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
+        if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
+            HMVMX_ASSERT_PREEMPT_CPUID(); \
+        HMVMX_STOP_EXIT_DISPATCH_PROF(); \
+    } while (0)
+# else
+# define HMVMX_ASSERT_PREEMPT_CPUID_VAR()   do { } while(0)
+# define HMVMX_ASSERT_PREEMPT_CPUID()       do { } while(0)
+# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { \
+        AssertPtr((a_pVCpu)); \
+        AssertPtr((a_pVmxTransient)); \
+        Assert(   (a_pVmxTransient)->fVMEntryFailed == false \
+               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE \
+               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MSR_LOAD \
+               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MACHINE_CHECK); \
+        Assert((a_pVmxTransient)->pVmcsInfo); \
+        Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
+        HMVMX_STOP_EXIT_DISPATCH_PROF(); \
+    } while (0)
+# endif
+
+# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { \
+        HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
+        Assert((a_pVmxTransient)->fIsNestedGuest); \
+    } while (0)
+
+# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { \
+        Log4Func(("\n")); \
+    } while (0)
+#else
+# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { \
+        HMVMX_STOP_EXIT_DISPATCH_PROF(); \
+        NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
+    } while (0)
+
+# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
+    do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)
+
+# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient)      do { } while (0)
+#endif
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/** Macro that does the necessary privilege checks and intercepted VM-exits for
+ *  guests that attempted to execute a VMX instruction. */
+# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrictTmp = vmxHCCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
+        if (rcStrictTmp == VINF_SUCCESS) \
+        { /* likely */ } \
+        else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
+        { \
+            Assert((a_pVCpu)->hm.s.Event.fPending); \
+            Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
+            return VINF_SUCCESS; \
+        } \
+        else \
+        { \
+            int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
+            AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
+        } \
+    } while (0)
+
+/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
+# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
+    do \
+    { \
+        VBOXSTRICTRC rcStrictTmp = vmxHCDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
+                                                           (a_pGCPtrEffAddr)); \
+        if (rcStrictTmp == VINF_SUCCESS) \
+        { /* likely */ } \
+        else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
+        { \
+            uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
+            Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
+            NOREF(uXcptTmp); \
+            return VINF_SUCCESS; \
+        } \
+        else \
+        { \
+            Log4Func(("vmxHCDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
+            return rcStrictTmp; \
+        } \
+    } while (0)
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * Advances the guest RIP by the specified number of bytes.
+ *
+ * @param   pVCpu       The cross context virtual CPU structure.
+ * @param   cbInstr     Number of bytes to advance the RIP by.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+DECLINLINE(void) vmxHCAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
+{
+    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);
+
+    /*
+     * Advance RIP.
+     *
+     * The upper 32 bits are only set when in 64-bit mode, so we have to detect
+     * when the addition causes a "carry" into the upper half and check whether
+     * we're in 64-bit and can go on with it or wether we should zap the top
+     * half. (Note! The 8086, 80186 and 80286 emulation is done exclusively in
+     * IEM, so we don't need to bother with pre-386 16-bit wraparound.)
+     *
+     * See PC wrap around tests in bs3-cpu-weird-1.
+     */
+    uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
+    uint64_t const uRipNext = uRipPrev + cbInstr;
+    if (RT_LIKELY(   !((uRipNext ^ uRipPrev) & RT_BIT_64(32))
+                  || CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
+        pVCpu->cpum.GstCtx.rip = uRipNext;
+    else
+        pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;
+
+    /*
+     * Clear RF and interrupt shadowing.
+     */
+    if (RT_LIKELY(!(pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_RF | X86_EFL_TF))))
+        pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_INHIBIT_SHADOW;
+    else
+    {
+        if ((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_RF | X86_EFL_TF)) == X86_EFL_TF)
+        {
+            /** @todo \#DB - single step. */
+        }
+        pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW);
+    }
+    AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);
+
+    /* Mark both RIP and RFLAGS as updated. */
+    ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+}
+
+
+/**
+ * Advances the guest RIP after reading it from the VMCS.
+ *
+ * @returns VBox status code, no informational status codes.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static int vmxHCAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    vmxHCReadToTransientSlow<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    /** @todo consider template here after checking callers.   */
+    int rc = vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
+    AssertRCReturn(rc, rc);
+
+    vmxHCAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Handle a condition that occurred while delivering an event through the guest or
+ * nested-guest IDT.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS if we should continue handling the VM-exit.
+ * @retval  VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
+ *          to continue execution of the guest which will delivery the \#DF.
+ * @retval  VINF_EM_RESET if we detected a triple-fault condition.
+ * @retval  VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ *          Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
+ *          is due to an EPT violation, PML full or SPP-related event.
+ *
+ * @remarks No-long-jump zone!!!
+ */
+static VBOXSTRICTRC vmxHCCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
+    HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
+    if (   pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
+        || pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
+        || pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
+        HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);
+
+    VBOXSTRICTRC   rcStrict       = VINF_SUCCESS;
+    PCVMXVMCSINFO  pVmcsInfo      = pVmxTransient->pVmcsInfo;
+    uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
+    uint32_t const uExitIntInfo   = pVmxTransient->uExitIntInfo;
+    if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
+    {
+        uint32_t const uIdtVector     = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
+        uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);
+
+        /*
+         * If the event was a software interrupt (generated with INT n) or a software exception
+         * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
+         * can handle the VM-exit and continue guest execution which will re-execute the
+         * instruction rather than re-injecting the exception, as that can cause premature
+         * trips to ring-3 before injection and involve TRPM which currently has no way of
+         * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
+         * the problem).
+         */
+        IEMXCPTRAISE     enmRaise;
+        IEMXCPTRAISEINFO fRaiseInfo;
+        if (   uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
+            || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
+            || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
+        {
+            enmRaise   = IEMXCPTRAISE_REEXEC_INSTR;
+            fRaiseInfo = IEMXCPTRAISEINFO_NONE;
+        }
+        else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
+        {
+            uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
+            uint8_t const  uExitVector     = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
+            Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);
+
+            uint32_t const fIdtVectorFlags  = vmxHCGetIemXcptFlags(uIdtVector, uIdtVectorType);
+            uint32_t const fExitVectorFlags = vmxHCGetIemXcptFlags(uExitVector, uExitVectorType);
+
+            enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
+
+            /* Determine a vectoring #PF condition, see comment in vmxHCExitXcptPF(). */
+            if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
+            {
+                pVmxTransient->fVectoringPF = true;
+                enmRaise = IEMXCPTRAISE_PREV_EVENT;
+            }
+        }
+        else
+        {
+            /*
+             * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
+             * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
+             * It is sufficient to reflect the original event to the guest after handling the VM-exit.
+             */
+            Assert(   uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
+                   || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
+                   || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
+            enmRaise   = IEMXCPTRAISE_PREV_EVENT;
+            fRaiseInfo = IEMXCPTRAISEINFO_NONE;
+        }
+
+        /*
+         * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
+         * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
+         * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
+         * subsequent VM-entry would fail, see @bugref{7445}.
+         *
+         * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
+         */
+        if (   uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
+            && enmRaise == IEMXCPTRAISE_PREV_EVENT
+            && (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+            && CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx))
+            CPUMClearInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
+
+        switch (enmRaise)
+        {
+            case IEMXCPTRAISE_CURRENT_XCPT:
+            {
+                Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
+                Assert(rcStrict == VINF_SUCCESS);
+                break;
+            }
+
+            case IEMXCPTRAISE_PREV_EVENT:
+            {
+                uint32_t u32ErrCode;
+                if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
+                    u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
+                else
+                    u32ErrCode = 0;
+
+                /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see vmxHCExitXcptPF(). */
+                STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflect);
+                vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */, u32ErrCode,
+                                     pVCpu->cpum.GstCtx.cr2);
+
+                Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
+                          VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode));
+                Assert(rcStrict == VINF_SUCCESS);
+                break;
+            }
+
+            case IEMXCPTRAISE_REEXEC_INSTR:
+                Assert(rcStrict == VINF_SUCCESS);
+                break;
+
+            case IEMXCPTRAISE_DOUBLE_FAULT:
+            {
+                /*
+                 * Determine a vectoring double #PF condition. Used later, when PGM evaluates the
+                 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
+                 */
+                if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
+                {
+                    pVmxTransient->fVectoringDoublePF = true;
+                    Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
+                          pVCpu->cpum.GstCtx.cr2));
+                    rcStrict = VINF_SUCCESS;
+                }
+                else
+                {
+                    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectConvertDF);
+                    vmxHCSetPendingXcptDF(pVCpu);
+                    Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
+                              uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
+                    rcStrict = VINF_HM_DOUBLE_FAULT;
+                }
+                break;
+            }
+
+            case IEMXCPTRAISE_TRIPLE_FAULT:
+            {
+                Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
+                          VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
+                rcStrict = VINF_EM_RESET;
+                break;
+            }
+
+            case IEMXCPTRAISE_CPU_HANG:
+            {
+                Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
+                rcStrict = VERR_EM_GUEST_CPU_HANG;
+                break;
+            }
+
+            default:
+            {
+                AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
+                rcStrict = VERR_VMX_IPE_2;
+                break;
+            }
+        }
+    }
+    else if (   (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
+             && !CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx))
+    {
+        if (    VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
+             && VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
+             && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
+        {
+            /*
+             * Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
+             * the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
+             * that virtual NMIs remain blocked until the IRET execution is completed.
+             *
+             * See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
+             */
+            CPUMSetInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
+            Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
+        }
+        else if (   pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
+                 || pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
+                 || pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
+        {
+            /*
+             * Execution of IRET caused an EPT violation, page-modification log-full event or
+             * SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
+             * guest or nested-guest NMI handler). We need to set the block-by-NMI field so
+             * that virtual NMIs remain blocked until the IRET execution is completed.
+             *
+             * See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
+             */
+            if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
+            {
+                CPUMSetInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
+                Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
+            }
+        }
+    }
+
+    Assert(   rcStrict == VINF_SUCCESS  || rcStrict == VINF_HM_DOUBLE_FAULT
+           || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
+    return rcStrict;
+}
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/**
+ * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
+ * guest attempting to execute a VMX instruction.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS if we should continue handling the VM-exit.
+ * @retval  VINF_HM_PENDING_XCPT if an exception was raised.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitReason     The VM-exit reason.
+ *
+ * @todo    NSTVMX: Document other error codes when VM-exit is implemented.
+ * @remarks No-long-jump zone!!!
+ */
+static VBOXSTRICTRC vmxHCCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
+{
+    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
+                              | CPUMCTX_EXTRN_CS  | CPUMCTX_EXTRN_EFER);
+
+    /*
+     * The physical CPU would have already checked the CPU mode/code segment.
+     * We shall just assert here for paranoia.
+     * See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
+     */
+    Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
+    Assert(   !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
+           ||  CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));
+
+    if (uExitReason == VMX_EXIT_VMXON)
+    {
+        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
+
+        /*
+         * We check CR4.VMXE because it is required to be always set while in VMX operation
+         * by physical CPUs and our CR4 read-shadow is only consulted when executing specific
+         * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
+         * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
+         */
+        if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
+        {
+            Log4Func(("CR4.VMXE is not set -> #UD\n"));
+            vmxHCSetPendingXcptUD(pVCpu);
+            return VINF_HM_PENDING_XCPT;
+        }
+    }
+    else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
+    {
+        /*
+         * The guest has not entered VMX operation but attempted to execute a VMX instruction
+         * (other than VMXON), we need to raise a #UD.
+         */
+        Log4Func(("Not in VMX root mode -> #UD\n"));
+        vmxHCSetPendingXcptUD(pVCpu);
+        return VINF_HM_PENDING_XCPT;
+    }
+
+    /* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Decodes the memory operand of an instruction that caused a VM-exit.
+ *
+ * The Exit qualification field provides the displacement field for memory
+ * operand instructions, if any.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @retval  VINF_SUCCESS if the operand was successfully decoded.
+ * @retval  VINF_HM_PENDING_XCPT if an exception was raised while decoding the
+ *          operand.
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   uExitInstrInfo  The VM-exit instruction information field.
+ * @param   enmMemAccess    The memory operand's access type (read or write).
+ * @param   GCPtrDisp       The instruction displacement field, if any. For
+ *                          RIP-relative addressing pass RIP + displacement here.
+ * @param   pGCPtrMem       Where to store the effective destination memory address.
+ *
+ * @remarks Warning! This function ASSUMES the instruction cannot be used in real or
+ *          virtual-8086 mode hence skips those checks while verifying if the
+ *          segment is valid.
+ */
+static VBOXSTRICTRC vmxHCDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
+                                            PRTGCPTR pGCPtrMem)
+{
+    Assert(pGCPtrMem);
+    Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
+    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER
+                              | CPUMCTX_EXTRN_CR0);
+
+    static uint64_t const s_auAddrSizeMasks[]   = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
+    static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
+    AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
+
+    VMXEXITINSTRINFO ExitInstrInfo;
+    ExitInstrInfo.u = uExitInstrInfo;
+    uint8_t const   uAddrSize     =  ExitInstrInfo.All.u3AddrSize;
+    uint8_t const   iSegReg       =  ExitInstrInfo.All.iSegReg;
+    bool const      fIdxRegValid  = !ExitInstrInfo.All.fIdxRegInvalid;
+    uint8_t const   iIdxReg       =  ExitInstrInfo.All.iIdxReg;
+    uint8_t const   uScale        =  ExitInstrInfo.All.u2Scaling;
+    bool const      fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
+    uint8_t const   iBaseReg      =  ExitInstrInfo.All.iBaseReg;
+    bool const      fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
+    bool const      fIsLongMode   =  CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
+
+    /*
+     * Validate instruction information.
+     * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
+     */
+    AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
+                          ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
+    AssertLogRelMsgReturn(iSegReg  < X86_SREG_COUNT,
+                          ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
+    AssertLogRelMsgReturn(fIsMemOperand,
+                          ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
+
+    /*
+     * Compute the complete effective address.
+     *
+     * See AMD instruction spec. 1.4.2 "SIB Byte Format"
+     * See AMD spec. 4.5.2 "Segment Registers".
+     */
+    RTGCPTR GCPtrMem = GCPtrDisp;
+    if (fBaseRegValid)
+        GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
+    if (fIdxRegValid)
+        GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
+
+    RTGCPTR const GCPtrOff = GCPtrMem;
+    if (   !fIsLongMode
+        || iSegReg >= X86_SREG_FS)
+        GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
+    GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
+
+    /*
+     * Validate effective address.
+     * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
+     */
+    uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
+    Assert(cbAccess > 0);
+    if (fIsLongMode)
+    {
+        if (X86_IS_CANONICAL(GCPtrMem))
+        {
+            *pGCPtrMem = GCPtrMem;
+            return VINF_SUCCESS;
+        }
+
+        /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
+         *        "Data Limit Checks in 64-bit Mode". */
+        Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
+        vmxHCSetPendingXcptGP(pVCpu, 0);
+        return VINF_HM_PENDING_XCPT;
+    }
+
+    /*
+     * This is a watered down version of iemMemApplySegment().
+     * Parts that are not applicable for VMX instructions like real-or-v8086 mode
+     * and segment CPL/DPL checks are skipped.
+     */
+    RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
+    RTGCPTR32 const GCPtrLast32  = GCPtrFirst32 + cbAccess - 1;
+    PCCPUMSELREG    pSel         = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
+
+    /* Check if the segment is present and usable. */
+    if (    pSel->Attr.n.u1Present
+        && !pSel->Attr.n.u1Unusable)
+    {
+        Assert(pSel->Attr.n.u1DescType);
+        if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
+        {
+            /* Check permissions for the data segment. */
+            if (   enmMemAccess == VMXMEMACCESS_WRITE
+                && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
+            {
+                Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
+                vmxHCSetPendingXcptGP(pVCpu, iSegReg);
+                return VINF_HM_PENDING_XCPT;
+            }
+
+            /* Check limits if it's a normal data segment. */
+            if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
+            {
+                if (   GCPtrFirst32 > pSel->u32Limit
+                    || GCPtrLast32  > pSel->u32Limit)
+                {
+                    Log4Func(("Data segment limit exceeded. "
+                              "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
+                              GCPtrLast32, pSel->u32Limit));
+                    if (iSegReg == X86_SREG_SS)
+                        vmxHCSetPendingXcptSS(pVCpu, 0);
+                    else
+                        vmxHCSetPendingXcptGP(pVCpu, 0);
+                    return VINF_HM_PENDING_XCPT;
+                }
+            }
+            else
+            {
+               /* Check limits if it's an expand-down data segment.
+                  Note! The upper boundary is defined by the B bit, not the G bit! */
+               if (   GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
+                   || GCPtrLast32  > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
+               {
+                   Log4Func(("Expand-down data segment limit exceeded. "
+                             "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
+                             GCPtrLast32, pSel->u32Limit));
+                   if (iSegReg == X86_SREG_SS)
+                       vmxHCSetPendingXcptSS(pVCpu, 0);
+                   else
+                       vmxHCSetPendingXcptGP(pVCpu, 0);
+                   return VINF_HM_PENDING_XCPT;
+               }
+            }
+        }
+        else
+        {
+            /* Check permissions for the code segment. */
+            if (   enmMemAccess == VMXMEMACCESS_WRITE
+                || (   enmMemAccess == VMXMEMACCESS_READ
+                    && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
+            {
+                Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
+                Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
+                vmxHCSetPendingXcptGP(pVCpu, 0);
+                return VINF_HM_PENDING_XCPT;
+            }
+
+            /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
+            if (   GCPtrFirst32 > pSel->u32Limit
+                || GCPtrLast32  > pSel->u32Limit)
+            {
+                Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
+                          GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
+                if (iSegReg == X86_SREG_SS)
+                    vmxHCSetPendingXcptSS(pVCpu, 0);
+                else
+                    vmxHCSetPendingXcptGP(pVCpu, 0);
+                return VINF_HM_PENDING_XCPT;
+            }
+        }
+    }
+    else
+    {
+        Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
+        vmxHCSetPendingXcptGP(pVCpu, 0);
+        return VINF_HM_PENDING_XCPT;
+    }
+
+    *pGCPtrMem = GCPtrMem;
+    return VINF_SUCCESS;
+}
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/**
+ * VM-exit helper for LMSW.
+ */
+static VBOXSTRICTRC vmxHCExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
+{
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitLmsw);
+    Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit helper for CLTS.
+ */
+static VBOXSTRICTRC vmxHCExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
+{
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitClts);
+    Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit helper for MOV from CRx (CRx read).
+ */
+static VBOXSTRICTRC vmxHCExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
+{
+    Assert(iCrReg < 16);
+    Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
+
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    if (iGReg == X86_GREG_xSP)
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
+    else
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+#ifdef VBOX_WITH_STATISTICS
+    switch (iCrReg)
+    {
+        case 0: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Read); break;
+        case 2: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Read); break;
+        case 3: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Read); break;
+        case 4: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Read); break;
+        case 8: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Read); break;
+    }
+#endif
+    Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit helper for MOV to CRx (CRx write).
+ */
+static VBOXSTRICTRC vmxHCExitMovToCrX(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
+{
+    HMVMX_CPUMCTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
+    AssertMsg(   rcStrict == VINF_SUCCESS
+              || rcStrict == VINF_IEM_RAISED_XCPT
+              || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    switch (iCrReg)
+    {
+        case 0:
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0
+                                                     | HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Write);
+            Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
+            break;
+
+        case 2:
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Write);
+            /* Nothing to do here, CR2 it's not part of the VMCS. */
+            break;
+
+        case 3:
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Write);
+            Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
+            break;
+
+        case 4:
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Write);
+#ifndef IN_NEM_DARWIN
+            Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
+                      pVCpu->cpum.GstCtx.cr4, pVCpu->hmr0.s.fLoadSaveGuestXcr0));
+#else
+            Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr4));
+#endif
+            break;
+
+        case 8:
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
+                             HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Write);
+            break;
+
+        default:
+            AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
+            break;
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit exception handler for \#PF (Page-fault exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC vmxHCExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+
+#ifndef IN_NEM_DARWIN
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (!VM_IS_VMX_NESTED_PAGING(pVM))
+    { /* likely */ }
+    else
+#endif
+    {
+#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF) && !defined(IN_NEM_DARWIN)
+        Assert(pVmxTransient->fIsNestedGuest || pVCpu->hmr0.s.fUsingDebugLoop);
+#endif
+        VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;                  /* In case it's a contributory or vectoring #PF. */
+        if (!pVmxTransient->fVectoringDoublePF)
+        {
+            vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
+                                   pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
+        }
+        else
+        {
+            /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
+            Assert(!pVmxTransient->fIsNestedGuest);
+            vmxHCSetPendingXcptDF(pVCpu);
+            Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
+        }
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
+        return VINF_SUCCESS;
+    }
+
+    Assert(!pVmxTransient->fIsNestedGuest);
+
+    /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
+       of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
+    if (pVmxTransient->fVectoringPF)
+    {
+        Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
+        return VINF_EM_RAW_INJECT_TRPM_EVENT;
+    }
+
+    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    Log4Func(("#PF: cs:rip=%#04x:%08RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pVCpu->cpum.GstCtx.cs.Sel,
+              pVCpu->cpum.GstCtx.rip, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pVCpu->cpum.GstCtx.cr3));
+
+    TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
+    rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, &pVCpu->cpum.GstCtx, (RTGCPTR)pVmxTransient->uExitQual);
+
+    Log4Func(("#PF: rc=%Rrc\n", rc));
+    if (rc == VINF_SUCCESS)
+    {
+        /*
+         * This is typically a shadow page table sync or a MMIO instruction. But we may have
+         * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
+         */
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+        TRPMResetTrap(pVCpu);
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPF);
+        return rc;
+    }
+
+    if (rc == VINF_EM_RAW_GUEST_TRAP)
+    {
+        if (!pVmxTransient->fVectoringDoublePF)
+        {
+            /* It's a guest page fault and needs to be reflected to the guest. */
+            uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
+            TRPMResetTrap(pVCpu);
+            VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;                 /* In case it's a contributory #PF. */
+            vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
+                                   uGstErrorCode, pVmxTransient->uExitQual);
+        }
+        else
+        {
+            /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
+            TRPMResetTrap(pVCpu);
+            VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;     /* Clear pending #PF to replace it with #DF. */
+            vmxHCSetPendingXcptDF(pVCpu);
+            Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
+        }
+
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
+        return VINF_SUCCESS;
+    }
+
+    TRPMResetTrap(pVCpu);
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPFEM);
+    return rc;
+}
+
+
+/**
+ * VM-exit exception handler for \#MF (Math Fault: floating point exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC vmxHCExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF);
+
+    int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CR0>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
+    {
+        /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
+        rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
+
+        /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
+         *        provides VM-exit instruction length. If this causes problem later,
+         *        disassemble the instruction like it's done on AMD-V. */
+        int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
+        AssertRCReturn(rc2, rc2);
+        return rc;
+    }
+
+    vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
+                           pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit exception handler for \#BP (Breakpoint exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC vmxHCExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP);
+
+    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict;
+    if (!pVmxTransient->fIsNestedGuest)
+        rcStrict = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx);
+    else
+        rcStrict = VINF_EM_RAW_GUEST_TRAP;
+
+    if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
+    {
+        vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                               pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_DBG_BREAKPOINT);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit exception handler for \#AC (Alignment-check exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC vmxHCExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /*
+     * Detect #ACs caused by host having enabled split-lock detection.
+     * Emulate such instructions.
+     */
+#define VMX_HC_EXIT_XCPT_AC_INITIAL_REGS    (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS)
+    int rc = vmxHCImportGuestState<VMX_HC_EXIT_XCPT_AC_INITIAL_REGS>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+    /** @todo detect split lock in cpu feature?   */
+    if (   /* 1. If 486-style alignment checks aren't enabled, then this must be a split-lock exception */
+           !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
+           /* 2. #AC cannot happen in rings 0-2 except for split-lock detection. */
+        || CPUMGetGuestCPL(pVCpu) != 3
+           /* 3. When the EFLAGS.AC != 0 this can only be a split-lock case. */
+        || !(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_AC) )
+    {
+        /*
+         * Check for debug/trace events and import state accordingly.
+         */
+        STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestACSplitLock);
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        if (   !DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK)
+#ifndef IN_NEM_DARWIN
+            && !VBOXVMM_VMX_SPLIT_LOCK_ENABLED()
+#endif
+            )
+        {
+            if (pVM->cCpus == 1)
+            {
+#if 0 /** @todo r=bird: This is potentially wrong.  Might have to just do a whole state sync above and mark everything changed to be safe... */
+                rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK,
+                                           VMX_HC_EXIT_XCPT_AC_INITIAL_REGS>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+#else
+                rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
+                                           VMX_HC_EXIT_XCPT_AC_INITIAL_REGS>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+#endif
+                AssertRCReturn(rc, rc);
+            }
+        }
+        else
+        {
+            rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
+                                       VMX_HC_EXIT_XCPT_AC_INITIAL_REGS>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+            AssertRCReturn(rc, rc);
+
+            VBOXVMM_XCPT_DF(pVCpu, &pVCpu->cpum.GstCtx);
+
+            if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK))
+            {
+                VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, DBGFEVENT_VMX_SPLIT_LOCK, DBGFEVENTCTX_HM, 0);
+                if (rcStrict != VINF_SUCCESS)
+                    return rcStrict;
+            }
+        }
+
+        /*
+         * Emulate the instruction.
+         *
+         * We have to ignore the LOCK prefix here as we must not retrigger the
+         * detection on the host.  This isn't all that satisfactory, though...
+         */
+        if (pVM->cCpus == 1)
+        {
+            Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC\n", pVCpu->cpum.GstCtx.cs.Sel,
+                      pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
+
+            /** @todo For SMP configs we should do a rendezvous here. */
+            VBOXSTRICTRC rcStrict = IEMExecOneIgnoreLock(pVCpu);
+            if (rcStrict == VINF_SUCCESS)
+#if 0 /** @todo r=bird: This is potentially wrong.  Might have to just do a whole state sync above and mark everything changed to be safe... */
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
+                                   HM_CHANGED_GUEST_RIP
+                                 | HM_CHANGED_GUEST_RFLAGS
+                                 | HM_CHANGED_GUEST_GPRS_MASK
+                                 | HM_CHANGED_GUEST_CS
+                                 | HM_CHANGED_GUEST_SS);
+#else
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+#endif
+            else if (rcStrict == VINF_IEM_RAISED_XCPT)
+            {
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+                rcStrict = VINF_SUCCESS;
+            }
+            return rcStrict;
+        }
+        Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC -> VINF_EM_EMULATE_SPLIT_LOCK\n",
+                  pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
+        return VINF_EM_EMULATE_SPLIT_LOCK;
+    }
+
+    STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC);
+    Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 cpl=%d -> #AC\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0, CPUMGetGuestCPL(pVCpu) ));
+
+    /* Re-inject it. We'll detect any nesting before getting here. */
+    vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                           pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit exception handler for \#DB (Debug exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC vmxHCExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB);
+
+    /*
+     * Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
+     */
+    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+
+    /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
+    uint64_t const uDR6 = X86_DR6_INIT_VAL
+                        | (pVmxTransient->uExitQual & (  X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3
+                                                       | X86_DR6_BD | X86_DR6_BS));
+    Log6Func(("uDR6=%#RX64 uExitQual=%#RX64\n", uDR6, pVmxTransient->uExitQual));
+
+    int rc;
+    if (!pVmxTransient->fIsNestedGuest)
+    {
+        rc = DBGFTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx, uDR6, VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
+
+        /*
+         * Prevents stepping twice over the same instruction when the guest is stepping using
+         * EFLAGS.TF and the hypervisor debugger is stepping using MTF.
+         * Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t").
+         */
+        if (   rc == VINF_EM_DBG_STEPPED
+            && (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG))
+        {
+            Assert(VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
+            rc = VINF_EM_RAW_GUEST_TRAP;
+        }
+    }
+    else
+        rc = VINF_EM_RAW_GUEST_TRAP;
+    Log6Func(("rc=%Rrc\n", rc));
+    if (rc == VINF_EM_RAW_GUEST_TRAP)
+    {
+        /*
+         * The exception was for the guest.  Update DR6, DR7.GD and
+         * IA32_DEBUGCTL.LBR before forwarding it.
+         * See Intel spec. 27.1 "Architectural State before a VM-Exit"
+         * and @sdmv3{077,622,17.2.3,Debug Status Register (DR6)}.
+         */
+#ifndef IN_NEM_DARWIN
+        VMMRZCallRing3Disable(pVCpu);
+        HM_DISABLE_PREEMPT(pVCpu);
+
+        pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
+        pVCpu->cpum.GstCtx.dr[6] |= uDR6;
+        if (CPUMIsGuestDebugStateActive(pVCpu))
+            ASMSetDR6(pVCpu->cpum.GstCtx.dr[6]);
+
+        HM_RESTORE_PREEMPT();
+        VMMRZCallRing3Enable(pVCpu);
+#else
+        /** @todo */
+#endif
+
+        rc = vmxHCImportGuestState<CPUMCTX_EXTRN_DR7>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+        AssertRCReturn(rc, rc);
+
+        /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
+        pVCpu->cpum.GstCtx.dr[7] &= ~(uint64_t)X86_DR7_GD;
+
+        /* Paranoia. */
+        pVCpu->cpum.GstCtx.dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
+        pVCpu->cpum.GstCtx.dr[7] |= X86_DR7_RA1_MASK;
+
+        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, pVCpu->cpum.GstCtx.dr[7]);
+        AssertRC(rc);
+
+        /*
+         * Raise #DB in the guest.
+         *
+         * It is important to reflect exactly what the VM-exit gave us (preserving the
+         * interruption-type) rather than use vmxHCSetPendingXcptDB() as the #DB could've
+         * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
+         * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
+         *
+         * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
+         * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
+         */
+        vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                             pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+        return VINF_SUCCESS;
+    }
+
+    /*
+     * Not a guest trap, must be a hypervisor related debug event then.
+     * Update DR6 in case someone is interested in it.
+     */
+    AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
+    AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
+    CPUMSetHyperDR6(pVCpu, uDR6);
+
+    return rc;
+}
+
+
+/**
+ * Hacks its way around the lovely mesa driver's backdoor accesses.
+ *
+ * @sa hmR0SvmHandleMesaDrvGp.
+ */
+static int vmxHCHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
+{
+    LogFunc(("cs:rip=%#04x:%08RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
+    RT_NOREF(pCtx);
+
+    /* For now we'll just skip the instruction. */
+    return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
+ * backdoor logging w/o checking what it is running inside.
+ *
+ * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
+ * backdoor port and magic numbers loaded in registers.
+ *
+ * @returns true if it is, false if it isn't.
+ * @sa      hmR0SvmIsMesaDrvGp.
+ */
+DECLINLINE(bool) vmxHCIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
+{
+    /* 0xed:  IN eAX,dx */
+    uint8_t abInstr[1];
+    if (pVmxTransient->cbExitInstr != sizeof(abInstr))
+        return false;
+
+    /* Check that it is #GP(0). */
+    if (pVmxTransient->uExitIntErrorCode != 0)
+        return false;
+
+    /* Check magic and port. */
+    Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX)));
+    /*Log(("vmxHCIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));*/
+    if (pCtx->rax != UINT32_C(0x564d5868))
+        return false;
+    if (pCtx->dx != UINT32_C(0x5658))
+        return false;
+
+    /* Flat ring-3 CS. */
+    AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
+    Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
+    /*Log(("vmxHCIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));*/
+    if (pCtx->cs.Attr.n.u2Dpl != 3)
+        return false;
+    if (pCtx->cs.u64Base != 0)
+        return false;
+
+    /* Check opcode. */
+    AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
+    Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
+    int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
+    /*Log(("vmxHCIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));*/
+    if (RT_FAILURE(rc))
+        return false;
+    if (abInstr[0] != 0xed)
+        return false;
+
+    return true;
+}
+
+
+/**
+ * VM-exit exception handler for \#GP (General-protection exception).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC vmxHCExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP);
+
+    PCPUMCTX            pCtx            = &pVCpu->cpum.GstCtx;
+    PVMXVMCSINFO        pVmcsInfo       = pVmxTransient->pVmcsInfo;
+#ifndef IN_NEM_DARWIN
+    PVMXVMCSINFOSHARED  pVmcsInfoShared = pVmcsInfo->pShared;
+    if (pVmcsInfoShared->RealMode.fRealOnV86Active)
+    { /* likely */ }
+    else
+#endif
+    {
+#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
+# ifndef IN_NEM_DARWIN
+        Assert(pVCpu->hmr0.s.fUsingDebugLoop || VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv || pVmxTransient->fIsNestedGuest);
+# else
+        Assert(/*pVCpu->hmr0.s.fUsingDebugLoop ||*/ VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv || pVmxTransient->fIsNestedGuest);
+# endif
+#endif
+        /*
+         * If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
+         * executing a nested-guest, reflect #GP to the guest or nested-guest.
+         */
+        int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
+        AssertRCReturn(rc, rc);
+        Log4Func(("Gst: cs:rip=%#04x:%08RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
+                  pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
+
+        if (    pVmxTransient->fIsNestedGuest
+            || !VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv
+            || !vmxHCIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
+            vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                                   pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+        else
+            rc = vmxHCHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
+        return rc;
+    }
+
+#ifndef IN_NEM_DARWIN
+    Assert(CPUMIsGuestInRealModeEx(pCtx));
+    Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest);
+    Assert(!pVmxTransient->fIsNestedGuest);
+
+    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        if (!CPUMIsGuestInRealModeEx(pCtx))
+        {
+            /*
+             * The guest is no longer in real-mode, check if we can continue executing the
+             * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
+             */
+            pVmcsInfoShared->RealMode.fRealOnV86Active = false;
+            if (HMCanExecuteVmxGuest(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx))
+            {
+                Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+            }
+            else
+            {
+                Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
+                rcStrict = VINF_EM_RESCHEDULE;
+            }
+        }
+        else
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        rcStrict = VINF_SUCCESS;
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+    }
+    return VBOXSTRICTRC_VAL(rcStrict);
+#endif
+}
+
+
+/**
+ * VM-exit exception handler for \#DE (Divide Error).
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC vmxHCExitXcptDE(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE);
+
+    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
+    if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE)
+    {
+        uint8_t cbInstr = 0;
+        VBOXSTRICTRC rc2 = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
+        if (rc2 == VINF_SUCCESS)
+            rcStrict = VINF_SUCCESS;    /* Restart instruction with modified guest register context. */
+        else if (rc2 == VERR_NOT_FOUND)
+            rcStrict = VERR_NOT_FOUND;  /* Deliver the exception. */
+        else
+            Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
+    }
+    else
+        rcStrict = VINF_SUCCESS;        /* Do nothing. */
+
+    /* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */
+    if (RT_FAILURE(rcStrict))
+    {
+        vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                               pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    Assert(rcStrict == VINF_SUCCESS || rcStrict == VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE);
+    return VBOXSTRICTRC_VAL(rcStrict);
+}
+
+
+/**
+ * VM-exit exception handler wrapper for all other exceptions that are not handled
+ * by a specific handler.
+ *
+ * This simply re-injects the exception back into the VM without any special
+ * processing.
+ *
+ * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
+ */
+static VBOXSTRICTRC vmxHCExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
+# ifndef IN_NEM_DARWIN
+    PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    AssertMsg(pVCpu->hmr0.s.fUsingDebugLoop || pVmcsInfo->pShared->RealMode.fRealOnV86Active || pVmxTransient->fIsNestedGuest,
+              ("uVector=%#x u32XcptBitmap=%#X32\n",
+               VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
+    NOREF(pVmcsInfo);
+# endif
+#endif
+
+    /*
+     * Re-inject the exception into the guest. This cannot be a double-fault condition which
+     * would have been handled while checking exits due to event delivery.
+     */
+    uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
+
+#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
+    int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+    Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%08RX64\n", uVector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+#endif
+
+#ifdef VBOX_WITH_STATISTICS
+    switch (uVector)
+    {
+        case X86_XCPT_DE:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE);     break;
+        case X86_XCPT_DB:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB);     break;
+        case X86_XCPT_BP:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP);     break;
+        case X86_XCPT_OF:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF);     break;
+        case X86_XCPT_BR:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBR);     break;
+        case X86_XCPT_UD:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestUD);     break;
+        case X86_XCPT_NM:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF);     break;
+        case X86_XCPT_DF:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDF);     break;
+        case X86_XCPT_TS:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestTS);     break;
+        case X86_XCPT_NP:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestNP);     break;
+        case X86_XCPT_SS:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestSS);     break;
+        case X86_XCPT_GP:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP);     break;
+        case X86_XCPT_PF:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);     break;
+        case X86_XCPT_MF:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF);     break;
+        case X86_XCPT_AC:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC);     break;
+        case X86_XCPT_XF:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXF);     break;
+        default:
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXcpUnk);
+            break;
+    }
+#endif
+
+    /* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
+    Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
+    NOREF(uVector);
+
+    /* Re-inject the original exception into the guest. */
+    vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
+                           pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit exception handler for all exceptions (except NMIs!).
+ *
+ * @remarks This may be called for both guests and nested-guests. Take care to not
+ *          make assumptions and avoid doing anything that is not relevant when
+ *          executing a nested-guest (e.g., Mesa driver hacks).
+ */
+static VBOXSTRICTRC vmxHCExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
+
+    /*
+     * If this VM-exit occurred while delivering an event through the guest IDT, take
+     * action based on the return code and additional hints (e.g. for page-faults)
+     * that will be updated in the VMX transient structure.
+     */
+    VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
+    if (rcStrict == VINF_SUCCESS)
+    {
+        /*
+         * If an exception caused a VM-exit due to delivery of an event, the original
+         * event may have to be re-injected into the guest. We shall reinject it and
+         * continue guest execution. However, page-fault is a complicated case and
+         * needs additional processing done in vmxHCExitXcptPF().
+         */
+        Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
+        uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
+        if (   !VCPU_2_VMXSTATE(pVCpu).Event.fPending
+            || uVector == X86_XCPT_PF)
+        {
+            switch (uVector)
+            {
+                case X86_XCPT_PF: return vmxHCExitXcptPF(pVCpu, pVmxTransient);
+                case X86_XCPT_GP: return vmxHCExitXcptGP(pVCpu, pVmxTransient);
+                case X86_XCPT_MF: return vmxHCExitXcptMF(pVCpu, pVmxTransient);
+                case X86_XCPT_DB: return vmxHCExitXcptDB(pVCpu, pVmxTransient);
+                case X86_XCPT_BP: return vmxHCExitXcptBP(pVCpu, pVmxTransient);
+                case X86_XCPT_AC: return vmxHCExitXcptAC(pVCpu, pVmxTransient);
+                case X86_XCPT_DE: return vmxHCExitXcptDE(pVCpu, pVmxTransient);
+                default:
+                    return vmxHCExitXcptOthers(pVCpu, pVmxTransient);
+            }
+        }
+        /* else: inject pending event before resuming guest execution. */
+    }
+    else if (rcStrict == VINF_HM_DOUBLE_FAULT)
+    {
+        Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    return rcStrict;
+}
+/** @} */
+
+
+/** @name VM-exit handlers.
+ * @{
+ */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+
+/**
+ * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
+ */
+HMVMX_EXIT_DECL vmxHCExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitExtInt);
+
+#ifndef IN_NEM_DARWIN
+    /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
+    if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
+        return VINF_SUCCESS;
+    return VINF_EM_RAW_INTERRUPT;
+#else
+    return VINF_SUCCESS;
+#endif
+}
+
+
+/**
+ * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
+
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);
+
+    uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
+    uint8_t const  uVector      = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
+    Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
+
+    PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    Assert(   !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
+           && uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
+    NOREF(pVmcsInfo);
+
+    VBOXSTRICTRC rcStrict;
+    switch (uExitIntType)
+    {
+#ifndef IN_NEM_DARWIN /* NMIs should never reach R3. */
+        /*
+         * Host physical NMIs:
+         *     This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
+         *     injected it ourselves and anything we inject is not going to cause a VM-exit directly
+         *     for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
+         *
+         *     See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
+         *     See Intel spec. 27.5.5 "Updating Non-Register State".
+         */
+        case VMX_EXIT_INT_INFO_TYPE_NMI:
+        {
+            rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
+            break;
+        }
+#endif
+
+        /*
+         * Privileged software exceptions (#DB from ICEBP),
+         * Software exceptions (#BP and #OF),
+         * Hardware exceptions:
+         *     Process the required exceptions and resume guest execution if possible.
+         */
+        case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
+            Assert(uVector == X86_XCPT_DB);
+            RT_FALL_THRU();
+        case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
+            Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
+            RT_FALL_THRU();
+        case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
+        {
+            NOREF(uVector);
+            vmxHCReadToTransient<  HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
+                                 | HMVMX_READ_EXIT_INSTR_LEN
+                                 | HMVMX_READ_IDT_VECTORING_INFO
+                                 | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
+            rcStrict = vmxHCExitXcpt(pVCpu, pVmxTransient);
+            break;
+        }
+
+        default:
+        {
+            VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
+            rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
+            AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
+            break;
+        }
+    }
+
+    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);
+
+    /* Evaluate and deliver pending events and resume guest execution. */
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIntWindow);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
+    {
+        AssertMsgFailed(("Unexpected NMI-window exit.\n"));
+        HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
+    }
+
+    Assert(!CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx));
+
+    /*
+     * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
+     * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
+     */
+    uint32_t fIntrState;
+    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
+    AssertRC(rc);
+    Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
+    if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
+    {
+        CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
+
+        fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
+        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
+        AssertRC(rc);
+    }
+
+    /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
+    vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);
+
+    /* Evaluate and deliver pending events and resume guest execution. */
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /*
+     * Get the state we need and update the exit history entry.
+     */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict;
+    PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
+                                                            EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
+                                                            pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
+    if (!pExitRec)
+    {
+        /*
+         * Regular CPUID instruction execution.
+         */
+        rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
+        if (rcStrict == VINF_SUCCESS)
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+        else if (rcStrict == VINF_IEM_RAISED_XCPT)
+        {
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+            rcStrict = VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        /*
+         * Frequent exit or something needing probing.  Get state and call EMHistoryExec.
+         */
+        int rc2 = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
+                                        IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+        AssertRCReturn(rc2, rc2);
+
+        Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
+
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+        Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CR4>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
+        return VINF_EM_RAW_EMULATE_INSTR;
+
+    AssertMsgFailed(("vmxHCExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
+    HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
+}
+
+
+/**
+ * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        /* If we get a spurious VM-exit when TSC offsetting is enabled,
+           we must reset offsetting on VM-entry. See @bugref{6634}. */
+        if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
+            pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        /* If we get a spurious VM-exit when TSC offsetting is enabled,
+           we must reset offsetting on VM-reentry. See @bugref{6634}. */
+        if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
+            pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedRdpmc(pVCpu, pVmxTransient->cbExitInstr);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
+    if (EMAreHypercallInstructionsEnabled(pVCpu))
+    {
+        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+        int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RIP
+                                       | CPUMCTX_EXTRN_RFLAGS
+                                       | CPUMCTX_EXTRN_CR0
+                                       | CPUMCTX_EXTRN_SS
+                                       | CPUMCTX_EXTRN_CS
+                                       | CPUMCTX_EXTRN_EFER>(pVCpu, pVmcsInfo, __FUNCTION__);
+        AssertRCReturn(rc, rc);
+
+        /* Perform the hypercall. */
+        rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
+        if (rcStrict == VINF_SUCCESS)
+        {
+            rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
+            AssertRCReturn(rc, rc);
+        }
+        else
+            Assert(   rcStrict == VINF_GIM_R3_HYPERCALL
+                   || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
+                   || RT_FAILURE(rcStrict));
+
+        /* If the hypercall changes anything other than guest's general-purpose registers,
+           we would need to reload the guest changed bits here before VM-entry. */
+    }
+    else
+        Log4Func(("Hypercalls not enabled\n"));
+
+    /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
+    if (RT_FAILURE(rcStrict))
+    {
+        vmxHCSetPendingXcptUD(pVCpu);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+#ifndef IN_NEM_DARWIN
+    Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging || pVCpu->hmr0.s.fUsingDebugLoop);
+#endif
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);
+
+    if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
+                         VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
+    if (rcStrict == VINF_SUCCESS)
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
+    if (RT_SUCCESS(rcStrict))
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+        if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
+            rcStrict = VINF_SUCCESS;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    return VINF_EM_RESET;
+}
+
+
+/**
+ * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);            /* Advancing the RIP above should've imported eflags. */
+    if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx))    /* Requires eflags. */
+        rc = VINF_SUCCESS;
+    else
+        rc = VINF_EM_HALT;
+
+    if (rc != VINF_SUCCESS)
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHltToR3);
+    return rc;
+}
+
+
+#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * VM-exit handler for instructions that result in a \#UD exception delivered to
+ * the guest.
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    vmxHCSetPendingXcptUD(pVCpu);
+    return VINF_SUCCESS;
+}
+#endif
+
+
+/**
+ * VM-exit handler for expiry of the VMX-preemption timer.
+ */
+HMVMX_EXIT_DECL vmxHCExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
+    pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+Log12(("vmxHCExitPreemptTimer:\n"));
+
+    /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
+    STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitPreemptTimer);
+    return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
+    ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
+                                                                                : HM_CHANGED_RAISED_XCPT_MASK);
+
+#ifndef IN_NEM_DARWIN
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
+    if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
+    {
+        pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
+        hmR0VmxUpdateStartVmFunction(pVCpu);
+    }
+#endif
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /** @todo Enable the new code after finding a reliably guest test-case. */
+#if 1
+    return VERR_EM_INTERPRETER;
+#else
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_INFO
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
+                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
+    AssertRCReturn(rc, rc);
+
+    /* Paranoia. Ensure this has a memory operand. */
+    Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);
+
+    uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
+    Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
+    uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
+                                                         : pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;
+
+    RTGCPTR GCPtrDesc;
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
+                                                  GCPtrDesc, uType);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+#endif
+}
+
+
+/**
+ * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
+ * VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+    AssertRCReturn(rc, rc);
+
+    rc = vmxHCCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
+    if (RT_FAILURE(rc))
+        return rc;
+
+    uint32_t const uInvalidReason = vmxHCCheckGuestState(pVCpu, pVmcsInfo);
+    NOREF(uInvalidReason);
+
+#ifdef VBOX_STRICT
+    uint32_t fIntrState;
+    uint64_t u64Val;
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_INSTR_INFO
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    vmxHCReadEntryXcptErrorCodeVmcs(pVCpu, pVmxTransient);
+
+    Log4(("uInvalidReason                             %u\n",     uInvalidReason));
+    Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO    %#RX32\n", pVmxTransient->uEntryIntInfo));
+    Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE    %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
+    Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH         %#RX32\n", pVmxTransient->cbEntryInstr));
+
+    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);            AssertRC(rc);
+    Log4(("VMX_VMCS32_GUEST_INT_STATE                 %#RX32\n", fIntrState));
+    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Val);                        AssertRC(rc);
+    Log4(("VMX_VMCS_GUEST_CR0                         %#RX64\n", u64Val));
+    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, &u64Val);                    AssertRC(rc);
+    Log4(("VMX_VMCS_CTRL_CR0_MASK                     %#RX64\n", u64Val));
+    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val);             AssertRC(rc);
+    Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW              %#RX64\n", u64Val));
+    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, &u64Val);                    AssertRC(rc);
+    Log4(("VMX_VMCS_CTRL_CR4_MASK                     %#RX64\n", u64Val));
+    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val);             AssertRC(rc);
+    Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW              %#RX64\n", u64Val));
+# ifndef IN_NEM_DARWIN
+    if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
+    {
+        rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, &u64Val);             AssertRC(rc);
+        Log4(("VMX_VMCS64_CTRL_EPTP_FULL                  %#RX64\n", u64Val));
+    }
+
+    hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
+# endif
+#endif
+
+    return VERR_VMX_INVALID_GUEST_STATE;
+}
+
+/**
+ * VM-exit handler for all undefined/unexpected reasons. Should never happen.
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    /*
+     * Cumulative notes of all recognized but unexpected VM-exits.
+     *
+     * 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
+     *    nested-paging is used.
+     *
+     * 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
+     *    emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
+     *    this function (and thereby stop VM execution) for handling such instructions.
+     *
+     *
+     * VMX_EXIT_INIT_SIGNAL:
+     *    INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
+     *    It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
+     *    VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
+     *
+     *    See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
+     *    See Intel spec. 29.3 "VMX Instructions" for "VMXON".
+     *    See Intel spec. "23.8 Restrictions on VMX operation".
+     *
+     * VMX_EXIT_SIPI:
+     *    SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
+     *    activity state is used. We don't make use of it as our guests don't have direct
+     *    access to the host local APIC.
+     *
+     *    See Intel spec. 25.3 "Other Causes of VM-exits".
+     *
+     * VMX_EXIT_IO_SMI:
+     * VMX_EXIT_SMI:
+     *    This can only happen if we support dual-monitor treatment of SMI, which can be
+     *    activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
+     *    monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
+     *    VMX root mode or receive an SMI. If we get here, something funny is going on.
+     *
+     *    See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
+     *    See Intel spec. 25.3 "Other Causes of VM-Exits"
+     *
+     * VMX_EXIT_ERR_MSR_LOAD:
+     *    Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
+     *    and typically indicates a bug in the hypervisor code. We thus cannot not resume
+     *    execution.
+     *
+     *    See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
+     *
+     * VMX_EXIT_ERR_MACHINE_CHECK:
+     *    Machine check exceptions indicates a fatal/unrecoverable hardware condition
+     *    including but not limited to system bus, ECC, parity, cache and TLB errors. A
+     *    #MC exception abort class exception is raised. We thus cannot assume a
+     *    reasonable chance of continuing any sort of execution and we bail.
+     *
+     *    See Intel spec. 15.1 "Machine-check Architecture".
+     *    See Intel spec. 27.1 "Architectural State Before A VM Exit".
+     *
+     * VMX_EXIT_PML_FULL:
+     * VMX_EXIT_VIRTUALIZED_EOI:
+     * VMX_EXIT_APIC_WRITE:
+     *    We do not currently support any of these features and thus they are all unexpected
+     *    VM-exits.
+     *
+     * VMX_EXIT_GDTR_IDTR_ACCESS:
+     * VMX_EXIT_LDTR_TR_ACCESS:
+     * VMX_EXIT_RDRAND:
+     * VMX_EXIT_RSM:
+     * VMX_EXIT_VMFUNC:
+     * VMX_EXIT_ENCLS:
+     * VMX_EXIT_RDSEED:
+     * VMX_EXIT_XSAVES:
+     * VMX_EXIT_XRSTORS:
+     * VMX_EXIT_UMWAIT:
+     * VMX_EXIT_TPAUSE:
+     * VMX_EXIT_LOADIWKEY:
+     *    These VM-exits are -not- caused unconditionally by execution of the corresponding
+     *    instruction. Any VM-exit for these instructions indicate a hardware problem,
+     *    unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
+     *
+     *    See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
+     */
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
+    HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
+}
+
+
+/**
+ * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
+ */
+HMVMX_EXIT_DECL vmxHCExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+
+    /** @todo Optimize this: We currently drag in the whole MSR state
+     * (CPUMCTX_EXTRN_ALL_MSRS) here.  We should optimize this to only get
+     * MSRs required.  That would require changes to IEM and possibly CPUM too.
+     * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
+    PVMXVMCSINFO   pVmcsInfo = pVmxTransient->pVmcsInfo;
+    uint32_t const idMsr     = pVCpu->cpum.GstCtx.ecx;
+    int            rc;
+    switch (idMsr)
+    {
+        default:
+            rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS>(pVCpu, pVmcsInfo,
+                                                                                                            __FUNCTION__);
+            AssertRCReturn(rc, rc);
+            break;
+        case MSR_K8_FS_BASE:
+            rc = vmxHCImportGuestState<  IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS
+                                       | CPUMCTX_EXTRN_FS>(pVCpu, pVmcsInfo, __FUNCTION__);
+            AssertRCReturn(rc, rc);
+            break;
+        case MSR_K8_GS_BASE:
+            rc = vmxHCImportGuestState<  IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS
+                                       | CPUMCTX_EXTRN_GS>(pVCpu, pVmcsInfo, __FUNCTION__);
+            AssertRCReturn(rc, rc);
+            break;
+    }
+
+    Log4Func(("ecx=%#RX32\n", idMsr));
+
+#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN)
+    Assert(!pVmxTransient->fIsNestedGuest);
+    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
+    {
+        if (   hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
+            && idMsr != MSR_K6_EFER)
+        {
+            AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
+            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
+        }
+        if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
+        {
+            Assert(pVmcsInfo->pvMsrBitmap);
+            uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
+            if (fMsrpm & VMXMSRPM_ALLOW_RD)
+            {
+                AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
+                HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
+            }
+        }
+    }
+#endif
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitRdmsr);
+    if (rcStrict == VINF_SUCCESS)
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ || rcStrict == VINF_EM_TRIPLE_FAULT,
+                  ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
+ */
+HMVMX_EXIT_DECL vmxHCExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+
+    /*
+     * The FS and GS base MSRs are not part of the above all-MSRs mask.
+     * Although we don't need to fetch the base as it will be overwritten shortly, while
+     * loading guest-state we would also load the entire segment register including limit
+     * and attributes and thus we need to load them here.
+     */
+    /** @todo Optimize this: We currently drag in the whole MSR state
+     * (CPUMCTX_EXTRN_ALL_MSRS) here.  We should optimize this to only get
+     * MSRs required.  That would require changes to IEM and possibly CPUM too.
+     * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    uint32_t const idMsr   = pVCpu->cpum.GstCtx.ecx;
+    int            rc;
+    switch (idMsr)
+    {
+        default:
+            rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS>(pVCpu, pVmcsInfo,
+                                                                                                            __FUNCTION__);
+            AssertRCReturn(rc, rc);
+            break;
+
+        case MSR_K8_FS_BASE:
+            rc = vmxHCImportGuestState<  IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS
+                                       | CPUMCTX_EXTRN_FS>(pVCpu, pVmcsInfo, __FUNCTION__);
+            AssertRCReturn(rc, rc);
+            break;
+        case MSR_K8_GS_BASE:
+            rc = vmxHCImportGuestState<  IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS
+                                       | CPUMCTX_EXTRN_GS>(pVCpu, pVmcsInfo, __FUNCTION__);
+            AssertRCReturn(rc, rc);
+            break;
+    }
+    Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitWrmsr);
+
+    if (rcStrict == VINF_SUCCESS)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+
+        /* If this is an X2APIC WRMSR access, update the APIC state as well. */
+        if (    idMsr == MSR_IA32_APICBASE
+            || (   idMsr >= MSR_IA32_X2APIC_START
+                && idMsr <= MSR_IA32_X2APIC_END))
+        {
+            /*
+             * We've already saved the APIC related guest-state (TPR) in post-run phase.
+             * When full APIC register virtualization is implemented we'll have to make
+             * sure APIC state is saved from the VMCS before IEM changes it.
+             */
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
+        }
+        else if (idMsr == MSR_IA32_TSC)        /* Windows 7 does this during bootup. See @bugref{6398}. */
+            pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+        else if (idMsr == MSR_K6_EFER)
+        {
+            /*
+             * If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
+             * as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
+             * We care about the other bits as well, SCE and NXE. See @bugref{7368}.
+             */
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
+        }
+
+        /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
+        if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
+        {
+            switch (idMsr)
+            {
+                case MSR_IA32_SYSENTER_CS:  ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR);  break;
+                case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
+                case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
+                case MSR_K8_FS_BASE:        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_FS);               break;
+                case MSR_K8_GS_BASE:        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_GS);               break;
+                case MSR_K6_EFER:           /* Nothing to do, already handled above. */                                    break;
+                default:
+                {
+#ifndef IN_NEM_DARWIN
+                    if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
+                        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
+                    else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
+                        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
+#else
+                    AssertMsgFailed(("TODO\n"));
+#endif
+                    break;
+                }
+            }
+        }
+#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN)
+        else
+        {
+            /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
+            switch (idMsr)
+            {
+                case MSR_IA32_SYSENTER_CS:
+                case MSR_IA32_SYSENTER_EIP:
+                case MSR_IA32_SYSENTER_ESP:
+                case MSR_K8_FS_BASE:
+                case MSR_K8_GS_BASE:
+                {
+                    AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
+                    HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
+                }
+
+                /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
+                default:
+                {
+                    if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
+                    {
+                        /* EFER MSR writes are always intercepted. */
+                        if (idMsr != MSR_K6_EFER)
+                        {
+                            AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
+                                             idMsr));
+                            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
+                        }
+                    }
+
+                    if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
+                    {
+                        Assert(pVmcsInfo->pvMsrBitmap);
+                        uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
+                        if (fMsrpm & VMXMSRPM_ALLOW_WR)
+                        {
+                            AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
+                            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
+                        }
+                    }
+                    break;
+                }
+            }
+        }
+#endif  /* VBOX_STRICT */
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    else
+        AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE || rcStrict == VINF_EM_TRIPLE_FAULT,
+                  ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /** @todo The guest has likely hit a contended spinlock. We might want to
+     *        poke a schedule different guest VCPU. */
+    int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
+    if (RT_SUCCESS(rc))
+        return VINF_EM_RAW_INTERRUPT;
+
+    AssertMsgFailed(("vmxHCExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
+    return rc;
+}
+
+
+/**
+ * VM-exit handler for when the TPR value is lowered below the specified
+ * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
+
+    /*
+     * The TPR shadow would've been synced with the APIC TPR in the post-run phase.
+     * We'll re-evaluate pending interrupts and inject them before the next VM
+     * entry so we can just continue execution here.
+     */
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTprBelowThreshold);
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
+ * VM-exit.
+ *
+ * @retval VINF_SUCCESS when guest execution can continue.
+ * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
+ * @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
+ *         incompatible guest state for VMX execution (real-on-v86 case).
+ */
+HMVMX_EXIT_DECL vmxHCExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+
+    VBOXSTRICTRC rcStrict;
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    uint64_t const uExitQual   = pVmxTransient->uExitQual;
+    uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
+    switch (uAccessType)
+    {
+        /*
+         * MOV to CRx.
+         */
+        case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
+        {
+            /*
+             * When PAE paging is used, the CPU will reload PAE PDPTEs from CR3 when the guest
+             * changes certain bits even in CR0, CR4 (and not just CR3). We are currently fine
+             * since IEM_CPUMCTX_EXTRN_MUST_MASK (used below) includes CR3 which will import
+             * PAE PDPTEs as well.
+             */
+            int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+            AssertRCReturn(rc, rc);
+
+            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
+#ifndef IN_NEM_DARWIN
+            uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
+#endif
+            uint8_t const  iGReg   = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
+            uint8_t const  iCrReg  = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
+
+            /*
+             * MOV to CR3 only cause a VM-exit when one or more of the following are true:
+             *   - When nested paging isn't used.
+             *   - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
+             *   - We are executing in the VM debug loop.
+             */
+#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
+# ifndef IN_NEM_DARWIN
+            Assert(   iCrReg != 3
+                   || !VM_IS_VMX_NESTED_PAGING(pVM)
+                   || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
+                   || pVCpu->hmr0.s.fUsingDebugLoop);
+# else
+            Assert(   iCrReg != 3
+                   || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
+# endif
+#endif
+
+            /* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
+            Assert(   iCrReg != 8
+                   || !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
+
+            rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
+            AssertMsg(   rcStrict == VINF_SUCCESS
+                      || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+#ifndef IN_NEM_DARWIN
+            /*
+             * This is a kludge for handling switches back to real mode when we try to use
+             * V86 mode to run real mode code directly.  Problem is that V86 mode cannot
+             * deal with special selector values, so we have to return to ring-3 and run
+             * there till the selector values are V86 mode compatible.
+             *
+             * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
+             *       latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
+             *       this function.
+             */
+            if (   iCrReg == 0
+                && rcStrict == VINF_SUCCESS
+                && !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
+                && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
+                && (uOldCr0 & X86_CR0_PE)
+                && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
+            {
+                /** @todo Check selectors rather than returning all the time.  */
+                Assert(!pVmxTransient->fIsNestedGuest);
+                Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
+                rcStrict = VINF_EM_RESCHEDULE_REM;
+            }
+#endif
+
+            break;
+        }
+
+        /*
+         * MOV from CRx.
+         */
+        case VMX_EXIT_QUAL_CRX_ACCESS_READ:
+        {
+            uint8_t const iGReg  = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
+            uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);
+
+            /*
+             * MOV from CR3 only cause a VM-exit when one or more of the following are true:
+             *   - When nested paging isn't used.
+             *   - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
+             *   - We are executing in the VM debug loop.
+             */
+#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
+# ifndef IN_NEM_DARWIN
+            Assert(   iCrReg != 3
+                   || !VM_IS_VMX_NESTED_PAGING(pVM)
+                   || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
+                   || pVCpu->hmr0.s.fLeaveDone);
+# else
+            Assert(   iCrReg != 3
+                   || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
+# endif
+#endif
+
+            /* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
+            Assert(   iCrReg != 8
+                   || !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
+
+            rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
+            break;
+        }
+
+        /*
+         * CLTS (Clear Task-Switch Flag in CR0).
+         */
+        case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
+        {
+            rcStrict = vmxHCExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
+            break;
+        }
+
+        /*
+         * LMSW (Load Machine-Status Word into CR0).
+         * LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
+         */
+        case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
+        {
+            RTGCPTR        GCPtrEffDst;
+            uint8_t const  cbInstr     = pVmxTransient->cbExitInstr;
+            uint16_t const uMsw        = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
+            bool const     fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
+            if (fMemOperand)
+            {
+                vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
+                GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
+            }
+            else
+                GCPtrEffDst = NIL_RTGCPTR;
+            rcStrict = vmxHCExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
+            break;
+        }
+
+        default:
+        {
+            AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
+            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
+        }
+    }
+
+    Assert((VCPU_2_VMXSTATE(pVCpu).fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
+                                   == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
+    Assert(rcStrict != VINF_IEM_RAISED_XCPT);
+
+    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
+    NOREF(pVM);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
+
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+#define VMX_HC_EXIT_IO_INSTR_INITIAL_REGS   (IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER)
+    /* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
+    int rc = vmxHCImportGuestState<VMX_HC_EXIT_IO_INSTR_INITIAL_REGS>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
+    uint32_t const uIOPort      = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
+    uint8_t  const uIOSize      = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
+    bool     const fIOWrite     = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
+    bool     const fIOString    = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
+    bool     const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
+    bool     const fDbgStepping = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
+    AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
+
+    /*
+     * Update exit history to see if this exit can be optimized.
+     */
+    VBOXSTRICTRC rcStrict;
+    PCEMEXITREC  pExitRec = NULL;
+    if (   !fGstStepping
+        && !fDbgStepping)
+        pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
+                                                    !fIOString
+                                                    ? !fIOWrite
+                                                    ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
+                                                    : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
+                                                    : !fIOWrite
+                                                    ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
+                                                    : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
+                                                    pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
+    if (!pExitRec)
+    {
+        static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 };                    /* Size of the I/O accesses in bytes. */
+        static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff };   /* AND masks for saving result in AL/AX/EAX. */
+
+        uint32_t const cbValue  = s_aIOSizes[uIOSize];
+        uint32_t const cbInstr  = pVmxTransient->cbExitInstr;
+        bool  fUpdateRipAlready = false; /* ugly hack, should be temporary. */
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        if (fIOString)
+        {
+            /*
+             * INS/OUTS - I/O String instruction.
+             *
+             * Use instruction-information if available, otherwise fall back on
+             * interpreting the instruction.
+             */
+            Log4Func(("cs:rip=%#04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
+            AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
+            bool const fInsOutsInfo = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
+            if (fInsOutsInfo)
+            {
+                vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
+                AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
+                AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
+                IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
+                bool const fRep           = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
+                if (fIOWrite)
+                    rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
+                                                    pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
+                else
+                {
+                    /*
+                     * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
+                     * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
+                     * See Intel Instruction spec. for "INS".
+                     * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
+                     */
+                    rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
+                }
+            }
+            else
+                rcStrict = IEMExecOne(pVCpu);
+
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
+            fUpdateRipAlready = true;
+        }
+        else
+        {
+            /*
+             * IN/OUT - I/O instruction.
+             */
+            Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
+            uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
+            Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
+            if (fIOWrite)
+            {
+                rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
+                STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite);
+#ifndef IN_NEM_DARWIN
+                if (    rcStrict == VINF_IOM_R3_IOPORT_WRITE
+                    && !pCtx->eflags.Bits.u1TF)
+                    rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
+#endif
+            }
+            else
+            {
+                uint32_t u32Result = 0;
+                rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
+                if (IOM_SUCCESS(rcStrict))
+                {
+                    /* Save result of I/O IN instr. in AL/AX/EAX. */
+                    pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
+                }
+#ifndef IN_NEM_DARWIN
+                if (    rcStrict == VINF_IOM_R3_IOPORT_READ
+                    && !pCtx->eflags.Bits.u1TF)
+                    rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
+#endif
+                STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIORead);
+            }
+        }
+
+        if (IOM_SUCCESS(rcStrict))
+        {
+            if (!fUpdateRipAlready)
+            {
+                vmxHCAdvanceGuestRipBy(pVCpu, cbInstr);
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
+            }
+
+            /*
+             * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
+             * while booting Fedora 17 64-bit guest.
+             *
+             * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
+             */
+            if (fIOString)
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
+
+            /*
+             * If any I/O breakpoints are armed, we need to check if one triggered
+             * and take appropriate action.
+             * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
+             */
+#if 1
+            AssertCompile(VMX_HC_EXIT_IO_INSTR_INITIAL_REGS & CPUMCTX_EXTRN_DR7);
+#else
+            AssertCompile(!(VMX_HC_EXIT_IO_INSTR_INITIAL_REGS & CPUMCTX_EXTRN_DR7));
+            rc = vmxHCImportGuestState<CPUMCTX_EXTRN_DR7>(pVCpu, pVmcsInfo);
+            AssertRCReturn(rc, rc);
+#endif
+
+            /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
+             *  execution engines about whether hyper BPs and such are pending. */
+            uint32_t const uDr7 = pCtx->dr[7];
+            if (RT_UNLIKELY(   (   (uDr7 & X86_DR7_ENABLED_MASK)
+                                && X86_DR7_ANY_RW_IO(uDr7)
+                                && (pCtx->cr4 & X86_CR4_DE))
+                            || DBGFBpIsHwIoArmed(pVM)))
+            {
+                STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxIoCheck);
+
+#ifndef IN_NEM_DARWIN
+                /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
+                VMMRZCallRing3Disable(pVCpu);
+                HM_DISABLE_PREEMPT(pVCpu);
+
+                bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
+
+                VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
+                if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
+                {
+                    /* Raise #DB. */
+                    if (fIsGuestDbgActive)
+                        ASMSetDR6(pCtx->dr[6]);
+                    if (pCtx->dr[7] != uDr7)
+                        VCPU_2_VMXSTATE(pVCpu).fCtxChanged |= HM_CHANGED_GUEST_DR7;
+
+                    vmxHCSetPendingXcptDB(pVCpu);
+                }
+                /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
+                   however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
+                else if (   rcStrict2 != VINF_SUCCESS
+                         && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
+                    rcStrict = rcStrict2;
+                AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
+
+                HM_RESTORE_PREEMPT();
+                VMMRZCallRing3Enable(pVCpu);
+#else
+                /** @todo */
+#endif
+            }
+        }
+
+#ifdef VBOX_STRICT
+        if (   rcStrict == VINF_IOM_R3_IOPORT_READ
+            || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
+            Assert(!fIOWrite);
+        else if (   rcStrict == VINF_IOM_R3_IOPORT_WRITE
+                 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
+                 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
+            Assert(fIOWrite);
+        else
+        {
+# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
+           *        statuses, that the VMM device and some others may return. See
+           *        IOM_SUCCESS() for guidance. */
+            AssertMsg(   RT_FAILURE(rcStrict)
+                      || rcStrict == VINF_SUCCESS
+                      || rcStrict == VINF_EM_RAW_EMULATE_INSTR
+                      || rcStrict == VINF_EM_DBG_BREAKPOINT
+                      || rcStrict == VINF_EM_RAW_GUEST_TRAP
+                      || rcStrict == VINF_EM_RAW_TO_R3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+# endif
+        }
+#endif
+        STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
+    }
+    else
+    {
+        /*
+         * Frequent exit or something needing probing.  Get state and call EMHistoryExec.
+         */
+        int rc2 = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
+                                        VMX_HC_EXIT_IO_INSTR_INITIAL_REGS>(pVCpu, pVmcsInfo, __FUNCTION__);
+        AssertRCReturn(rc2, rc2);
+        STAM_COUNTER_INC(!fIOString ? fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIORead
+                         : fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringRead);
+        Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
+              fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));
+
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+        Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /* Check if this task-switch occurred while delivery an event through the guest IDT. */
+    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+    if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
+    {
+        vmxHCReadToTransient<HMVMX_READ_IDT_VECTORING_INFO>(pVCpu, pVmxTransient);
+        if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
+        {
+            uint32_t uErrCode;
+            if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
+            {
+                vmxHCReadToTransient<HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
+                uErrCode = pVmxTransient->uIdtVectoringErrorCode;
+            }
+            else
+                uErrCode = 0;
+
+            RTGCUINTPTR GCPtrFaultAddress;
+            if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
+                GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
+            else
+                GCPtrFaultAddress = 0;
+
+            vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+
+            vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
+                                 pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);
+
+            Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
+                      VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
+            return VINF_EM_RAW_INJECT_TRPM_EVENT;
+        }
+    }
+
+    /* Fall back to the interpreter to emulate the task-switch. */
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
+    return VERR_EM_INTERPRETER;
+}
+
+
+/**
+ * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
+    int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+    AssertRC(rc);
+    return VINF_EM_DBG_STEPPED;
+}
+
+
+/**
+ * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitApicAccess);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_LEN
+                         | HMVMX_READ_EXIT_INTERRUPTION_INFO
+                         | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
+                         | HMVMX_READ_IDT_VECTORING_INFO
+                         | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
+
+    /*
+     * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
+     */
+    VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
+        if (RT_UNLIKELY(VCPU_2_VMXSTATE(pVCpu).Event.fPending))
+        {
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
+            return VINF_EM_RAW_INJECT_TRPM_EVENT;
+        }
+    }
+    else
+    {
+        Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
+        return rcStrict;
+    }
+
+    /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
+    uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
+    switch (uAccessType)
+    {
+#ifndef IN_NEM_DARWIN
+        case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
+        case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
+        {
+            AssertMsg(   !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
+                      || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
+                      ("vmxHCExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
+
+            RTGCPHYS GCPhys = VCPU_2_VMXSTATE(pVCpu).vmx.u64GstMsrApicBase;    /* Always up-to-date, as it is not part of the VMCS. */
+            GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
+            GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
+            Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
+                 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
+
+            rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu,
+                                            uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys);
+            Log4Func(("IOMR0MmioPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+            if (   rcStrict == VINF_SUCCESS
+                || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
+                || rcStrict == VERR_PAGE_NOT_PRESENT)
+            {
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
+                                                         | HM_CHANGED_GUEST_APIC_TPR);
+                rcStrict = VINF_SUCCESS;
+            }
+            break;
+        }
+#else
+        /** @todo */
+#endif
+
+        default:
+        {
+            Log4Func(("uAccessType=%#x\n", uAccessType));
+            rcStrict = VINF_EM_RAW_EMULATE_INSTR;
+            break;
+        }
+    }
+
+    if (rcStrict != VINF_SUCCESS)
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchApicAccessToR3);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+
+    /*
+     * We might also get this VM-exit if the nested-guest isn't intercepting MOV DRx accesses.
+     * In such a case, rather than disabling MOV DRx intercepts and resuming execution, we
+     * must emulate the MOV DRx access.
+     */
+    if (!pVmxTransient->fIsNestedGuest)
+    {
+        /* We should -not- get this VM-exit if the guest's debug registers were active. */
+        if (   pVmxTransient->fWasGuestDebugStateActive
+#ifdef VMX_WITH_MAYBE_ALWAYS_INTERCEPT_MOV_DRX
+            && !pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fAlwaysInterceptMovDRx
+#endif
+           )
+        {
+            AssertMsgFailed(("Unexpected MOV DRx exit\n"));
+            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
+        }
+
+        if (   !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction
+            && !pVmxTransient->fWasHyperDebugStateActive)
+        {
+            Assert(!DBGFIsStepping(pVCpu));
+            Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
+
+            /* Whether we disable intercepting MOV DRx instructions and resume
+               the current one, or emulate it and keep intercepting them is
+               configurable.  Though it usually comes down to whether there are
+               any new DR6 & DR7 bits (RTM) we want to hide from the guest. */
+#ifdef VMX_WITH_MAYBE_ALWAYS_INTERCEPT_MOV_DRX
+            bool const fResumeInstruction = !pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fAlwaysInterceptMovDRx;
+#else
+            bool const fResumeInstruction = true;
+#endif
+            if (fResumeInstruction)
+            {
+                pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
+                int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+                AssertRC(rc);
+            }
+
+#ifndef IN_NEM_DARWIN
+            /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
+            VMMRZCallRing3Disable(pVCpu);
+            HM_DISABLE_PREEMPT(pVCpu);
+
+            /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
+            CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
+            Assert(CPUMIsGuestDebugStateActive(pVCpu));
+
+            HM_RESTORE_PREEMPT();
+            VMMRZCallRing3Enable(pVCpu);
+#else
+            CPUMR3NemActivateGuestDebugState(pVCpu);
+            Assert(CPUMIsGuestDebugStateActive(pVCpu));
+            Assert(!CPUMIsHyperDebugStateActive(pVCpu));
+#endif
+
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxContextSwitch);
+            if (fResumeInstruction)
+            {
+#ifdef VBOX_WITH_STATISTICS
+                vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+                if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
+                    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
+                else
+                    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
+#endif
+                return VINF_SUCCESS;
+            }
+        }
+    }
+
+    /*
+     * Import state.  We must have DR7 loaded here as it's always consulted,
+     * both for reading and writing.  The other debug registers are never
+     * exported as such.
+     */
+    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<  IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
+                                   | CPUMCTX_EXTRN_GPRS_MASK
+                                   | CPUMCTX_EXTRN_DR7>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    uint8_t const iGReg  = VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual);
+    uint8_t const iDrReg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
+    Log4Func(("cs:rip=%#04x:%08RX64 r%d %s dr%d\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, iGReg,
+              VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE ? "->" : "<-", iDrReg));
+
+    VBOXSTRICTRC  rcStrict;
+    if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
+    {
+        /*
+         * Write DRx register.
+         */
+        rcStrict = IEMExecDecodedMovDRxWrite(pVCpu, pVmxTransient->cbExitInstr, iDrReg, iGReg);
+        AssertMsg(   rcStrict == VINF_SUCCESS
+                  || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+        if (rcStrict == VINF_SUCCESS)
+       {
+            /** @todo r=bird: Not sure why we always flag DR7 as modified here, but I've
+             * kept it for now to avoid breaking something non-obvious. */
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
+                                                                | HM_CHANGED_GUEST_DR7);
+            /* Update the DR6 register if guest debug state is active, otherwise we'll
+               trash it when calling CPUMR0DebugStateMaybeSaveGuestAndRestoreHost. */
+            if (iDrReg == 6 && CPUMIsGuestDebugStateActive(pVCpu))
+                ASMSetDR6(pVCpu->cpum.GstCtx.dr[6]);
+            Log4Func(("r%d=%#RX64 => dr%d=%#RX64\n", iGReg, pVCpu->cpum.GstCtx.aGRegs[iGReg].u,
+                      iDrReg, pVCpu->cpum.GstCtx.dr[iDrReg]));
+        }
+        else if (rcStrict == VINF_IEM_RAISED_XCPT)
+        {
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+            rcStrict = VINF_SUCCESS;
+        }
+
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
+    }
+    else
+    {
+        /*
+         * Read DRx register into a general purpose register.
+         */
+        rcStrict = IEMExecDecodedMovDRxRead(pVCpu, pVmxTransient->cbExitInstr, iGReg, iDrReg);
+        AssertMsg(   rcStrict == VINF_SUCCESS
+                  || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+
+        if (rcStrict == VINF_SUCCESS)
+        {
+            if (iGReg == X86_GREG_xSP)
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
+                                                                    | HM_CHANGED_GUEST_RSP);
+            else
+                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+        }
+        else if (rcStrict == VINF_IEM_RAISED_XCPT)
+        {
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+            rcStrict = VINF_SUCCESS;
+        }
+
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+#ifndef IN_NEM_DARWIN
+    Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_INSTR_LEN
+                         | HMVMX_READ_EXIT_INTERRUPTION_INFO
+                         | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
+                         | HMVMX_READ_IDT_VECTORING_INFO
+                         | HMVMX_READ_IDT_VECTORING_ERROR_CODE
+                         | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
+
+    /*
+     * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
+     */
+    VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        /*
+         * In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
+         * instruction emulation to inject the original event. Otherwise, injecting the original event
+         * using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
+         */
+        if (!VCPU_2_VMXSTATE(pVCpu).Event.fPending)
+        { /* likely */ }
+        else
+        {
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+            /** @todo NSTVMX: Think about how this should be handled. */
+            if (pVmxTransient->fIsNestedGuest)
+                return VERR_VMX_IPE_3;
+# endif
+            return VINF_EM_RAW_INJECT_TRPM_EVENT;
+        }
+    }
+    else
+    {
+        Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
+        return rcStrict;
+    }
+
+    /*
+     * Get sufficient state and update the exit history entry.
+     */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
+    PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
+                                                            EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
+                                                            pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
+    if (!pExitRec)
+    {
+        /*
+         * If we succeed, resume guest execution.
+         * If we fail in interpreting the instruction because we couldn't get the guest physical address
+         * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
+         * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
+         * weird case. See @bugref{6043}.
+         */
+        PVMCC    pVM  = pVCpu->CTX_SUFF(pVM);
+/** @todo bird: We can probably just go straight to IOM here and assume that
+ *        it's MMIO, then fall back on PGM if that hunch didn't work out so
+ *        well.  However, we need to address that aliasing workarounds that
+ *        PGMR0Trap0eHandlerNPMisconfig implements.  So, some care is needed.
+ *
+ *        Might also be interesting to see if we can get this done more or
+ *        less locklessly inside IOM.  Need to consider the lookup table
+ *        updating and use a bit more carefully first (or do all updates via
+ *        rendezvous) */
+        rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, &pVCpu->cpum.GstCtx, GCPhys, UINT32_MAX);
+        Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
+        if (   rcStrict == VINF_SUCCESS
+            || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
+            || rcStrict == VERR_PAGE_NOT_PRESENT)
+        {
+            /* Successfully handled MMIO operation. */
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
+                                                     | HM_CHANGED_GUEST_APIC_TPR);
+            rcStrict = VINF_SUCCESS;
+        }
+    }
+    else
+    {
+        /*
+         * Frequent exit or something needing probing. Call EMHistoryExec.
+         */
+        Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
+
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+        Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+    return rcStrict;
+#else
+    AssertFailed();
+    return VERR_VMX_IPE_3; /* Should never happen with Apple HV in R3. */
+#endif
+}
+
+
+/**
+ * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+#ifndef IN_NEM_DARWIN
+    Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_LEN
+                         | HMVMX_READ_EXIT_INTERRUPTION_INFO
+                         | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
+                         | HMVMX_READ_IDT_VECTORING_INFO
+                         | HMVMX_READ_IDT_VECTORING_ERROR_CODE
+                         | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
+
+    /*
+     * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
+     */
+    VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        /*
+         * If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
+         * we shall resolve the nested #PF and re-inject the original event.
+         */
+        if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
+            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflectNPF);
+    }
+    else
+    {
+        Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
+        return rcStrict;
+    }
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    RTGCPHYS const GCPhys    = pVmxTransient->uGuestPhysicalAddr;
+    uint64_t const uExitQual = pVmxTransient->uExitQual;
+    AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));
+
+    RTGCUINT uErrorCode = 0;
+    if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH)
+        uErrorCode |= X86_TRAP_PF_ID;
+    if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
+        uErrorCode |= X86_TRAP_PF_RW;
+    if (uExitQual & (VMX_EXIT_QUAL_EPT_ENTRY_READ | VMX_EXIT_QUAL_EPT_ENTRY_WRITE | VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE))
+        uErrorCode |= X86_TRAP_PF_P;
+
+    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%08RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));
+
+    PVMCC    pVM  = pVCpu->CTX_SUFF(pVM);
+
+    /*
+     * Handle the pagefault trap for the nested shadow table.
+     */
+    TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
+    rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, pCtx, GCPhys);
+    TRPMResetTrap(pVCpu);
+
+    /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
+    if (   rcStrict == VINF_SUCCESS
+        || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
+        || rcStrict == VERR_PAGE_NOT_PRESENT)
+    {
+        /* Successfully synced our nested page tables. */
+        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitReasonNpf);
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
+        return VINF_SUCCESS;
+    }
+    Log4Func(("EPT return to ring-3 rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+
+#else /* IN_NEM_DARWIN */
+    PVM pVM = pVCpu->CTX_SUFF(pVM);
+    uint64_t const uHostTsc = ASMReadTSC(); RT_NOREF(uHostTsc);
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
+    vmxHCImportGuestRip(pVCpu);
+    vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
+
+    /*
+     * Ask PGM for information about the given GCPhys.  We need to check if we're
+     * out of sync first.
+     */
+    NEMHCDARWINHMACPCCSTATE State = { RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE),
+                                      false,
+                                      false };
+    PGMPHYSNEMPAGEINFO      Info;
+    int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pVmxTransient->uGuestPhysicalAddr, State.fWriteAccess, &Info,
+                                       nemR3DarwinHandleMemoryAccessPageCheckerCallback, &State);
+    if (RT_SUCCESS(rc))
+    {
+        if (Info.fNemProt & (  RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
+                             ? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
+        {
+            if (State.fCanResume)
+            {
+                Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting\n",
+                      pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+                      pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
+                      Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
+                      State.fDidSomething ? "" : " no-change"));
+                EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
+                                 pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
+                return VINF_SUCCESS;
+            }
+        }
+
+        Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
+              Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
+              State.fDidSomething ? "" : " no-change"));
+    }
+    else
+        Log4(("MemExit/%u: %04x:%08RX64: %RGp rc=%Rrc%s; emulating\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              pVmxTransient->uGuestPhysicalAddr, rc, State.fDidSomething ? " modified-backing" : ""));
+
+    /*
+     * Emulate the memory access, either access handler or special memory.
+     */
+    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
+                                              RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
+                                            ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
+                                            : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
+                                            pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
+
+    rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    VBOXSTRICTRC rcStrict;
+    if (!pExitRec)
+        rcStrict = IEMExecOne(pVCpu);
+    else
+    {
+        /* Frequent access or probing. */
+        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+        Log4(("MemExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+    }
+
+    ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+
+    Log4Func(("EPT return rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
+    return rcStrict;
+#endif /* IN_NEM_DARWIN */
+}
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+
+/**
+ * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_INFO
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
+                                   | CPUMCTX_EXTRN_SREG_MASK
+                                   | CPUMCTX_EXTRN_HWVIRT
+                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
+       otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
+    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+        if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+            rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
+    }
+    Assert(rcStrict != VINF_IEM_RAISED_XCPT);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_INFO
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
+                                   | CPUMCTX_EXTRN_SREG_MASK
+                                   | CPUMCTX_EXTRN_HWVIRT
+                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_INFO
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
+                                   | CPUMCTX_EXTRN_SREG_MASK
+                                   | CPUMCTX_EXTRN_HWVIRT
+                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /*
+     * Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
+     * thus might not need to import the shadow VMCS state, it's safer just in case
+     * code elsewhere dares look at unsynced VMCS fields.
+     */
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_INFO
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
+                                   | CPUMCTX_EXTRN_SREG_MASK
+                                   | CPUMCTX_EXTRN_HWVIRT
+                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+    if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
+        HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+
+# if 0 //ndef IN_NEM_DARWIN /** @todo this needs serious tuning still, slows down things enormously. */
+        /* Try for exit optimization.  This is on the following instruction
+           because it would be a waste of time to have to reinterpret the
+           already decoded vmwrite instruction. */
+        PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndType(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_VMREAD));
+        if (pExitRec)
+        {
+            /* Frequent access or probing. */
+            rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+            AssertRCReturn(rc, rc);
+
+            rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
+            Log4(("vmread/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
+                  pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
+                  VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+        }
+# endif
+    }
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
+       otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
+    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
+        if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
+            rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
+    }
+    Assert(rcStrict != VINF_IEM_RAISED_XCPT);
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /*
+     * Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
+     * gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
+     * flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
+     */
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_INFO
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
+                                   | CPUMCTX_EXTRN_SREG_MASK
+                                   | CPUMCTX_EXTRN_HWVIRT
+                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+    if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
+        HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_CR4
+                                   | CPUMCTX_EXTRN_HWVIRT
+                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_INFO
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
+                                   | CPUMCTX_EXTRN_SREG_MASK
+                                   | CPUMCTX_EXTRN_HWVIRT
+                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_INFO
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
+                                   | CPUMCTX_EXTRN_SREG_MASK
+                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+/**
+ * VM-exit handler for INVEPT (VMX_EXIT_INVEPT). Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitInvept(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_INFO
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
+                                   | CPUMCTX_EXTRN_SREG_MASK
+                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+    AssertRCReturn(rc, rc);
+
+    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
+
+    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
+
+    VBOXSTRICTRC rcStrict = IEMExecDecodedInvept(pVCpu, &ExitInfo);
+    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
+    else if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+/** @} */
+
+
+#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
+/** @name Nested-guest VM-exit handlers.
+ * @{
+ */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
+
+/**
+ * Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);
+
+    uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
+    uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
+    Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));
+
+    switch (uExitIntType)
+    {
+# ifndef IN_NEM_DARWIN
+        /*
+         * Physical NMIs:
+         *     We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
+         */
+        case VMX_EXIT_INT_INFO_TYPE_NMI:
+            return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
+# endif
+
+        /*
+         * Hardware exceptions,
+         * Software exceptions,
+         * Privileged software exceptions:
+         *     Figure out if the exception must be delivered to the guest or the nested-guest.
+         */
+        case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
+        case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
+        case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
+        {
+            vmxHCReadToTransient<  HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
+                                 | HMVMX_READ_EXIT_INSTR_LEN
+                                 | HMVMX_READ_IDT_VECTORING_INFO
+                                 | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
+
+            PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+            if (CPUMIsGuestVmxXcptInterceptSet(pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo), pVmxTransient->uExitIntErrorCode))
+            {
+                /* Exit qualification is required for debug and page-fault exceptions. */
+                vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+
+                /*
+                 * For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
+                 * software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
+                 * length. However, if delivery of a software interrupt, software exception or privileged
+                 * software exception causes a VM-exit, that too provides the VM-exit instruction length.
+                 */
+                VMXVEXITINFO const      ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
+                VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT(pVmxTransient->uExitIntInfo,
+                                                                               pVmxTransient->uExitIntErrorCode,
+                                                                               pVmxTransient->uIdtVectoringInfo,
+                                                                               pVmxTransient->uIdtVectoringErrorCode);
+#ifdef DEBUG_ramshankar
+                vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
+                Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n",
+                          pVmxTransient->uExitIntInfo, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
+                if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
+                    Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n",
+                              pVmxTransient->uIdtVectoringInfo, pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
+#endif
+                return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
+            }
+
+            /* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in vmxHCExitXcptPF. */
+            Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
+            return vmxHCExitXcpt(pVCpu, pVmxTransient);
+        }
+
+        /*
+         * Software interrupts:
+         *    VM-exits cannot be caused by software interrupts.
+         *
+         * External interrupts:
+         *    This should only happen when "acknowledge external interrupts on VM-exit"
+         *    control is set. However, we never set this when executing a guest or
+         *    nested-guest. For nested-guests it is emulated while injecting interrupts into
+         *    the guest.
+         */
+        case VMX_EXIT_INT_INFO_TYPE_SW_INT:
+        case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
+        default:
+        {
+            VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
+            return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
+        }
+    }
+}
+
+
+/**
+ * Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
+ * Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    return IEMExecVmxVmexitTripleFault(pVCpu);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
+        return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
+    return vmxHCExitIntWindow(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
+        return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
+    return vmxHCExitIntWindow(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
+ * Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_LEN
+                         | HMVMX_READ_IDT_VECTORING_INFO
+                         | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
+
+    VMXVEXITINFO const      ExitInfo      = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
+    VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
+                                                                            pVmxTransient->uIdtVectoringErrorCode);
+    return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return vmxHCExitHlt(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
+    {
+        vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                             | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
+        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return vmxHCExitInvlpg(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return vmxHCExitRdpmc(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
+ * (VMX_EXIT_VMWRITE). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    Assert(   pVmxTransient->uExitReason == VMX_EXIT_VMREAD
+           || pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);
+
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
+
+    uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
+    Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
+    uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
+
+    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
+    if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
+        u64VmcsField &= UINT64_C(0xffffffff);
+
+    if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
+    {
+        vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                             | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+
+    if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
+        return vmxHCExitVmread(pVCpu, pVmxTransient);
+    return vmxHCExitVmwrite(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+
+    return vmxHCExitRdtsc(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+
+    VBOXSTRICTRC rcStrict;
+    uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
+    switch (uAccessType)
+    {
+        case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
+        {
+            uint8_t const iCrReg   = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
+            uint8_t const iGReg    = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
+            Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
+            uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;
+
+            bool fIntercept;
+            switch (iCrReg)
+            {
+                case 0:
+                case 4:
+                    fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
+                    break;
+
+                case 3:
+                    fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
+                    break;
+
+                case 8:
+                    fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
+                    break;
+
+                default:
+                    fIntercept = false;
+                    break;
+            }
+            if (fIntercept)
+            {
+                VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
+                rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+            }
+            else
+            {
+                int const rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+                AssertRCReturn(rc, rc);
+                rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
+            }
+            break;
+        }
+
+        case VMX_EXIT_QUAL_CRX_ACCESS_READ:
+        {
+            /*
+             * CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
+             * CR2 reads do not cause a VM-exit.
+             * CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
+             * CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
+             */
+            uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
+            if (   iCrReg == 3
+                || iCrReg == 8)
+            {
+                static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
+                                                                  0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
+                uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
+                if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept))
+                {
+                    VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
+                    rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+                }
+                else
+                {
+                    uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
+                    rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
+                }
+            }
+            else
+            {
+                AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
+                HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
+            }
+            break;
+        }
+
+        case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
+        {
+            PCVMXVVMCS const pVmcsNstGst  = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
+            uint64_t const   uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
+            uint64_t const   uReadShadow  = pVmcsNstGst->u64Cr0ReadShadow.u;
+            if (   (uGstHostMask & X86_CR0_TS)
+                && (uReadShadow  & X86_CR0_TS))
+            {
+                VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
+                rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+            }
+            else
+                rcStrict = vmxHCExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
+            break;
+        }
+
+        case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:        /* LMSW (Load Machine-Status Word into CR0) */
+        {
+            RTGCPTR        GCPtrEffDst;
+            uint16_t const uNewMsw     = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
+            bool const     fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
+            if (fMemOperand)
+            {
+                vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
+                GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
+            }
+            else
+                GCPtrEffDst = NIL_RTGCPTR;
+
+            if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
+            {
+                VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
+                ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
+                rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+            }
+            else
+                rcStrict = vmxHCExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
+            break;
+        }
+
+        default:
+        {
+            AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
+            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
+        }
+    }
+
+    if (rcStrict == VINF_IEM_RAISED_XCPT)
+    {
+        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
+        rcStrict = VINF_SUCCESS;
+    }
+    return rcStrict;
+}
+
+
+/**
+ * Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
+    {
+        vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                             | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
+        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return vmxHCExitMovDRx(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+
+    uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
+    uint8_t  const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
+    AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);
+
+    static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 };   /* Size of the I/O accesses in bytes. */
+    uint8_t const cbAccess = s_aIOSizes[uIOSize];
+    if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
+    {
+        /*
+         * IN/OUT instruction:
+         *   - Provides VM-exit instruction length.
+         *
+         * INS/OUTS instruction:
+         *   - Provides VM-exit instruction length.
+         *   - Provides Guest-linear address.
+         *   - Optionally provides VM-exit instruction info (depends on CPU feature).
+         */
+        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+
+        /* Make sure we don't use stale/uninitialized VMX-transient info. below. */
+        pVmxTransient->ExitInstrInfo.u  = 0;
+        pVmxTransient->uGuestLinearAddr = 0;
+
+        bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
+        bool const fIOString       = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
+        if (fIOString)
+        {
+            vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
+            if (fVmxInsOutsInfo)
+            {
+                Assert(RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
+                vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
+            }
+        }
+
+        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_AND_LIN_ADDR_FROM_TRANSIENT(pVmxTransient);
+        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return vmxHCExitIoInstr(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
+ */
+HMVMX_EXIT_DECL vmxHCExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    uint32_t fMsrpm;
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
+        fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
+    else
+        fMsrpm = VMXMSRPM_EXIT_RD;
+
+    if (fMsrpm & VMXMSRPM_EXIT_RD)
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return vmxHCExitRdmsr(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
+ */
+HMVMX_EXIT_DECL vmxHCExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    uint32_t fMsrpm;
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
+        fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
+    else
+        fMsrpm = VMXMSRPM_EXIT_WR;
+
+    if (fMsrpm & VMXMSRPM_EXIT_WR)
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return vmxHCExitWrmsr(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return vmxHCExitMwait(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
+    vmxHCReadToTransient<HMVMX_READ_GUEST_PENDING_DBG_XCPTS>(pVCpu, pVmxTransient);
+    VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_DBG_XCPTS_FROM_TRANSIENT(pVmxTransient);
+    return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return vmxHCExitMonitor(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
+     *        PAUSE when executing a nested-guest? If it does not, we would not need
+     *        to check for the intercepts here. Just call VM-exit... */
+
+    /* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
+    if (   CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
+        || CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return vmxHCExitPause(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for when the TPR value is lowered below the
+ * specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
+    {
+        vmxHCReadToTransient<HMVMX_READ_GUEST_PENDING_DBG_XCPTS>(pVCpu, pVmxTransient);
+        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_DBG_XCPTS_FROM_TRANSIENT(pVmxTransient);
+        return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
+    }
+    return vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
+ * VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_LEN
+                         | HMVMX_READ_IDT_VECTORING_INFO
+                         | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
+
+    Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));
+
+    Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
+              VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));
+
+    VMXVEXITINFO const      ExitInfo      = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
+    VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
+                                                                            pVmxTransient->uIdtVectoringErrorCode);
+    return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
+    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+    return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
+    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+    return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return vmxHCExitRdtscp(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
+ */
+HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+    }
+    return vmxHCExitWbinvd(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
+    {
+        Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
+        vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                             | HMVMX_READ_EXIT_INSTR_INFO
+                             | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+    }
+    return vmxHCExitInvpcid(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for invalid-guest state
+ * (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+    /*
+     * Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
+     * So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
+     * Handle it like it's in an invalid guest state of the outer guest.
+     *
+     * When the fast path is implemented, this should be changed to cause the corresponding
+     * nested-guest VM-exit.
+     */
+    return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for instructions that cause VM-exits unconditionally
+ * and only provide the instruction length.
+ *
+ * Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+#ifdef VBOX_STRICT
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    switch (pVmxTransient->uExitReason)
+    {
+        case VMX_EXIT_ENCLS:
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
+            break;
+
+        case VMX_EXIT_VMFUNC:
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC));
+            break;
+    }
+#endif
+
+    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
+    return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for instructions that provide instruction length as
+ * well as more information.
+ *
+ * Unconditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+
+# ifdef VBOX_STRICT
+    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+    switch (pVmxTransient->uExitReason)
+    {
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+        case VMX_EXIT_LDTR_TR_ACCESS:
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
+            break;
+
+        case VMX_EXIT_RDRAND:
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
+            break;
+
+        case VMX_EXIT_RDSEED:
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
+            break;
+
+        case VMX_EXIT_XSAVES:
+        case VMX_EXIT_XRSTORS:
+            /** @todo NSTVMX: Verify XSS-bitmap. */
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
+            break;
+
+        case VMX_EXIT_UMWAIT:
+        case VMX_EXIT_TPAUSE:
+            Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
+            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
+            break;
+
+        case VMX_EXIT_LOADIWKEY:
+            Assert(CPUMIsGuestVmxProcCtls3Set(pCtx, VMX_PROC_CTLS3_LOADIWKEY_EXIT));
+            break;
+    }
+# endif
+
+    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                         | HMVMX_READ_EXIT_INSTR_LEN
+                         | HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
+    VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
+    return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
+}
+
+# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
+
+/**
+ * Nested-guest VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitEptViolationNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_EPT))
+    {
+        vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
+                             | HMVMX_READ_EXIT_INSTR_LEN
+                             | HMVMX_READ_EXIT_INTERRUPTION_INFO
+                             | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
+                             | HMVMX_READ_IDT_VECTORING_INFO
+                             | HMVMX_READ_IDT_VECTORING_ERROR_CODE
+                             | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
+        int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
+        AssertRCReturn(rc, rc);
+
+        /*
+         * If it's our VMEXIT, we're responsible for re-injecting any event which delivery
+         * might have triggered this VMEXIT.  If we forward the problem to the inner VMM,
+         * it's its problem to deal with that issue and we'll clear the recovered event.
+         */
+        VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
+        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
+        { /*likely*/ }
+        else
+        {
+            Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
+            return rcStrict;
+        }
+        uint32_t const fClearEventOnForward = VCPU_2_VMXSTATE(pVCpu).Event.fPending; /* paranoia. should not inject events below.  */
+
+        RTGCPHYS const GCPhysNestedFault = pVmxTransient->uGuestPhysicalAddr;
+        uint64_t const uExitQual         = pVmxTransient->uExitQual;
+
+        RTGCPTR GCPtrNestedFault;
+        bool const fIsLinearAddrValid = RT_BOOL(uExitQual & VMX_EXIT_QUAL_EPT_LINEAR_ADDR_VALID);
+        if (fIsLinearAddrValid)
+        {
+            vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
+            GCPtrNestedFault = pVmxTransient->uGuestLinearAddr;
+        }
+        else
+            GCPtrNestedFault = 0;
+
+        RTGCUINT const uErr = ((uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH) ? X86_TRAP_PF_ID : 0)
+                            | ((uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)       ? X86_TRAP_PF_RW : 0)
+                            | ((uExitQual & (  VMX_EXIT_QUAL_EPT_ENTRY_READ
+                                             | VMX_EXIT_QUAL_EPT_ENTRY_WRITE
+                                             | VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE))  ? X86_TRAP_PF_P  : 0);
+
+        PGMPTWALK Walk;
+        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        rcStrict = PGMR0NestedTrap0eHandlerNestedPaging(pVCpu, PGMMODE_EPT, uErr, pCtx, GCPhysNestedFault,
+                                                        fIsLinearAddrValid, GCPtrNestedFault, &Walk);
+        Log7Func(("PGM (uExitQual=%#RX64, %RGp, %RGv) -> %Rrc (fFailed=%d)\n",
+                  uExitQual, GCPhysNestedFault, GCPtrNestedFault, VBOXSTRICTRC_VAL(rcStrict), Walk.fFailed));
+        if (RT_SUCCESS(rcStrict))
+            return rcStrict;
+
+        if (fClearEventOnForward)
+            VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
+
+        VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
+                                                                                pVmxTransient->uIdtVectoringErrorCode);
+        if (Walk.fFailed & PGM_WALKFAIL_EPT_VIOLATION)
+        {
+            VMXVEXITINFO const ExitInfo
+                = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_AND_GST_ADDRESSES(VMX_EXIT_EPT_VIOLATION,
+                                                                              pVmxTransient->uExitQual,
+                                                                              pVmxTransient->cbExitInstr,
+                                                                              pVmxTransient->uGuestLinearAddr,
+                                                                              pVmxTransient->uGuestPhysicalAddr);
+            return IEMExecVmxVmexitEptViolation(pVCpu, &ExitInfo, &ExitEventInfo);
+        }
+
+        AssertMsgReturn(Walk.fFailed & PGM_WALKFAIL_EPT_MISCONFIG,
+                        ("uErr=%#RX32 uExitQual=%#RX64 GCPhysNestedFault=%#RGp GCPtrNestedFault=%#RGv\n",
+                         (uint32_t)uErr, uExitQual, GCPhysNestedFault, GCPtrNestedFault),
+                        rcStrict);
+        return IEMExecVmxVmexitEptMisconfig(pVCpu, pVmxTransient->uGuestPhysicalAddr, &ExitEventInfo);
+    }
+
+    return vmxHCExitEptViolation(pVCpu, pVmxTransient);
+}
+
+
+/**
+ * Nested-guest VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
+ * Conditional VM-exit.
+ */
+HMVMX_EXIT_DECL vmxHCExitEptMisconfigNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
+{
+    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
+    Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
+
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_EPT))
+    {
+        vmxHCReadToTransient<HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
+        int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
+        AssertRCReturn(rc, rc);
+
+        PGMPTWALK Walk;
+        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        RTGCPHYS const GCPhysNestedFault = pVmxTransient->uGuestPhysicalAddr;
+        VBOXSTRICTRC rcStrict = PGMR0NestedTrap0eHandlerNestedPaging(pVCpu, PGMMODE_EPT, X86_TRAP_PF_RSVD, pCtx,
+                                                                     GCPhysNestedFault, false /* fIsLinearAddrValid */,
+                                                                     0 /* GCPtrNestedFault */, &Walk);
+        if (RT_SUCCESS(rcStrict))
+        {
+            AssertMsgFailed(("Shouldn't happen with the way we have programmed the EPT shadow tables\n"));
+            return rcStrict;
+        }
+
+        AssertMsg(Walk.fFailed & PGM_WALKFAIL_EPT_MISCONFIG, ("GCPhysNestedFault=%#RGp\n", GCPhysNestedFault));
+        vmxHCReadToTransient<  HMVMX_READ_IDT_VECTORING_INFO
+                             | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
+
+        VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
+                                                                                pVmxTransient->uIdtVectoringErrorCode);
+        return IEMExecVmxVmexitEptMisconfig(pVCpu, pVmxTransient->uGuestPhysicalAddr, &ExitEventInfo);
+    }
+
+    return vmxHCExitEptMisconfig(pVCpu, pVmxTransient);
+}
+
+# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
+
+/** @} */
+#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
+
+
+/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
+ *  probes.
+ *
+ * The following few functions and associated structure contains the bloat
+ * necessary for providing detailed debug events and dtrace probes as well as
+ * reliable host side single stepping.  This works on the principle of
+ * "subclassing" the normal execution loop and workers.  We replace the loop
+ * method completely and override selected helpers to add necessary adjustments
+ * to their core operation.
+ *
+ * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
+ * any performance for debug and analysis features.
+ *
+ * @{
+ */
+
+/**
+ * Transient per-VCPU debug state of VMCS and related info. we save/restore in
+ * the debug run loop.
+ */
+typedef struct VMXRUNDBGSTATE
+{
+    /** The RIP we started executing at.  This is for detecting that we stepped.  */
+    uint64_t    uRipStart;
+    /** The CS we started executing with.  */
+    uint16_t    uCsStart;
+
+    /** Whether we've actually modified the 1st execution control field. */
+    bool        fModifiedProcCtls : 1;
+    /** Whether we've actually modified the 2nd execution control field. */
+    bool        fModifiedProcCtls2 : 1;
+    /** Whether we've actually modified the exception bitmap. */
+    bool        fModifiedXcptBitmap : 1;
+
+    /** We desire the modified the CR0 mask to be cleared. */
+    bool        fClearCr0Mask : 1;
+    /** We desire the modified the CR4 mask to be cleared. */
+    bool        fClearCr4Mask : 1;
+    /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
+    uint32_t    fCpe1Extra;
+    /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
+    uint32_t    fCpe1Unwanted;
+    /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
+    uint32_t    fCpe2Extra;
+    /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
+    uint32_t    bmXcptExtra;
+    /** The sequence number of the Dtrace provider settings the state was
+     *  configured against. */
+    uint32_t    uDtraceSettingsSeqNo;
+    /** VM-exits to check (one bit per VM-exit). */
+    uint32_t    bmExitsToCheck[3];
+
+    /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
+    uint32_t    fProcCtlsInitial;
+    /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
+    uint32_t    fProcCtls2Initial;
+    /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
+    uint32_t    bmXcptInitial;
+} VMXRUNDBGSTATE;
+AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
+typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
+
+
+/**
+ * Initializes the VMXRUNDBGSTATE structure.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure of the
+ *                          calling EMT.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   pDbgState       The debug state to initialize.
+ */
+static void vmxHCRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
+{
+    pDbgState->uRipStart            = pVCpu->cpum.GstCtx.rip;
+    pDbgState->uCsStart             = pVCpu->cpum.GstCtx.cs.Sel;
+
+    pDbgState->fModifiedProcCtls    = false;
+    pDbgState->fModifiedProcCtls2   = false;
+    pDbgState->fModifiedXcptBitmap  = false;
+    pDbgState->fClearCr0Mask        = false;
+    pDbgState->fClearCr4Mask        = false;
+    pDbgState->fCpe1Extra           = 0;
+    pDbgState->fCpe1Unwanted        = 0;
+    pDbgState->fCpe2Extra           = 0;
+    pDbgState->bmXcptExtra          = 0;
+    pDbgState->fProcCtlsInitial     = pVmxTransient->pVmcsInfo->u32ProcCtls;
+    pDbgState->fProcCtls2Initial    = pVmxTransient->pVmcsInfo->u32ProcCtls2;
+    pDbgState->bmXcptInitial        = pVmxTransient->pVmcsInfo->u32XcptBitmap;
+}
+
+
+/**
+ * Updates the VMSC fields with changes requested by @a pDbgState.
+ *
+ * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
+ * immediately before executing guest code, i.e. when interrupts are disabled.
+ * We don't check status codes here as we cannot easily assert or return in the
+ * latter case.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   pDbgState       The debug state.
+ */
+static void vmxHCPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
+{
+    /*
+     * Ensure desired flags in VMCS control fields are set.
+     * (Ignoring write failure here, as we're committed and it's just debug extras.)
+     *
+     * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
+     *       there should be no stale data in pCtx at this point.
+     */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+    if (   (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
+        || (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
+    {
+        pVmcsInfo->u32ProcCtls |= pDbgState->fCpe1Extra;
+        pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
+        VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
+        Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
+        pDbgState->fModifiedProcCtls   = true;
+    }
+
+    if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
+    {
+        pVmcsInfo->u32ProcCtls2  |= pDbgState->fCpe2Extra;
+        VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
+        Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
+        pDbgState->fModifiedProcCtls2  = true;
+    }
+
+    if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
+    {
+        pVmcsInfo->u32XcptBitmap |= pDbgState->bmXcptExtra;
+        VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
+        Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
+        pDbgState->fModifiedXcptBitmap = true;
+    }
+
+    if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
+    {
+        pVmcsInfo->u64Cr0Mask = 0;
+        VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, 0);
+        Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
+    }
+
+    if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
+    {
+        pVmcsInfo->u64Cr4Mask = 0;
+        VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, 0);
+        Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
+    }
+
+    NOREF(pVCpu);
+}
+
+
+/**
+ * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
+ * re-entry next time around.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   pDbgState       The debug state.
+ * @param   rcStrict        The return code from executing the guest using single
+ *                          stepping.
+ */
+static VBOXSTRICTRC vmxHCRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
+                                             VBOXSTRICTRC rcStrict)
+{
+    /*
+     * Restore VM-exit control settings as we may not reenter this function the
+     * next time around.
+     */
+    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
+
+    /* We reload the initial value, trigger what we can of recalculations the
+       next time around.  From the looks of things, that's all that's required atm. */
+    if (pDbgState->fModifiedProcCtls)
+    {
+        if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
+            pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
+        int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
+        AssertRC(rc2);
+        pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
+    }
+
+    /* We're currently the only ones messing with this one, so just restore the
+       cached value and reload the field. */
+    if (   pDbgState->fModifiedProcCtls2
+        && pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
+    {
+        int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
+        AssertRC(rc2);
+        pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
+    }
+
+    /* If we've modified the exception bitmap, we restore it and trigger
+       reloading and partial recalculation the next time around. */
+    if (pDbgState->fModifiedXcptBitmap)
+    {
+        int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pDbgState->bmXcptInitial);
+        AssertRC(rc2);
+        pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
+    }
+
+    return rcStrict;
+}
+
+
+/**
+ * Configures VM-exit controls for current DBGF and DTrace settings.
+ *
+ * This updates @a pDbgState and the VMCS execution control fields to reflect
+ * the necessary VM-exits demanded by DBGF and DTrace.
+ *
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure. May update
+ *                          fUpdatedTscOffsettingAndPreemptTimer.
+ * @param   pDbgState       The debug state.
+ */
+static void vmxHCPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
+{
+#ifndef IN_NEM_DARWIN
+    /*
+     * Take down the dtrace serial number so we can spot changes.
+     */
+    pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
+    ASMCompilerBarrier();
+#endif
+
+    /*
+     * We'll rebuild most of the middle block of data members (holding the
+     * current settings) as we go along here, so start by clearing it all.
+     */
+    pDbgState->bmXcptExtra      = 0;
+    pDbgState->fCpe1Extra       = 0;
+    pDbgState->fCpe1Unwanted    = 0;
+    pDbgState->fCpe2Extra       = 0;
+    for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
+        pDbgState->bmExitsToCheck[i] = 0;
+
+    /*
+     * Software interrupts (INT XXh) - no idea how to trigger these...
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (   DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
+        || VBOXVMM_INT_SOFTWARE_ENABLED())
+    {
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
+    }
+
+    /*
+     * INT3 breakpoints - triggered by #BP exceptions.
+     */
+    if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
+        pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
+
+    /*
+     * Exception bitmap and XCPT events+probes.
+     */
+    for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
+        if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
+            pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
+
+    if (VBOXVMM_XCPT_DE_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
+    if (VBOXVMM_XCPT_DB_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
+    if (VBOXVMM_XCPT_BP_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
+    if (VBOXVMM_XCPT_OF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
+    if (VBOXVMM_XCPT_BR_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
+    if (VBOXVMM_XCPT_UD_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
+    if (VBOXVMM_XCPT_NM_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
+    if (VBOXVMM_XCPT_DF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
+    if (VBOXVMM_XCPT_TS_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
+    if (VBOXVMM_XCPT_NP_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
+    if (VBOXVMM_XCPT_SS_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
+    if (VBOXVMM_XCPT_GP_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
+    if (VBOXVMM_XCPT_PF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
+    if (VBOXVMM_XCPT_MF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
+    if (VBOXVMM_XCPT_AC_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
+    if (VBOXVMM_XCPT_XF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
+    if (VBOXVMM_XCPT_VE_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
+    if (VBOXVMM_XCPT_SX_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
+
+    if (pDbgState->bmXcptExtra)
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
+
+    /*
+     * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
+     *
+     * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
+     *       So, when adding/changing/removing please don't forget to update it.
+     *
+     * Some of the macros are picking up local variables to save horizontal space,
+     * (being able to see it in a table is the lesser evil here).
+     */
+#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
+        (    DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
+         ||  RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
+#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
+        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
+        {   AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
+            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
+        } else do { } while (0)
+#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
+        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
+        { \
+            (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
+            AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
+            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
+        } else do { } while (0)
+#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
+        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
+        { \
+            (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
+            AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
+            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
+        } else do { } while (0)
+#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
+        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
+        { \
+            (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
+            AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
+            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
+        } else do { } while (0)
+
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH,         VMX_EXIT_TASK_SWITCH);   /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION,   VMX_EXIT_EPT_VIOLATION); /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG,   VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS,    VMX_EXIT_APIC_ACCESS);   /* feature dependent, nothing to enable here */
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE,     VMX_EXIT_APIC_WRITE);    /* feature dependent, nothing to enable here */
+
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID,              VMX_EXIT_CPUID);         /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID,              VMX_EXIT_CPUID);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC,             VMX_EXIT_GETSEC);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC,             VMX_EXIT_GETSEC);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT,               VMX_EXIT_HLT,      VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT,               VMX_EXIT_HLT);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD,               VMX_EXIT_INVD);          /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD,               VMX_EXIT_INVD);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG,             VMX_EXIT_INVLPG,   VMX_PROC_CTLS_INVLPG_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG,             VMX_EXIT_INVLPG);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC,              VMX_EXIT_RDPMC,    VMX_PROC_CTLS_RDPMC_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC,              VMX_EXIT_RDPMC);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC,              VMX_EXIT_RDTSC,    VMX_PROC_CTLS_RDTSC_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC,              VMX_EXIT_RDTSC);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM,                VMX_EXIT_RSM);           /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM,                VMX_EXIT_RSM);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL,           VMX_EXIT_VMCALL);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL,           VMX_EXIT_VMCALL);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR,        VMX_EXIT_VMCLEAR);       /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR,        VMX_EXIT_VMCLEAR);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH,       VMX_EXIT_VMLAUNCH);      /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH,       VMX_EXIT_VMLAUNCH);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD,        VMX_EXIT_VMPTRLD);       /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD,        VMX_EXIT_VMPTRLD);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST,        VMX_EXIT_VMPTRST);       /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST,        VMX_EXIT_VMPTRST);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD,         VMX_EXIT_VMREAD);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD,         VMX_EXIT_VMREAD);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME,       VMX_EXIT_VMRESUME);      /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME,       VMX_EXIT_VMRESUME);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE,        VMX_EXIT_VMWRITE);       /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE,        VMX_EXIT_VMWRITE);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF,         VMX_EXIT_VMXOFF);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF,         VMX_EXIT_VMXOFF);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON,          VMX_EXIT_VMXON);         /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON,          VMX_EXIT_VMXON);
+
+    if (   IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
+        || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
+    {
+        int rc = vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo,
+                                         CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
+        AssertRC(rc);
+
+#if 0 /** @todo fix me */
+        pDbgState->fClearCr0Mask = true;
+        pDbgState->fClearCr4Mask = true;
+#endif
+        if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
+            pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
+        if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
+            pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
+        pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
+        /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT.  It would
+                 require clearing here and in the loop if we start using it. */
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
+    }
+    else
+    {
+        if (pDbgState->fClearCr0Mask)
+        {
+            pDbgState->fClearCr0Mask = false;
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR0);
+        }
+        if (pDbgState->fClearCr4Mask)
+        {
+            pDbgState->fClearCr4Mask = false;
+            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR4);
+        }
+    }
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ,           VMX_EXIT_MOV_CRX);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE,          VMX_EXIT_MOV_CRX);
+
+    if (   IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
+        || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
+    {
+        /** @todo later, need to fix handler as it assumes this won't usually happen. */
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
+    }
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ,           VMX_EXIT_MOV_DRX);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE,          VMX_EXIT_MOV_DRX);
+
+    SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR,              VMX_EXIT_RDMSR,    VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR,              VMX_EXIT_RDMSR);
+    SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR,              VMX_EXIT_WRMSR,    VMX_PROC_CTLS_USE_MSR_BITMAPS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR,              VMX_EXIT_WRMSR);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT,              VMX_EXIT_MWAIT,    VMX_PROC_CTLS_MWAIT_EXIT);   /* paranoia */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT,              VMX_EXIT_MWAIT);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR,            VMX_EXIT_MONITOR,  VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR,            VMX_EXIT_MONITOR);
+#if 0 /** @todo too slow, fix handler. */
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE,              VMX_EXIT_PAUSE,    VMX_PROC_CTLS_PAUSE_EXIT);
+#endif
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE,              VMX_EXIT_PAUSE);
+
+    if (   IS_EITHER_ENABLED(pVM, INSTR_SGDT)
+        || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
+        || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
+        || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
+    {
+        pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
+    }
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
+
+    if (   IS_EITHER_ENABLED(pVM, INSTR_SLDT)
+        || IS_EITHER_ENABLED(pVM, INSTR_STR)
+        || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
+        || IS_EITHER_ENABLED(pVM, INSTR_LTR))
+    {
+        pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
+        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
+    }
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT,               VMX_EXIT_LDTR_TR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR,                VMX_EXIT_LDTR_TR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT,               VMX_EXIT_LDTR_TR_ACCESS);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR,                VMX_EXIT_LDTR_TR_ACCESS);
+
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT,         VMX_EXIT_INVEPT);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT,         VMX_EXIT_INVEPT);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP,             VMX_EXIT_RDTSCP,   VMX_PROC_CTLS_RDTSC_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP,             VMX_EXIT_RDTSCP);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID,        VMX_EXIT_INVVPID);       /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID,        VMX_EXIT_INVVPID);
+    SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD,             VMX_EXIT_WBINVD,   VMX_PROC_CTLS2_WBINVD_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD,             VMX_EXIT_WBINVD);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV,             VMX_EXIT_XSETBV);        /* unconditional */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV,             VMX_EXIT_XSETBV);
+    SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND,             VMX_EXIT_RDRAND,   VMX_PROC_CTLS2_RDRAND_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND,             VMX_EXIT_RDRAND);
+    SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID,        VMX_EXIT_INVPCID,  VMX_PROC_CTLS_INVLPG_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID,        VMX_EXIT_INVPCID);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC,         VMX_EXIT_VMFUNC);        /* unconditional for the current setup */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC,         VMX_EXIT_VMFUNC);
+    SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED,             VMX_EXIT_RDSEED,   VMX_PROC_CTLS2_RDSEED_EXIT);
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED,             VMX_EXIT_RDSEED);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES,             VMX_EXIT_XSAVES);        /* unconditional (enabled by host, guest cfg) */
+    SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES,              VMX_EXIT_XSAVES);
+    SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS,            VMX_EXIT_XRSTORS);       /* unconditional (enabled by host, guest cfg) */
+    SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS,            VMX_EXIT_XRSTORS);
+
+#undef IS_EITHER_ENABLED
+#undef SET_ONLY_XBM_IF_EITHER_EN
+#undef SET_CPE1_XBM_IF_EITHER_EN
+#undef SET_CPEU_XBM_IF_EITHER_EN
+#undef SET_CPE2_XBM_IF_EITHER_EN
+
+    /*
+     * Sanitize the control stuff.
+     */
+    pDbgState->fCpe2Extra       &= g_HmMsrs.u.vmx.ProcCtls2.n.allowed1;
+    if (pDbgState->fCpe2Extra)
+        pDbgState->fCpe1Extra   |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
+    pDbgState->fCpe1Extra       &= g_HmMsrs.u.vmx.ProcCtls.n.allowed1;
+    pDbgState->fCpe1Unwanted    &= ~g_HmMsrs.u.vmx.ProcCtls.n.allowed0;
+#ifndef IN_NEM_DARWIN
+    if (pVCpu->hmr0.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
+        pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+    }
+#else
+    if (pVCpu->nem.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
+    {
+        pVCpu->nem.s.fDebugWantRdTscExit ^= true;
+        pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
+    }
+#endif
+
+    Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
+          pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
+          pDbgState->fClearCr0Mask ? " clr-cr0" : "",
+          pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
+}
+
+
+/**
+ * Fires off DBGF events and dtrace probes for a VM-exit, when it's
+ * appropriate.
+ *
+ * The caller has checked the VM-exit against the
+ * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
+ * already, so we don't have to do that either.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   uExitReason     The VM-exit reason.
+ *
+ * @remarks The name of this function is displayed by dtrace, so keep it short
+ *          and to the point. No longer than 33 chars long, please.
+ */
+static VBOXSTRICTRC vmxHCHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
+{
+    /*
+     * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
+     * same time check whether any corresponding Dtrace event is enabled (fDtrace).
+     *
+     * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
+     *       does.  Must add/change/remove both places.  Same ordering, please.
+     *
+     *       Added/removed events must also be reflected in the next section
+     *       where we dispatch dtrace events.
+     */
+    bool            fDtrace1   = false;
+    bool            fDtrace2   = false;
+    DBGFEVENTTYPE   enmEvent1  = DBGFEVENT_END;
+    DBGFEVENTTYPE   enmEvent2  = DBGFEVENT_END;
+    uint32_t        uEventArg  = 0;
+#define SET_EXIT(a_EventSubName) \
+        do { \
+            enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_,  a_EventSubName); \
+            fDtrace2  = RT_CONCAT3(VBOXVMM_EXIT_,   a_EventSubName, _ENABLED)(); \
+        } while (0)
+#define SET_BOTH(a_EventSubName) \
+        do { \
+            enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
+            enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_,  a_EventSubName); \
+            fDtrace1  = RT_CONCAT3(VBOXVMM_INSTR_,  a_EventSubName, _ENABLED)(); \
+            fDtrace2  = RT_CONCAT3(VBOXVMM_EXIT_,   a_EventSubName, _ENABLED)(); \
+        } while (0)
+    switch (uExitReason)
+    {
+        case VMX_EXIT_MTF:
+            return vmxHCExitMtf(pVCpu, pVmxTransient);
+
+        case VMX_EXIT_XCPT_OR_NMI:
+        {
+            uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
+            switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
+            {
+                case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
+                case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
+                case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
+                    if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
+                    {
+                        if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
+                        {
+                            vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE>(pVCpu, pVmxTransient);
+                            uEventArg = pVmxTransient->uExitIntErrorCode;
+                        }
+                        enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
+                        switch (enmEvent1)
+                        {
+                            case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
+                            case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
+                            case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
+                            case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
+                            case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
+                            case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
+                            case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
+                            case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
+                            case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
+                            case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
+                            case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
+                            case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
+                            case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
+                            case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
+                            case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
+                            case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
+                            case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
+                            case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
+                            default:                                                      break;
+                        }
+                    }
+                    else
+                        AssertFailed();
+                    break;
+
+                case VMX_EXIT_INT_INFO_TYPE_SW_INT:
+                    uEventArg = idxVector;
+                    enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
+                    fDtrace1  = VBOXVMM_INT_SOFTWARE_ENABLED();
+                    break;
+            }
+            break;
+        }
+
+        case VMX_EXIT_TRIPLE_FAULT:
+            enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
+            //fDtrace1  = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
+            break;
+        case VMX_EXIT_TASK_SWITCH:      SET_EXIT(TASK_SWITCH); break;
+        case VMX_EXIT_EPT_VIOLATION:    SET_EXIT(VMX_EPT_VIOLATION); break;
+        case VMX_EXIT_EPT_MISCONFIG:    SET_EXIT(VMX_EPT_MISCONFIG); break;
+        case VMX_EXIT_APIC_ACCESS:      SET_EXIT(VMX_VAPIC_ACCESS); break;
+        case VMX_EXIT_APIC_WRITE:       SET_EXIT(VMX_VAPIC_WRITE); break;
+
+        /* Instruction specific VM-exits: */
+        case VMX_EXIT_CPUID:            SET_BOTH(CPUID); break;
+        case VMX_EXIT_GETSEC:           SET_BOTH(GETSEC); break;
+        case VMX_EXIT_HLT:              SET_BOTH(HALT); break;
+        case VMX_EXIT_INVD:             SET_BOTH(INVD); break;
+        case VMX_EXIT_INVLPG:           SET_BOTH(INVLPG); break;
+        case VMX_EXIT_RDPMC:            SET_BOTH(RDPMC); break;
+        case VMX_EXIT_RDTSC:            SET_BOTH(RDTSC); break;
+        case VMX_EXIT_RSM:              SET_BOTH(RSM); break;
+        case VMX_EXIT_VMCALL:           SET_BOTH(VMM_CALL); break;
+        case VMX_EXIT_VMCLEAR:          SET_BOTH(VMX_VMCLEAR); break;
+        case VMX_EXIT_VMLAUNCH:         SET_BOTH(VMX_VMLAUNCH); break;
+        case VMX_EXIT_VMPTRLD:          SET_BOTH(VMX_VMPTRLD); break;
+        case VMX_EXIT_VMPTRST:          SET_BOTH(VMX_VMPTRST); break;
+        case VMX_EXIT_VMREAD:           SET_BOTH(VMX_VMREAD); break;
+        case VMX_EXIT_VMRESUME:         SET_BOTH(VMX_VMRESUME); break;
+        case VMX_EXIT_VMWRITE:          SET_BOTH(VMX_VMWRITE); break;
+        case VMX_EXIT_VMXOFF:           SET_BOTH(VMX_VMXOFF); break;
+        case VMX_EXIT_VMXON:            SET_BOTH(VMX_VMXON); break;
+        case VMX_EXIT_MOV_CRX:
+            vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+            if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
+                SET_BOTH(CRX_READ);
+            else
+                SET_BOTH(CRX_WRITE);
+            uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
+            break;
+        case VMX_EXIT_MOV_DRX:
+            vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+            if (   VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
+                == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
+                SET_BOTH(DRX_READ);
+            else
+                SET_BOTH(DRX_WRITE);
+            uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
+            break;
+        case VMX_EXIT_RDMSR:            SET_BOTH(RDMSR); break;
+        case VMX_EXIT_WRMSR:            SET_BOTH(WRMSR); break;
+        case VMX_EXIT_MWAIT:            SET_BOTH(MWAIT); break;
+        case VMX_EXIT_MONITOR:          SET_BOTH(MONITOR); break;
+        case VMX_EXIT_PAUSE:            SET_BOTH(PAUSE); break;
+        case VMX_EXIT_GDTR_IDTR_ACCESS:
+            vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
+            switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
+            {
+                case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
+                case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
+                case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
+                case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
+            }
+            break;
+
+        case VMX_EXIT_LDTR_TR_ACCESS:
+            vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
+            switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
+            {
+                case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
+                case VMX_YYTR_INSINFO_II_STR:  SET_BOTH(STR); break;
+                case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
+                case VMX_YYTR_INSINFO_II_LTR:  SET_BOTH(LTR); break;
+            }
+            break;
+
+        case VMX_EXIT_INVEPT:           SET_BOTH(VMX_INVEPT); break;
+        case VMX_EXIT_RDTSCP:           SET_BOTH(RDTSCP); break;
+        case VMX_EXIT_INVVPID:          SET_BOTH(VMX_INVVPID); break;
+        case VMX_EXIT_WBINVD:           SET_BOTH(WBINVD); break;
+        case VMX_EXIT_XSETBV:           SET_BOTH(XSETBV); break;
+        case VMX_EXIT_RDRAND:           SET_BOTH(RDRAND); break;
+        case VMX_EXIT_INVPCID:          SET_BOTH(VMX_INVPCID); break;
+        case VMX_EXIT_VMFUNC:           SET_BOTH(VMX_VMFUNC); break;
+        case VMX_EXIT_RDSEED:           SET_BOTH(RDSEED); break;
+        case VMX_EXIT_XSAVES:           SET_BOTH(XSAVES); break;
+        case VMX_EXIT_XRSTORS:          SET_BOTH(XRSTORS); break;
+
+        /* Events that aren't relevant at this point. */
+        case VMX_EXIT_EXT_INT:
+        case VMX_EXIT_INT_WINDOW:
+        case VMX_EXIT_NMI_WINDOW:
+        case VMX_EXIT_TPR_BELOW_THRESHOLD:
+        case VMX_EXIT_PREEMPT_TIMER:
+        case VMX_EXIT_IO_INSTR:
+            break;
+
+        /* Errors and unexpected events. */
+        case VMX_EXIT_INIT_SIGNAL:
+        case VMX_EXIT_SIPI:
+        case VMX_EXIT_IO_SMI:
+        case VMX_EXIT_SMI:
+        case VMX_EXIT_ERR_INVALID_GUEST_STATE:
+        case VMX_EXIT_ERR_MSR_LOAD:
+        case VMX_EXIT_ERR_MACHINE_CHECK:
+        case VMX_EXIT_PML_FULL:
+        case VMX_EXIT_VIRTUALIZED_EOI:
+            break;
+
+        default:
+            AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
+            break;
+    }
+#undef SET_BOTH
+#undef SET_EXIT
+
+    /*
+     * Dtrace tracepoints go first.   We do them here at once so we don't
+     * have to copy the guest state saving and stuff a few dozen times.
+     * Down side is that we've got to repeat the switch, though this time
+     * we use enmEvent since the probes are a subset of what DBGF does.
+     */
+    if (fDtrace1 || fDtrace2)
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+        vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
+        switch (enmEvent1)
+        {
+            /** @todo consider which extra parameters would be helpful for each probe.   */
+            case DBGFEVENT_END: break;
+            case DBGFEVENT_XCPT_DE:                 VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_DB:                 VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
+            case DBGFEVENT_XCPT_BP:                 VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_OF:                 VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_BR:                 VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_UD:                 VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_NM:                 VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_DF:                 VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_TS:                 VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
+            case DBGFEVENT_XCPT_NP:                 VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
+            case DBGFEVENT_XCPT_SS:                 VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
+            case DBGFEVENT_XCPT_GP:                 VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
+            case DBGFEVENT_XCPT_PF:                 VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
+            case DBGFEVENT_XCPT_MF:                 VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_AC:                 VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_XF:                 VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_VE:                 VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
+            case DBGFEVENT_XCPT_SX:                 VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
+            case DBGFEVENT_INTERRUPT_SOFTWARE:      VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_INSTR_CPUID:             VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
+            case DBGFEVENT_INSTR_GETSEC:            VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_HALT:              VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_INVD:              VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_INVLPG:            VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RDPMC:             VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RDTSC:             VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RSM:               VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_CRX_READ:          VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_INSTR_CRX_WRITE:         VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_INSTR_DRX_READ:          VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_INSTR_DRX_WRITE:         VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_INSTR_RDMSR:             VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
+            case DBGFEVENT_INSTR_WRMSR:             VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
+                                                                        RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
+            case DBGFEVENT_INSTR_MWAIT:             VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_MONITOR:           VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_PAUSE:             VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_SGDT:              VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_SIDT:              VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_LGDT:              VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_LIDT:              VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_SLDT:              VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_STR:               VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_LLDT:              VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_LTR:               VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RDTSCP:            VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_WBINVD:            VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_XSETBV:            VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RDRAND:            VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_RDSEED:            VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_XSAVES:            VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_XRSTORS:           VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMM_CALL:          VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMCLEAR:       VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMLAUNCH:      VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMPTRLD:       VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMPTRST:       VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMREAD:        VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMRESUME:      VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMWRITE:       VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMXOFF:        VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMXON:         VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_INVEPT:        VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_INVVPID:       VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_INVPCID:       VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
+            case DBGFEVENT_INSTR_VMX_VMFUNC:        VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
+            default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
+        }
+        switch (enmEvent2)
+        {
+            /** @todo consider which extra parameters would be helpful for each probe. */
+            case DBGFEVENT_END: break;
+            case DBGFEVENT_EXIT_TASK_SWITCH:        VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_CPUID:              VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
+            case DBGFEVENT_EXIT_GETSEC:             VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_HALT:               VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_INVD:               VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_INVLPG:             VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RDPMC:              VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RDTSC:              VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RSM:                VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_CRX_READ:           VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_EXIT_CRX_WRITE:          VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_EXIT_DRX_READ:           VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_EXIT_DRX_WRITE:          VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
+            case DBGFEVENT_EXIT_RDMSR:              VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
+            case DBGFEVENT_EXIT_WRMSR:              VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
+                                                                       RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
+            case DBGFEVENT_EXIT_MWAIT:              VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_MONITOR:            VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_PAUSE:              VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_SGDT:               VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_SIDT:               VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_LGDT:               VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_LIDT:               VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_SLDT:               VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_STR:                VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_LLDT:               VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_LTR:                VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RDTSCP:             VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_WBINVD:             VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_XSETBV:             VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RDRAND:             VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_RDSEED:             VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_XSAVES:             VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_XRSTORS:            VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMM_CALL:           VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMCLEAR:        VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMLAUNCH:       VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMPTRLD:        VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMPTRST:        VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMREAD:         VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMRESUME:       VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMWRITE:        VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMXOFF:         VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMXON:          VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_INVEPT:         VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_INVVPID:        VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_INVPCID:        VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VMFUNC:         VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG:  VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_EPT_VIOLATION:  VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS:   VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
+            case DBGFEVENT_EXIT_VMX_VAPIC_WRITE:    VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
+            default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
+        }
+    }
+
+    /*
+     * Fire of the DBGF event, if enabled (our check here is just a quick one,
+     * the DBGF call will do a full check).
+     *
+     * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
+     * Note! If we have to events, we prioritize the first, i.e. the instruction
+     *       one, in order to avoid event nesting.
+     */
+    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
+    if (   enmEvent1 != DBGFEVENT_END
+        && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
+    {
+        vmxHCImportGuestState<CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+        VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+    else if (   enmEvent2 != DBGFEVENT_END
+             && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
+    {
+        vmxHCImportGuestState<CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+        VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    return VINF_SUCCESS;
+}
+
+
+/**
+ * Single-stepping VM-exit filtering.
+ *
+ * This is preprocessing the VM-exits and deciding whether we've gotten far
+ * enough to return VINF_EM_DBG_STEPPED already.  If not, normal VM-exit
+ * handling is performed.
+ *
+ * @returns Strict VBox status code (i.e. informational status codes too).
+ * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
+ * @param   pVmxTransient   The VMX-transient structure.
+ * @param   pDbgState       The debug state.
+ */
+DECLINLINE(VBOXSTRICTRC) vmxHCRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
+{
+    /*
+     * Expensive (saves context) generic dtrace VM-exit probe.
+     */
+    uint32_t const uExitReason = pVmxTransient->uExitReason;
+    if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
+    { /* more likely */ }
+    else
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
+        int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+        AssertRC(rc);
+        VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
+    }
+
+#ifndef IN_NEM_DARWIN
+    /*
+     * Check for host NMI, just to get that out of the way.
+     */
+    if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
+    { /* normally likely */ }
+    else
+    {
+        vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);
+        uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
+        if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
+            return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
+    }
+#endif
+
+    /*
+     * Check for single stepping event if we're stepping.
+     */
+    if (VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
+    {
+        switch (uExitReason)
+        {
+            case VMX_EXIT_MTF:
+                return vmxHCExitMtf(pVCpu, pVmxTransient);
+
+            /* Various events: */
+            case VMX_EXIT_XCPT_OR_NMI:
+            case VMX_EXIT_EXT_INT:
+            case VMX_EXIT_TRIPLE_FAULT:
+            case VMX_EXIT_INT_WINDOW:
+            case VMX_EXIT_NMI_WINDOW:
+            case VMX_EXIT_TASK_SWITCH:
+            case VMX_EXIT_TPR_BELOW_THRESHOLD:
+            case VMX_EXIT_APIC_ACCESS:
+            case VMX_EXIT_EPT_VIOLATION:
+            case VMX_EXIT_EPT_MISCONFIG:
+            case VMX_EXIT_PREEMPT_TIMER:
+
+            /* Instruction specific VM-exits: */
+            case VMX_EXIT_CPUID:
+            case VMX_EXIT_GETSEC:
+            case VMX_EXIT_HLT:
+            case VMX_EXIT_INVD:
+            case VMX_EXIT_INVLPG:
+            case VMX_EXIT_RDPMC:
+            case VMX_EXIT_RDTSC:
+            case VMX_EXIT_RSM:
+            case VMX_EXIT_VMCALL:
+            case VMX_EXIT_VMCLEAR:
+            case VMX_EXIT_VMLAUNCH:
+            case VMX_EXIT_VMPTRLD:
+            case VMX_EXIT_VMPTRST:
+            case VMX_EXIT_VMREAD:
+            case VMX_EXIT_VMRESUME:
+            case VMX_EXIT_VMWRITE:
+            case VMX_EXIT_VMXOFF:
+            case VMX_EXIT_VMXON:
+            case VMX_EXIT_MOV_CRX:
+            case VMX_EXIT_MOV_DRX:
+            case VMX_EXIT_IO_INSTR:
+            case VMX_EXIT_RDMSR:
+            case VMX_EXIT_WRMSR:
+            case VMX_EXIT_MWAIT:
+            case VMX_EXIT_MONITOR:
+            case VMX_EXIT_PAUSE:
+            case VMX_EXIT_GDTR_IDTR_ACCESS:
+            case VMX_EXIT_LDTR_TR_ACCESS:
+            case VMX_EXIT_INVEPT:
+            case VMX_EXIT_RDTSCP:
+            case VMX_EXIT_INVVPID:
+            case VMX_EXIT_WBINVD:
+            case VMX_EXIT_XSETBV:
+            case VMX_EXIT_RDRAND:
+            case VMX_EXIT_INVPCID:
+            case VMX_EXIT_VMFUNC:
+            case VMX_EXIT_RDSEED:
+            case VMX_EXIT_XSAVES:
+            case VMX_EXIT_XRSTORS:
+            {
+                int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
+                AssertRCReturn(rc, rc);
+                if (   pVCpu->cpum.GstCtx.rip    != pDbgState->uRipStart
+                    || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
+                    return VINF_EM_DBG_STEPPED;
+                break;
+            }
+
+            /* Errors and unexpected events: */
+            case VMX_EXIT_INIT_SIGNAL:
+            case VMX_EXIT_SIPI:
+            case VMX_EXIT_IO_SMI:
+            case VMX_EXIT_SMI:
+            case VMX_EXIT_ERR_INVALID_GUEST_STATE:
+            case VMX_EXIT_ERR_MSR_LOAD:
+            case VMX_EXIT_ERR_MACHINE_CHECK:
+            case VMX_EXIT_PML_FULL:
+            case VMX_EXIT_VIRTUALIZED_EOI:
+            case VMX_EXIT_APIC_WRITE:  /* Some talk about this being fault like, so I guess we must process it? */
+                break;
+
+            default:
+                AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
+                break;
+        }
+    }
+
+    /*
+     * Check for debugger event breakpoints and dtrace probes.
+     */
+    if (   uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
+        && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
+    {
+        VBOXSTRICTRC rcStrict = vmxHCHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
+        if (rcStrict != VINF_SUCCESS)
+            return rcStrict;
+    }
+
+    /*
+     * Normal processing.
+     */
+#ifdef HMVMX_USE_FUNCTION_TABLE
+    return g_aVMExitHandlers[uExitReason].pfn(pVCpu, pVmxTransient);
+#else
+    return vmxHCHandleExit(pVCpu, pVmxTransient, uExitReason);
+#endif
+}
+
+/** @} */