path: root/src/VBox/VMM/VMMAll/VMXAllTemplate.cpp.h

/* $Id: VMXAllTemplate.cpp.h $ */
/** @file
 * HM VMX (Intel VT-x) - Code template for our own hypervisor and the NEM darwin backend using Apple's Hypervisor.framework.
 */

/*
 * Copyright (C) 2012-2023 Oracle and/or its affiliates.
 *
 * This file is part of VirtualBox base platform packages, as
 * available from https://www.virtualbox.org.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation, in version 3 of the
 * License.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, see <https://www.gnu.org/licenses>.
 *
 * SPDX-License-Identifier: GPL-3.0-only
 */


/*********************************************************************************************************************************
*   Defined Constants And Macros                                                                                                 *
*********************************************************************************************************************************/
#if !defined(VMX_VMCS_WRITE_16) || !defined(VMX_VMCS_WRITE_32) || !defined(VMX_VMCS_WRITE_64) || !defined(VMX_VMCS_WRITE_64)
# error "At least one of the VMX_VMCS_WRITE_16, VMX_VMCS_WRITE_32, VMX_VMCS_WRITE_64 or VMX_VMCS_WRITE_64 is missing"
#endif


#if !defined(VMX_VMCS_READ_16) || !defined(VMX_VMCS_READ_32) || !defined(VMX_VMCS_READ_64) || !defined(VMX_VMCS_READ_64)
# error "At least one of the VMX_VMCS_READ_16, VMX_VMCS_READ_32, VMX_VMCS_READ_64 or VMX_VMCS_READ_64 is missing"
#endif

/** Enables condensing of VMREAD instructions, see vmxHCReadToTransient(). */
#define HMVMX_WITH_CONDENSED_VMREADS

/** Use the function table. */
#define HMVMX_USE_FUNCTION_TABLE

/** Determine which tagged-TLB flush handler to use. */
#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID             0
#define HMVMX_FLUSH_TAGGED_TLB_EPT                  1
#define HMVMX_FLUSH_TAGGED_TLB_VPID                 2
#define HMVMX_FLUSH_TAGGED_TLB_NONE                 3

/** Assert that all the given fields have been read from the VMCS. */
#ifdef VBOX_STRICT
# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) \
        do { \
            uint32_t const fVmcsFieldRead = ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead); \
            Assert((fVmcsFieldRead & (a_fReadFields)) == (a_fReadFields)); \
        } while (0)
#else
# define HMVMX_ASSERT_READ(a_pVmxTransient, a_fReadFields) do { } while (0)
#endif

/**
 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
 * guest using hardware-assisted VMX.
 *
 * This excludes state like GPRs (other than RSP) which are always are
 * swapped and restored across the world-switch and also registers like EFER,
 * MSR which cannot be modified by the guest without causing a VM-exit.
 */
#define HMVMX_CPUMCTX_EXTRN_ALL      (  CPUMCTX_EXTRN_RIP             \
                                      | CPUMCTX_EXTRN_RFLAGS          \
                                      | CPUMCTX_EXTRN_RSP             \
                                      | CPUMCTX_EXTRN_SREG_MASK       \
                                      | CPUMCTX_EXTRN_TABLE_MASK      \
                                      | CPUMCTX_EXTRN_KERNEL_GS_BASE  \
                                      | CPUMCTX_EXTRN_SYSCALL_MSRS    \
                                      | CPUMCTX_EXTRN_SYSENTER_MSRS   \
                                      | CPUMCTX_EXTRN_TSC_AUX         \
                                      | CPUMCTX_EXTRN_OTHER_MSRS      \
                                      | CPUMCTX_EXTRN_CR0             \
                                      | CPUMCTX_EXTRN_CR3             \
                                      | CPUMCTX_EXTRN_CR4             \
                                      | CPUMCTX_EXTRN_DR7             \
                                      | CPUMCTX_EXTRN_HWVIRT          \
                                      | CPUMCTX_EXTRN_INHIBIT_INT     \
                                      | CPUMCTX_EXTRN_INHIBIT_NMI)

/**
 * Exception bitmap mask for real-mode guests (real-on-v86).
 *
 * We need to intercept all exceptions manually except:
 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
 *   due to bugs in Intel CPUs.
 * - \#PF need not be intercepted even in real-mode if we have nested paging
 * support.
 */
#define HMVMX_REAL_MODE_XCPT_MASK    (  RT_BIT(X86_XCPT_DE)  /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI)   \
                                      | RT_BIT(X86_XCPT_BP)             | RT_BIT(X86_XCPT_OF)    | RT_BIT(X86_XCPT_BR)    \
                                      | RT_BIT(X86_XCPT_UD)             | RT_BIT(X86_XCPT_NM)    | RT_BIT(X86_XCPT_DF)    \
                                      | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS)    | RT_BIT(X86_XCPT_NP)    \
                                      | RT_BIT(X86_XCPT_SS)             | RT_BIT(X86_XCPT_GP)   /* RT_BIT(X86_XCPT_PF) */ \
                                      | RT_BIT(X86_XCPT_MF)  /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC)    \
                                      | RT_BIT(X86_XCPT_XF))

/** Maximum VM-instruction error number. */
#define HMVMX_INSTR_ERROR_MAX        28

/** Profiling macro. */
#ifdef HM_PROFILE_EXIT_DISPATCH
# define HMVMX_START_EXIT_DISPATCH_PROF()           STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
# define HMVMX_STOP_EXIT_DISPATCH_PROF()            STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitDispatch, ed)
#else
# define HMVMX_START_EXIT_DISPATCH_PROF()           do { } while (0)
# define HMVMX_STOP_EXIT_DISPATCH_PROF()            do { } while (0)
#endif

#ifndef IN_NEM_DARWIN
/** Assert that preemption is disabled or covered by thread-context hooks. */
# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu)          Assert(   VMMR0ThreadCtxHookIsEnabled((a_pVCpu))   \
                                                            || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))

/** Assert that we haven't migrated CPUs when thread-context hooks are not
 *  used. */
# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu)              AssertMsg(   VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
                                                               || (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
                                                               ("Illegal migration! Entered on CPU %u Current %u\n", \
                                                               (a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()))
#else
# define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu)          do { } while (0)
# define HMVMX_ASSERT_CPU_SAFE(a_pVCpu)              do { } while (0)
#endif

/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
 *  context. */
#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz)  AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
                                                              ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
                                                              (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))

/** Log the VM-exit reason with an easily visible marker to identify it in a
 *  potential sea of logging data. */
#define HMVMX_LOG_EXIT(a_pVCpu, a_uExitReason) \
    do { \
        Log4(("VM-exit: vcpu[%RU32] %85s -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu, \
             HMGetVmxExitName(a_uExitReason))); \
    } while (0) \


/*********************************************************************************************************************************
*   Structures and Typedefs                                                                                                      *
*********************************************************************************************************************************/
/**
 * Memory operand read or write access.
 */
typedef enum VMXMEMACCESS
{
    VMXMEMACCESS_READ  = 0,
    VMXMEMACCESS_WRITE = 1
} VMXMEMACCESS;


/**
 * VMX VM-exit handler.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 */
#ifndef HMVMX_USE_FUNCTION_TABLE
typedef VBOXSTRICTRC               FNVMXEXITHANDLER(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
#else
typedef DECLCALLBACKTYPE(VBOXSTRICTRC, FNVMXEXITHANDLER,(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient));
/** Pointer to VM-exit handler. */
typedef FNVMXEXITHANDLER          *PFNVMXEXITHANDLER;
#endif

/**
 * VMX VM-exit handler, non-strict status code.
 *
 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
 *
 * @returns VBox status code, no informational status code returned.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 *
 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
 *          use of that status code will be replaced with VINF_EM_SOMETHING
 *          later when switching over to IEM.
 */
#ifndef HMVMX_USE_FUNCTION_TABLE
typedef int                        FNVMXEXITHANDLERNSRC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
#else
typedef FNVMXEXITHANDLER           FNVMXEXITHANDLERNSRC;
#endif


/*********************************************************************************************************************************
*   Internal Functions                                                                                                           *
*********************************************************************************************************************************/
#ifndef HMVMX_USE_FUNCTION_TABLE
DECLINLINE(VBOXSTRICTRC)           vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
# define HMVMX_EXIT_DECL           DECLINLINE(VBOXSTRICTRC)
# define HMVMX_EXIT_NSRC_DECL      DECLINLINE(int)
#else
# define HMVMX_EXIT_DECL           static DECLCALLBACK(VBOXSTRICTRC)
# define HMVMX_EXIT_NSRC_DECL      HMVMX_EXIT_DECL
#endif
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
DECLINLINE(VBOXSTRICTRC)           vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient);
#endif

static int vmxHCImportGuestStateEx(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat);

/** @name VM-exit handler prototypes.
 * @{
 */
static FNVMXEXITHANDLER            vmxHCExitXcptOrNmi;
static FNVMXEXITHANDLER            vmxHCExitExtInt;
static FNVMXEXITHANDLER            vmxHCExitTripleFault;
static FNVMXEXITHANDLERNSRC        vmxHCExitIntWindow;
static FNVMXEXITHANDLERNSRC        vmxHCExitNmiWindow;
static FNVMXEXITHANDLER            vmxHCExitTaskSwitch;
static FNVMXEXITHANDLER            vmxHCExitCpuid;
static FNVMXEXITHANDLER            vmxHCExitGetsec;
static FNVMXEXITHANDLER            vmxHCExitHlt;
static FNVMXEXITHANDLERNSRC        vmxHCExitInvd;
static FNVMXEXITHANDLER            vmxHCExitInvlpg;
static FNVMXEXITHANDLER            vmxHCExitRdpmc;
static FNVMXEXITHANDLER            vmxHCExitVmcall;
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
static FNVMXEXITHANDLER            vmxHCExitVmclear;
static FNVMXEXITHANDLER            vmxHCExitVmlaunch;
static FNVMXEXITHANDLER            vmxHCExitVmptrld;
static FNVMXEXITHANDLER            vmxHCExitVmptrst;
static FNVMXEXITHANDLER            vmxHCExitVmread;
static FNVMXEXITHANDLER            vmxHCExitVmresume;
static FNVMXEXITHANDLER            vmxHCExitVmwrite;
static FNVMXEXITHANDLER            vmxHCExitVmxoff;
static FNVMXEXITHANDLER            vmxHCExitVmxon;
static FNVMXEXITHANDLER            vmxHCExitInvvpid;
# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
static FNVMXEXITHANDLER            vmxHCExitInvept;
# endif
#endif
static FNVMXEXITHANDLER            vmxHCExitRdtsc;
static FNVMXEXITHANDLER            vmxHCExitMovCRx;
static FNVMXEXITHANDLER            vmxHCExitMovDRx;
static FNVMXEXITHANDLER            vmxHCExitIoInstr;
static FNVMXEXITHANDLER            vmxHCExitRdmsr;
static FNVMXEXITHANDLER            vmxHCExitWrmsr;
static FNVMXEXITHANDLER            vmxHCExitMwait;
static FNVMXEXITHANDLER            vmxHCExitMtf;
static FNVMXEXITHANDLER            vmxHCExitMonitor;
static FNVMXEXITHANDLER            vmxHCExitPause;
static FNVMXEXITHANDLERNSRC        vmxHCExitTprBelowThreshold;
static FNVMXEXITHANDLER            vmxHCExitApicAccess;
static FNVMXEXITHANDLER            vmxHCExitEptViolation;
static FNVMXEXITHANDLER            vmxHCExitEptMisconfig;
static FNVMXEXITHANDLER            vmxHCExitRdtscp;
static FNVMXEXITHANDLER            vmxHCExitPreemptTimer;
static FNVMXEXITHANDLERNSRC        vmxHCExitWbinvd;
static FNVMXEXITHANDLER            vmxHCExitXsetbv;
static FNVMXEXITHANDLER            vmxHCExitInvpcid;
#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
static FNVMXEXITHANDLERNSRC        vmxHCExitSetPendingXcptUD;
#endif
static FNVMXEXITHANDLERNSRC        vmxHCExitErrInvalidGuestState;
static FNVMXEXITHANDLERNSRC        vmxHCExitErrUnexpected;
/** @} */

#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
/** @name Nested-guest VM-exit handler prototypes.
 * @{
 */
static FNVMXEXITHANDLER            vmxHCExitXcptOrNmiNested;
static FNVMXEXITHANDLER            vmxHCExitTripleFaultNested;
static FNVMXEXITHANDLERNSRC        vmxHCExitIntWindowNested;
static FNVMXEXITHANDLERNSRC        vmxHCExitNmiWindowNested;
static FNVMXEXITHANDLER            vmxHCExitTaskSwitchNested;
static FNVMXEXITHANDLER            vmxHCExitHltNested;
static FNVMXEXITHANDLER            vmxHCExitInvlpgNested;
static FNVMXEXITHANDLER            vmxHCExitRdpmcNested;
static FNVMXEXITHANDLER            vmxHCExitVmreadVmwriteNested;
static FNVMXEXITHANDLER            vmxHCExitRdtscNested;
static FNVMXEXITHANDLER            vmxHCExitMovCRxNested;
static FNVMXEXITHANDLER            vmxHCExitMovDRxNested;
static FNVMXEXITHANDLER            vmxHCExitIoInstrNested;
static FNVMXEXITHANDLER            vmxHCExitRdmsrNested;
static FNVMXEXITHANDLER            vmxHCExitWrmsrNested;
static FNVMXEXITHANDLER            vmxHCExitMwaitNested;
static FNVMXEXITHANDLER            vmxHCExitMtfNested;
static FNVMXEXITHANDLER            vmxHCExitMonitorNested;
static FNVMXEXITHANDLER            vmxHCExitPauseNested;
static FNVMXEXITHANDLERNSRC        vmxHCExitTprBelowThresholdNested;
static FNVMXEXITHANDLER            vmxHCExitApicAccessNested;
static FNVMXEXITHANDLER            vmxHCExitApicWriteNested;
static FNVMXEXITHANDLER            vmxHCExitVirtEoiNested;
static FNVMXEXITHANDLER            vmxHCExitRdtscpNested;
static FNVMXEXITHANDLERNSRC        vmxHCExitWbinvdNested;
static FNVMXEXITHANDLER            vmxHCExitInvpcidNested;
static FNVMXEXITHANDLERNSRC        vmxHCExitErrInvalidGuestStateNested;
static FNVMXEXITHANDLER            vmxHCExitInstrNested;
static FNVMXEXITHANDLER            vmxHCExitInstrWithInfoNested;
# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
static FNVMXEXITHANDLER            vmxHCExitEptViolationNested;
static FNVMXEXITHANDLER            vmxHCExitEptMisconfigNested;
# endif
/** @} */
#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */


/*********************************************************************************************************************************
*   Global Variables                                                                                                             *
*********************************************************************************************************************************/
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
/**
 * Array of all VMCS fields.
 * Any fields added to the VT-x spec. should be added here.
 *
 * Currently only used to derive shadow VMCS fields for hardware-assisted execution
 * of nested-guests.
 */
static const uint32_t g_aVmcsFields[] =
{
    /* 16-bit control fields. */
    VMX_VMCS16_VPID,
    VMX_VMCS16_POSTED_INT_NOTIFY_VECTOR,
    VMX_VMCS16_EPTP_INDEX,
    VMX_VMCS16_HLAT_PREFIX_SIZE,

    /* 16-bit guest-state fields. */
    VMX_VMCS16_GUEST_ES_SEL,
    VMX_VMCS16_GUEST_CS_SEL,
    VMX_VMCS16_GUEST_SS_SEL,
    VMX_VMCS16_GUEST_DS_SEL,
    VMX_VMCS16_GUEST_FS_SEL,
    VMX_VMCS16_GUEST_GS_SEL,
    VMX_VMCS16_GUEST_LDTR_SEL,
    VMX_VMCS16_GUEST_TR_SEL,
    VMX_VMCS16_GUEST_INTR_STATUS,
    VMX_VMCS16_GUEST_PML_INDEX,

    /* 16-bits host-state fields. */
    VMX_VMCS16_HOST_ES_SEL,
    VMX_VMCS16_HOST_CS_SEL,
    VMX_VMCS16_HOST_SS_SEL,
    VMX_VMCS16_HOST_DS_SEL,
    VMX_VMCS16_HOST_FS_SEL,
    VMX_VMCS16_HOST_GS_SEL,
    VMX_VMCS16_HOST_TR_SEL,

    /* 64-bit control fields. */
    VMX_VMCS64_CTRL_IO_BITMAP_A_FULL,
    VMX_VMCS64_CTRL_IO_BITMAP_A_HIGH,
    VMX_VMCS64_CTRL_IO_BITMAP_B_FULL,
    VMX_VMCS64_CTRL_IO_BITMAP_B_HIGH,
    VMX_VMCS64_CTRL_MSR_BITMAP_FULL,
    VMX_VMCS64_CTRL_MSR_BITMAP_HIGH,
    VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL,
    VMX_VMCS64_CTRL_EXIT_MSR_STORE_HIGH,
    VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL,
    VMX_VMCS64_CTRL_EXIT_MSR_LOAD_HIGH,
    VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL,
    VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_HIGH,
    VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL,
    VMX_VMCS64_CTRL_EXEC_VMCS_PTR_HIGH,
    VMX_VMCS64_CTRL_EXEC_PML_ADDR_FULL,
    VMX_VMCS64_CTRL_EXEC_PML_ADDR_HIGH,
    VMX_VMCS64_CTRL_TSC_OFFSET_FULL,
    VMX_VMCS64_CTRL_TSC_OFFSET_HIGH,
    VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL,
    VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_HIGH,
    VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL,
    VMX_VMCS64_CTRL_APIC_ACCESSADDR_HIGH,
    VMX_VMCS64_CTRL_POSTED_INTR_DESC_FULL,
    VMX_VMCS64_CTRL_POSTED_INTR_DESC_HIGH,
    VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL,
    VMX_VMCS64_CTRL_VMFUNC_CTRLS_HIGH,
    VMX_VMCS64_CTRL_EPTP_FULL,
    VMX_VMCS64_CTRL_EPTP_HIGH,
    VMX_VMCS64_CTRL_EOI_BITMAP_0_FULL,
    VMX_VMCS64_CTRL_EOI_BITMAP_0_HIGH,
    VMX_VMCS64_CTRL_EOI_BITMAP_1_FULL,
    VMX_VMCS64_CTRL_EOI_BITMAP_1_HIGH,
    VMX_VMCS64_CTRL_EOI_BITMAP_2_FULL,
    VMX_VMCS64_CTRL_EOI_BITMAP_2_HIGH,
    VMX_VMCS64_CTRL_EOI_BITMAP_3_FULL,
    VMX_VMCS64_CTRL_EOI_BITMAP_3_HIGH,
    VMX_VMCS64_CTRL_EPTP_LIST_FULL,
    VMX_VMCS64_CTRL_EPTP_LIST_HIGH,
    VMX_VMCS64_CTRL_VMREAD_BITMAP_FULL,
    VMX_VMCS64_CTRL_VMREAD_BITMAP_HIGH,
    VMX_VMCS64_CTRL_VMWRITE_BITMAP_FULL,
    VMX_VMCS64_CTRL_VMWRITE_BITMAP_HIGH,
    VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_FULL,
    VMX_VMCS64_CTRL_VE_XCPT_INFO_ADDR_HIGH,
    VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_FULL,
    VMX_VMCS64_CTRL_XSS_EXITING_BITMAP_HIGH,
    VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_FULL,
    VMX_VMCS64_CTRL_ENCLS_EXITING_BITMAP_HIGH,
    VMX_VMCS64_CTRL_SPPTP_FULL,
    VMX_VMCS64_CTRL_SPPTP_HIGH,
    VMX_VMCS64_CTRL_TSC_MULTIPLIER_FULL,
    VMX_VMCS64_CTRL_TSC_MULTIPLIER_HIGH,
    VMX_VMCS64_CTRL_PROC_EXEC3_FULL,
    VMX_VMCS64_CTRL_PROC_EXEC3_HIGH,
    VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_FULL,
    VMX_VMCS64_CTRL_ENCLV_EXITING_BITMAP_HIGH,
    VMX_VMCS64_CTRL_PCONFIG_EXITING_BITMAP_FULL,
    VMX_VMCS64_CTRL_PCONFIG_EXITING_BITMAP_HIGH,
    VMX_VMCS64_CTRL_HLAT_PTR_FULL,
    VMX_VMCS64_CTRL_HLAT_PTR_HIGH,
    VMX_VMCS64_CTRL_EXIT2_FULL,
    VMX_VMCS64_CTRL_EXIT2_HIGH,

    /* 64-bit read-only data fields. */
    VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,
    VMX_VMCS64_RO_GUEST_PHYS_ADDR_HIGH,

    /* 64-bit guest-state fields. */
    VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL,
    VMX_VMCS64_GUEST_VMCS_LINK_PTR_HIGH,
    VMX_VMCS64_GUEST_DEBUGCTL_FULL,
    VMX_VMCS64_GUEST_DEBUGCTL_HIGH,
    VMX_VMCS64_GUEST_PAT_FULL,
    VMX_VMCS64_GUEST_PAT_HIGH,
    VMX_VMCS64_GUEST_EFER_FULL,
    VMX_VMCS64_GUEST_EFER_HIGH,
    VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL,
    VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_HIGH,
    VMX_VMCS64_GUEST_PDPTE0_FULL,
    VMX_VMCS64_GUEST_PDPTE0_HIGH,
    VMX_VMCS64_GUEST_PDPTE1_FULL,
    VMX_VMCS64_GUEST_PDPTE1_HIGH,
    VMX_VMCS64_GUEST_PDPTE2_FULL,
    VMX_VMCS64_GUEST_PDPTE2_HIGH,
    VMX_VMCS64_GUEST_PDPTE3_FULL,
    VMX_VMCS64_GUEST_PDPTE3_HIGH,
    VMX_VMCS64_GUEST_BNDCFGS_FULL,
    VMX_VMCS64_GUEST_BNDCFGS_HIGH,
    VMX_VMCS64_GUEST_RTIT_CTL_FULL,
    VMX_VMCS64_GUEST_RTIT_CTL_HIGH,
    VMX_VMCS64_GUEST_PKRS_FULL,
    VMX_VMCS64_GUEST_PKRS_HIGH,

    /* 64-bit host-state fields. */
    VMX_VMCS64_HOST_PAT_FULL,
    VMX_VMCS64_HOST_PAT_HIGH,
    VMX_VMCS64_HOST_EFER_FULL,
    VMX_VMCS64_HOST_EFER_HIGH,
    VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL,
    VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_HIGH,
    VMX_VMCS64_HOST_PKRS_FULL,
    VMX_VMCS64_HOST_PKRS_HIGH,

    /* 32-bit control fields. */
    VMX_VMCS32_CTRL_PIN_EXEC,
    VMX_VMCS32_CTRL_PROC_EXEC,
    VMX_VMCS32_CTRL_EXCEPTION_BITMAP,
    VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK,
    VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH,
    VMX_VMCS32_CTRL_CR3_TARGET_COUNT,
    VMX_VMCS32_CTRL_EXIT,
    VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT,
    VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT,
    VMX_VMCS32_CTRL_ENTRY,
    VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT,
    VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO,
    VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE,
    VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH,
    VMX_VMCS32_CTRL_TPR_THRESHOLD,
    VMX_VMCS32_CTRL_PROC_EXEC2,
    VMX_VMCS32_CTRL_PLE_GAP,
    VMX_VMCS32_CTRL_PLE_WINDOW,

    /* 32-bits read-only fields. */
    VMX_VMCS32_RO_VM_INSTR_ERROR,
    VMX_VMCS32_RO_EXIT_REASON,
    VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,
    VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE,
    VMX_VMCS32_RO_IDT_VECTORING_INFO,
    VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,
    VMX_VMCS32_RO_EXIT_INSTR_LENGTH,
    VMX_VMCS32_RO_EXIT_INSTR_INFO,

    /* 32-bit guest-state fields. */
    VMX_VMCS32_GUEST_ES_LIMIT,
    VMX_VMCS32_GUEST_CS_LIMIT,
    VMX_VMCS32_GUEST_SS_LIMIT,
    VMX_VMCS32_GUEST_DS_LIMIT,
    VMX_VMCS32_GUEST_FS_LIMIT,
    VMX_VMCS32_GUEST_GS_LIMIT,
    VMX_VMCS32_GUEST_LDTR_LIMIT,
    VMX_VMCS32_GUEST_TR_LIMIT,
    VMX_VMCS32_GUEST_GDTR_LIMIT,
    VMX_VMCS32_GUEST_IDTR_LIMIT,
    VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
    VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
    VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
    VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
    VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
    VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS,
    VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS,
    VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS,
    VMX_VMCS32_GUEST_INT_STATE,
    VMX_VMCS32_GUEST_ACTIVITY_STATE,
    VMX_VMCS32_GUEST_SMBASE,
    VMX_VMCS32_GUEST_SYSENTER_CS,
    VMX_VMCS32_PREEMPT_TIMER_VALUE,

    /* 32-bit host-state fields. */
    VMX_VMCS32_HOST_SYSENTER_CS,

    /* Natural-width control fields. */
    VMX_VMCS_CTRL_CR0_MASK,
    VMX_VMCS_CTRL_CR4_MASK,
    VMX_VMCS_CTRL_CR0_READ_SHADOW,
    VMX_VMCS_CTRL_CR4_READ_SHADOW,
    VMX_VMCS_CTRL_CR3_TARGET_VAL0,
    VMX_VMCS_CTRL_CR3_TARGET_VAL1,
    VMX_VMCS_CTRL_CR3_TARGET_VAL2,
    VMX_VMCS_CTRL_CR3_TARGET_VAL3,

    /* Natural-width read-only data fields. */
    VMX_VMCS_RO_EXIT_QUALIFICATION,
    VMX_VMCS_RO_IO_RCX,
    VMX_VMCS_RO_IO_RSI,
    VMX_VMCS_RO_IO_RDI,
    VMX_VMCS_RO_IO_RIP,
    VMX_VMCS_RO_GUEST_LINEAR_ADDR,

    /* Natural-width guest-state field */
    VMX_VMCS_GUEST_CR0,
    VMX_VMCS_GUEST_CR3,
    VMX_VMCS_GUEST_CR4,
    VMX_VMCS_GUEST_ES_BASE,
    VMX_VMCS_GUEST_CS_BASE,
    VMX_VMCS_GUEST_SS_BASE,
    VMX_VMCS_GUEST_DS_BASE,
    VMX_VMCS_GUEST_FS_BASE,
    VMX_VMCS_GUEST_GS_BASE,
    VMX_VMCS_GUEST_LDTR_BASE,
    VMX_VMCS_GUEST_TR_BASE,
    VMX_VMCS_GUEST_GDTR_BASE,
    VMX_VMCS_GUEST_IDTR_BASE,
    VMX_VMCS_GUEST_DR7,
    VMX_VMCS_GUEST_RSP,
    VMX_VMCS_GUEST_RIP,
    VMX_VMCS_GUEST_RFLAGS,
    VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
    VMX_VMCS_GUEST_SYSENTER_ESP,
    VMX_VMCS_GUEST_SYSENTER_EIP,
    VMX_VMCS_GUEST_S_CET,
    VMX_VMCS_GUEST_SSP,
    VMX_VMCS_GUEST_INTR_SSP_TABLE_ADDR,

    /* Natural-width host-state fields */
    VMX_VMCS_HOST_CR0,
    VMX_VMCS_HOST_CR3,
    VMX_VMCS_HOST_CR4,
    VMX_VMCS_HOST_FS_BASE,
    VMX_VMCS_HOST_GS_BASE,
    VMX_VMCS_HOST_TR_BASE,
    VMX_VMCS_HOST_GDTR_BASE,
    VMX_VMCS_HOST_IDTR_BASE,
    VMX_VMCS_HOST_SYSENTER_ESP,
    VMX_VMCS_HOST_SYSENTER_EIP,
    VMX_VMCS_HOST_RSP,
    VMX_VMCS_HOST_RIP,
    VMX_VMCS_HOST_S_CET,
    VMX_VMCS_HOST_SSP,
    VMX_VMCS_HOST_INTR_SSP_TABLE_ADDR
};
#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */

#ifdef HMVMX_USE_FUNCTION_TABLE
/**
 * VMX_EXIT dispatch table.
 */
static const struct CLANG11NOTHROWWEIRDNESS { PFNVMXEXITHANDLER pfn; } g_aVMExitHandlers[VMX_EXIT_MAX + 1] =
{
    /*  0  VMX_EXIT_XCPT_OR_NMI             */  { vmxHCExitXcptOrNmi },
    /*  1  VMX_EXIT_EXT_INT                 */  { vmxHCExitExtInt },
    /*  2  VMX_EXIT_TRIPLE_FAULT            */  { vmxHCExitTripleFault },
    /*  3  VMX_EXIT_INIT_SIGNAL             */  { vmxHCExitErrUnexpected },
    /*  4  VMX_EXIT_SIPI                    */  { vmxHCExitErrUnexpected },
    /*  5  VMX_EXIT_IO_SMI                  */  { vmxHCExitErrUnexpected },
    /*  6  VMX_EXIT_SMI                     */  { vmxHCExitErrUnexpected },
    /*  7  VMX_EXIT_INT_WINDOW              */  { vmxHCExitIntWindow },
    /*  8  VMX_EXIT_NMI_WINDOW              */  { vmxHCExitNmiWindow },
    /*  9  VMX_EXIT_TASK_SWITCH             */  { vmxHCExitTaskSwitch },
    /* 10  VMX_EXIT_CPUID                   */  { vmxHCExitCpuid },
    /* 11  VMX_EXIT_GETSEC                  */  { vmxHCExitGetsec },
    /* 12  VMX_EXIT_HLT                     */  { vmxHCExitHlt },
    /* 13  VMX_EXIT_INVD                    */  { vmxHCExitInvd },
    /* 14  VMX_EXIT_INVLPG                  */  { vmxHCExitInvlpg },
    /* 15  VMX_EXIT_RDPMC                   */  { vmxHCExitRdpmc },
    /* 16  VMX_EXIT_RDTSC                   */  { vmxHCExitRdtsc },
    /* 17  VMX_EXIT_RSM                     */  { vmxHCExitErrUnexpected },
    /* 18  VMX_EXIT_VMCALL                  */  { vmxHCExitVmcall },
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
    /* 19  VMX_EXIT_VMCLEAR                 */  { vmxHCExitVmclear },
    /* 20  VMX_EXIT_VMLAUNCH                */  { vmxHCExitVmlaunch },
    /* 21  VMX_EXIT_VMPTRLD                 */  { vmxHCExitVmptrld },
    /* 22  VMX_EXIT_VMPTRST                 */  { vmxHCExitVmptrst },
    /* 23  VMX_EXIT_VMREAD                  */  { vmxHCExitVmread },
    /* 24  VMX_EXIT_VMRESUME                */  { vmxHCExitVmresume },
    /* 25  VMX_EXIT_VMWRITE                 */  { vmxHCExitVmwrite },
    /* 26  VMX_EXIT_VMXOFF                  */  { vmxHCExitVmxoff },
    /* 27  VMX_EXIT_VMXON                   */  { vmxHCExitVmxon },
#else
    /* 19  VMX_EXIT_VMCLEAR                 */  { vmxHCExitSetPendingXcptUD },
    /* 20  VMX_EXIT_VMLAUNCH                */  { vmxHCExitSetPendingXcptUD },
    /* 21  VMX_EXIT_VMPTRLD                 */  { vmxHCExitSetPendingXcptUD },
    /* 22  VMX_EXIT_VMPTRST                 */  { vmxHCExitSetPendingXcptUD },
    /* 23  VMX_EXIT_VMREAD                  */  { vmxHCExitSetPendingXcptUD },
    /* 24  VMX_EXIT_VMRESUME                */  { vmxHCExitSetPendingXcptUD },
    /* 25  VMX_EXIT_VMWRITE                 */  { vmxHCExitSetPendingXcptUD },
    /* 26  VMX_EXIT_VMXOFF                  */  { vmxHCExitSetPendingXcptUD },
    /* 27  VMX_EXIT_VMXON                   */  { vmxHCExitSetPendingXcptUD },
#endif
    /* 28  VMX_EXIT_MOV_CRX                 */  { vmxHCExitMovCRx },
    /* 29  VMX_EXIT_MOV_DRX                 */  { vmxHCExitMovDRx },
    /* 30  VMX_EXIT_IO_INSTR                */  { vmxHCExitIoInstr },
    /* 31  VMX_EXIT_RDMSR                   */  { vmxHCExitRdmsr },
    /* 32  VMX_EXIT_WRMSR                   */  { vmxHCExitWrmsr },
    /* 33  VMX_EXIT_ERR_INVALID_GUEST_STATE */  { vmxHCExitErrInvalidGuestState },
    /* 34  VMX_EXIT_ERR_MSR_LOAD            */  { vmxHCExitErrUnexpected },
    /* 35  UNDEFINED                        */  { vmxHCExitErrUnexpected },
    /* 36  VMX_EXIT_MWAIT                   */  { vmxHCExitMwait },
    /* 37  VMX_EXIT_MTF                     */  { vmxHCExitMtf },
    /* 38  UNDEFINED                        */  { vmxHCExitErrUnexpected },
    /* 39  VMX_EXIT_MONITOR                 */  { vmxHCExitMonitor },
    /* 40  VMX_EXIT_PAUSE                   */  { vmxHCExitPause },
    /* 41  VMX_EXIT_ERR_MACHINE_CHECK       */  { vmxHCExitErrUnexpected },
    /* 42  UNDEFINED                        */  { vmxHCExitErrUnexpected },
    /* 43  VMX_EXIT_TPR_BELOW_THRESHOLD     */  { vmxHCExitTprBelowThreshold },
    /* 44  VMX_EXIT_APIC_ACCESS             */  { vmxHCExitApicAccess },
    /* 45  VMX_EXIT_VIRTUALIZED_EOI         */  { vmxHCExitErrUnexpected },
    /* 46  VMX_EXIT_GDTR_IDTR_ACCESS        */  { vmxHCExitErrUnexpected },
    /* 47  VMX_EXIT_LDTR_TR_ACCESS          */  { vmxHCExitErrUnexpected },
    /* 48  VMX_EXIT_EPT_VIOLATION           */  { vmxHCExitEptViolation },
    /* 49  VMX_EXIT_EPT_MISCONFIG           */  { vmxHCExitEptMisconfig },
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
    /* 50  VMX_EXIT_INVEPT                  */  { vmxHCExitInvept },
#else
    /* 50  VMX_EXIT_INVEPT                  */  { vmxHCExitSetPendingXcptUD },
#endif
    /* 51  VMX_EXIT_RDTSCP                  */  { vmxHCExitRdtscp },
    /* 52  VMX_EXIT_PREEMPT_TIMER           */  { vmxHCExitPreemptTimer },
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
    /* 53  VMX_EXIT_INVVPID                 */  { vmxHCExitInvvpid },
#else
    /* 53  VMX_EXIT_INVVPID                 */  { vmxHCExitSetPendingXcptUD },
#endif
    /* 54  VMX_EXIT_WBINVD                  */  { vmxHCExitWbinvd },
    /* 55  VMX_EXIT_XSETBV                  */  { vmxHCExitXsetbv },
    /* 56  VMX_EXIT_APIC_WRITE              */  { vmxHCExitErrUnexpected },
    /* 57  VMX_EXIT_RDRAND                  */  { vmxHCExitErrUnexpected },
    /* 58  VMX_EXIT_INVPCID                 */  { vmxHCExitInvpcid },
    /* 59  VMX_EXIT_VMFUNC                  */  { vmxHCExitErrUnexpected },
    /* 60  VMX_EXIT_ENCLS                   */  { vmxHCExitErrUnexpected },
    /* 61  VMX_EXIT_RDSEED                  */  { vmxHCExitErrUnexpected },
    /* 62  VMX_EXIT_PML_FULL                */  { vmxHCExitErrUnexpected },
    /* 63  VMX_EXIT_XSAVES                  */  { vmxHCExitErrUnexpected },
    /* 64  VMX_EXIT_XRSTORS                 */  { vmxHCExitErrUnexpected },
    /* 65  UNDEFINED                        */  { vmxHCExitErrUnexpected },
    /* 66  VMX_EXIT_SPP_EVENT               */  { vmxHCExitErrUnexpected },
    /* 67  VMX_EXIT_UMWAIT                  */  { vmxHCExitErrUnexpected },
    /* 68  VMX_EXIT_TPAUSE                  */  { vmxHCExitErrUnexpected },
    /* 69  VMX_EXIT_LOADIWKEY               */  { vmxHCExitErrUnexpected },
};
#endif /* HMVMX_USE_FUNCTION_TABLE */

#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
{
    /*  0 */ "(Not Used)",
    /*  1 */ "VMCALL executed in VMX root operation.",
    /*  2 */ "VMCLEAR with invalid physical address.",
    /*  3 */ "VMCLEAR with VMXON pointer.",
    /*  4 */ "VMLAUNCH with non-clear VMCS.",
    /*  5 */ "VMRESUME with non-launched VMCS.",
    /*  6 */ "VMRESUME after VMXOFF",
    /*  7 */ "VM-entry with invalid control fields.",
    /*  8 */ "VM-entry with invalid host state fields.",
    /*  9 */ "VMPTRLD with invalid physical address.",
    /* 10 */ "VMPTRLD with VMXON pointer.",
    /* 11 */ "VMPTRLD with incorrect revision identifier.",
    /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
    /* 13 */ "VMWRITE to read-only VMCS component.",
    /* 14 */ "(Not Used)",
    /* 15 */ "VMXON executed in VMX root operation.",
    /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
    /* 17 */ "VM-entry with non-launched executing VMCS.",
    /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
    /* 19 */ "VMCALL with non-clear VMCS.",
    /* 20 */ "VMCALL with invalid VM-exit control fields.",
    /* 21 */ "(Not Used)",
    /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
    /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
    /* 24 */ "VMCALL with invalid SMM-monitor features.",
    /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
    /* 26 */ "VM-entry with events blocked by MOV SS.",
    /* 27 */ "(Not Used)",
    /* 28 */ "Invalid operand to INVEPT/INVVPID."
};
#endif /* VBOX_STRICT && LOG_ENABLED */


/**
 * Gets the CR0 guest/host mask.
 *
 * These bits typically does not change through the lifetime of a VM. Any bit set in
 * this mask is owned by the host/hypervisor and would cause a VM-exit when modified
 * by the guest.
 *
 * @returns The CR0 guest/host mask.
 * @param   pVCpu   The cross context virtual CPU structure.
 */
static uint64_t vmxHCGetFixedCr0Mask(PCVMCPUCC pVCpu)
{
    /*
     * Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
     * to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
     *
     * Furthermore, modifications to any bits that are reserved/unspecified currently
     * by the Intel spec. must also cause a VM-exit. This prevents unpredictable behavior
     * when future CPUs specify and use currently reserved/unspecified bits.
     */
    /** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
     *        enmGuestMode to be in-sync with the current mode. See @bugref{6398}
     *        and @bugref{6944}. */
    PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
    AssertCompile(RT_HI_U32(VMX_EXIT_HOST_CR0_IGNORE_MASK) == UINT32_C(0xffffffff));    /* Paranoia. */
    return (  X86_CR0_PE
            | X86_CR0_NE
            | (VM_IS_VMX_NESTED_PAGING(pVM) ? 0 : X86_CR0_WP)
            | X86_CR0_PG
            | VMX_EXIT_HOST_CR0_IGNORE_MASK);
}


/**
 * Gets the CR4 guest/host mask.
 *
 * These bits typically does not change through the lifetime of a VM. Any bit set in
 * this mask is owned by the host/hypervisor and would cause a VM-exit when modified
 * by the guest.
 *
 * @returns The CR4 guest/host mask.
 * @param   pVCpu   The cross context virtual CPU structure.
 */
static uint64_t vmxHCGetFixedCr4Mask(PCVMCPUCC pVCpu)
{
    /*
     * We construct a mask of all CR4 bits that the guest can modify without causing
     * a VM-exit. Then invert this mask to obtain all CR4 bits that should cause
     * a VM-exit when the guest attempts to modify them when executing using
     * hardware-assisted VMX.
     *
     * When a feature is not exposed to the guest (and may be present on the host),
     * we want to intercept guest modifications to the bit so we can emulate proper
     * behavior (e.g., #GP).
     *
     * Furthermore, only modifications to those bits that don't require immediate
     * emulation is allowed. For e.g., PCIDE is excluded because the behavior
     * depends on CR3 which might not always be the guest value while executing
     * using hardware-assisted VMX.
     */
    PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
    bool fFsGsBase    = pVM->cpum.ro.GuestFeatures.fFsGsBase;
#ifdef IN_NEM_DARWIN
    bool fXSaveRstor  = pVM->cpum.ro.GuestFeatures.fXSaveRstor;
#endif
    bool fFxSaveRstor = pVM->cpum.ro.GuestFeatures.fFxSaveRstor;

    /*
     * Paranoia.
     * Ensure features exposed to the guest are present on the host.
     */
    AssertStmt(!fFsGsBase    || g_CpumHostFeatures.s.fFsGsBase,    fFsGsBase = 0);
#ifdef IN_NEM_DARWIN
    AssertStmt(!fXSaveRstor  || g_CpumHostFeatures.s.fXSaveRstor,  fXSaveRstor = 0);
#endif
    AssertStmt(!fFxSaveRstor || g_CpumHostFeatures.s.fFxSaveRstor, fFxSaveRstor = 0);

    uint64_t const fGstMask = X86_CR4_PVI
                            | X86_CR4_TSD
                            | X86_CR4_DE
                            | X86_CR4_MCE
                            | X86_CR4_PCE
                            | X86_CR4_OSXMMEEXCPT
                            | (fFsGsBase    ? X86_CR4_FSGSBASE : 0)
#ifdef IN_NEM_DARWIN /* On native VT-x setting OSXSAVE must exit as we need to load guest XCR0 (see
                        fLoadSaveGuestXcr0). These exits are not needed on Darwin as that's not our problem. */
                            | (fXSaveRstor  ? X86_CR4_OSXSAVE  : 0)
#endif
                            | (fFxSaveRstor ? X86_CR4_OSFXSR   : 0);
    return ~fGstMask;
}


/**
 * Adds one or more exceptions to the exception bitmap and commits it to the current
 * VMCS.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 * @param   uXcptMask       The exception(s) to add.
 */
static void vmxHCAddXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
{
    PVMXVMCSINFO pVmcsInfo   = pVmxTransient->pVmcsInfo;
    uint32_t     uXcptBitmap = pVmcsInfo->u32XcptBitmap;
    if ((uXcptBitmap & uXcptMask) != uXcptMask)
    {
        uXcptBitmap |= uXcptMask;
        int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
        AssertRC(rc);
        pVmcsInfo->u32XcptBitmap = uXcptBitmap;
    }
}


/**
 * Adds an exception to the exception bitmap and commits it to the current VMCS.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 * @param   uXcpt           The exception to add.
 */
static void vmxHCAddXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
{
    Assert(uXcpt <= X86_XCPT_LAST);
    vmxHCAddXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT_32(uXcpt));
}


/**
 * Remove one or more exceptions from the exception bitmap and commits it to the
 * current VMCS.
 *
 * This takes care of not removing the exception intercept if a nested-guest
 * requires the exception to be intercepted.
 *
 * @returns VBox status code.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 * @param   uXcptMask       The exception(s) to remove.
 */
static int vmxHCRemoveXcptInterceptMask(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
{
    PVMXVMCSINFO pVmcsInfo   = pVmxTransient->pVmcsInfo;
    uint32_t     uXcptBitmap = pVmcsInfo->u32XcptBitmap;
    if (uXcptBitmap & uXcptMask)
    {
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
        if (!pVmxTransient->fIsNestedGuest)
        { /* likely */ }
        else
            uXcptMask &= ~pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u32XcptBitmap;
#endif
#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
        uXcptMask &= ~(  RT_BIT(X86_XCPT_BP)
                       | RT_BIT(X86_XCPT_DE)
                       | RT_BIT(X86_XCPT_NM)
                       | RT_BIT(X86_XCPT_TS)
                       | RT_BIT(X86_XCPT_UD)
                       | RT_BIT(X86_XCPT_NP)
                       | RT_BIT(X86_XCPT_SS)
                       | RT_BIT(X86_XCPT_GP)
                       | RT_BIT(X86_XCPT_PF)
                       | RT_BIT(X86_XCPT_MF));
#elif defined(HMVMX_ALWAYS_TRAP_PF)
        uXcptMask &= ~RT_BIT(X86_XCPT_PF);
#endif
        if (uXcptMask)
        {
            /* Validate we are not removing any essential exception intercepts. */
#ifndef IN_NEM_DARWIN
            Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging || !(uXcptMask & RT_BIT(X86_XCPT_PF)));
#else
            Assert(!(uXcptMask & RT_BIT(X86_XCPT_PF)));
#endif
            NOREF(pVCpu);
            Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
            Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));

            /* Remove it from the exception bitmap. */
            uXcptBitmap &= ~uXcptMask;

            /* Commit and update the cache if necessary. */
            if (pVmcsInfo->u32XcptBitmap != uXcptBitmap)
            {
                int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
                AssertRC(rc);
                pVmcsInfo->u32XcptBitmap = uXcptBitmap;
            }
        }
    }
    return VINF_SUCCESS;
}


/**
 * Remove an exceptions from the exception bitmap and commits it to the current
 * VMCS.
 *
 * @returns VBox status code.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 * @param   uXcpt           The exception to remove.
 */
static int vmxHCRemoveXcptIntercept(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
{
    return vmxHCRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
}

#ifdef VBOX_WITH_NESTED_HWVIRT_VMX

/**
 * Loads the shadow VMCS specified by the VMCS info. object.
 *
 * @returns VBox status code.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks Can be called with interrupts disabled.
 */
static int vmxHCLoadShadowVmcs(PVMXVMCSINFO pVmcsInfo)
{
    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
    Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);

    int rc = VMXLoadVmcs(pVmcsInfo->HCPhysShadowVmcs);
    if (RT_SUCCESS(rc))
        pVmcsInfo->fShadowVmcsState |= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
    return rc;
}


/**
 * Clears the shadow VMCS specified by the VMCS info. object.
 *
 * @returns VBox status code.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks Can be called with interrupts disabled.
 */
static int vmxHCClearShadowVmcs(PVMXVMCSINFO pVmcsInfo)
{
    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
    Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);

    int rc = VMXClearVmcs(pVmcsInfo->HCPhysShadowVmcs);
    if (RT_SUCCESS(rc))
        pVmcsInfo->fShadowVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
    return rc;
}


/**
 * Switches from and to the specified VMCSes.
 *
 * @returns VBox status code.
 * @param   pVmcsInfoFrom   The VMCS info. object we are switching from.
 * @param   pVmcsInfoTo     The VMCS info. object we are switching to.
 *
 * @remarks Called with interrupts disabled.
 */
static int vmxHCSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
{
    /*
     * Clear the VMCS we are switching out if it has not already been cleared.
     * This will sync any CPU internal data back to the VMCS.
     */
    if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
    {
        int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
        if (RT_SUCCESS(rc))
        {
            /*
             * The shadow VMCS, if any, would not be active at this point since we
             * would have cleared it while importing the virtual hardware-virtualization
             * state as part the VMLAUNCH/VMRESUME VM-exit. Hence, there's no need to
             * clear the shadow VMCS here, just assert for safety.
             */
            Assert(!pVmcsInfoFrom->pvShadowVmcs || pVmcsInfoFrom->fShadowVmcsState == VMX_V_VMCS_LAUNCH_STATE_CLEAR);
        }
        else
            return rc;
    }

    /*
     * Clear the VMCS we are switching to if it has not already been cleared.
     * This will initialize the VMCS launch state to "clear" required for loading it.
     *
     * See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
     */
    if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
    {
        int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
        if (RT_SUCCESS(rc))
        { /* likely */ }
        else
            return rc;
    }

    /*
     * Finally, load the VMCS we are switching to.
     */
    return hmR0VmxLoadVmcs(pVmcsInfoTo);
}


/**
 * Switches between the guest VMCS and the nested-guest VMCS as specified by the
 * caller.
 *
 * @returns VBox status code.
 * @param   pVCpu                   The cross context virtual CPU structure.
 * @param   fSwitchToNstGstVmcs     Whether to switch to the nested-guest VMCS (pass
 *                                  true) or guest VMCS (pass false).
 */
static int vmxHCSwitchToGstOrNstGstVmcs(PVMCPUCC pVCpu, bool fSwitchToNstGstVmcs)
{
    /* Ensure we have synced everything from the guest-CPU context to the VMCS before switching. */
    HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);

    PVMXVMCSINFO pVmcsInfoFrom;
    PVMXVMCSINFO pVmcsInfoTo;
    if (fSwitchToNstGstVmcs)
    {
        pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfo;
        pVmcsInfoTo   = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
    }
    else
    {
        pVmcsInfoFrom = &pVCpu->hmr0.s.vmx.VmcsInfoNstGst;
        pVmcsInfoTo   = &pVCpu->hmr0.s.vmx.VmcsInfo;
    }

    /*
     * Disable interrupts to prevent being preempted while we switch the current VMCS as the
     * preemption hook code path acquires the current VMCS.
     */
    RTCCUINTREG const fEFlags = ASMIntDisableFlags();

    int rc = vmxHCSwitchVmcs(pVmcsInfoFrom, pVmcsInfoTo);
    if (RT_SUCCESS(rc))
    {
        pVCpu->hmr0.s.vmx.fSwitchedToNstGstVmcs           = fSwitchToNstGstVmcs;
        pVCpu->hm.s.vmx.fSwitchedToNstGstVmcsCopyForRing3 = fSwitchToNstGstVmcs;

        /*
         * If we are switching to a VMCS that was executed on a different host CPU or was
         * never executed before, flag that we need to export the host state before executing
         * guest/nested-guest code using hardware-assisted VMX.
         *
         * This could probably be done in a preemptible context since the preemption hook
         * will flag the necessary change in host context. However, since preemption is
         * already disabled and to avoid making assumptions about host specific code in
         * RTMpCpuId when called with preemption enabled, we'll do this while preemption is
         * disabled.
         */
        if (pVmcsInfoTo->idHostCpuState == RTMpCpuId())
        { /* likely */ }
        else
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE);

        ASMSetFlags(fEFlags);

        /*
         * We use a different VM-exit MSR-store areas for the guest and nested-guest. Hence,
         * flag that we need to update the host MSR values there. Even if we decide in the
         * future to share the VM-exit MSR-store area page between the guest and nested-guest,
         * if its content differs, we would have to update the host MSRs anyway.
         */
        pVCpu->hmr0.s.vmx.fUpdatedHostAutoMsrs = false;
    }
    else
        ASMSetFlags(fEFlags);
    return rc;
}

#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
#ifdef VBOX_STRICT

/**
 * Reads the VM-entry interruption-information field from the VMCS into the VMX
 * transient structure.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 */
DECLINLINE(void) vmxHCReadEntryIntInfoVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
    AssertRC(rc);
}


/**
 * Reads the VM-entry exception error code field from the VMCS into
 * the VMX transient structure.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 */
DECLINLINE(void) vmxHCReadEntryXcptErrorCodeVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
    AssertRC(rc);
}


/**
 * Reads the VM-entry exception error code field from the VMCS into
 * the VMX transient structure.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 */
DECLINLINE(void) vmxHCReadEntryInstrLenVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
    AssertRC(rc);
}

#endif /* VBOX_STRICT */


/**
 * Reads VMCS fields into the VMXTRANSIENT structure, slow path version.
 *
 * Don't call directly unless the it's likely that some or all of the fields
 * given in @a a_fReadMask have already been read.
 *
 * @tparam  a_fReadMask     The fields to read.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 */
template<uint32_t const a_fReadMask>
static void vmxHCReadToTransientSlow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    AssertCompile((a_fReadMask & ~(  HMVMX_READ_EXIT_QUALIFICATION
                                   | HMVMX_READ_EXIT_INSTR_LEN
                                   | HMVMX_READ_EXIT_INSTR_INFO
                                   | HMVMX_READ_IDT_VECTORING_INFO
                                   | HMVMX_READ_IDT_VECTORING_ERROR_CODE
                                   | HMVMX_READ_EXIT_INTERRUPTION_INFO
                                   | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
                                   | HMVMX_READ_GUEST_LINEAR_ADDR
                                   | HMVMX_READ_GUEST_PHYSICAL_ADDR
                                   | HMVMX_READ_GUEST_PENDING_DBG_XCPTS
                                   )) == 0);

    if ((pVmxTransient->fVmcsFieldsRead & a_fReadMask) != a_fReadMask)
    {
        uint32_t const fVmcsFieldsRead = pVmxTransient->fVmcsFieldsRead;

        if (   (a_fReadMask      & HMVMX_READ_EXIT_QUALIFICATION)
            && !(fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
        {
            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION,          &pVmxTransient->uExitQual);
            AssertRC(rc);
        }
        if (    (a_fReadMask     & HMVMX_READ_EXIT_INSTR_LEN)
            && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH,         &pVmxTransient->cbExitInstr);
            AssertRC(rc);
        }
        if (   (a_fReadMask      & HMVMX_READ_EXIT_INSTR_INFO)
            && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO,           &pVmxTransient->ExitInstrInfo.u);
            AssertRC(rc);
        }
        if (   (a_fReadMask      & HMVMX_READ_IDT_VECTORING_INFO)
            && !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO,        &pVmxTransient->uIdtVectoringInfo);
            AssertRC(rc);
        }
        if (   (a_fReadMask      & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
            && !(fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,  &pVmxTransient->uIdtVectoringErrorCode);
            AssertRC(rc);
        }
        if (   (a_fReadMask      & HMVMX_READ_EXIT_INTERRUPTION_INFO)
            && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,    &pVmxTransient->uExitIntInfo);
            AssertRC(rc);
        }
        if (    (a_fReadMask     & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
            && !(fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
            AssertRC(rc);
        }
        if (   (a_fReadMask      & HMVMX_READ_GUEST_LINEAR_ADDR)
            && !(fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
        {
            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR,           &pVmxTransient->uGuestLinearAddr);
            AssertRC(rc);
        }
        if (   (a_fReadMask      & HMVMX_READ_GUEST_PHYSICAL_ADDR)
            && !(fVmcsFieldsRead & HMVMX_READ_GUEST_PHYSICAL_ADDR))
        {
            int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,      &pVmxTransient->uGuestPhysicalAddr);
            AssertRC(rc);
        }
        if (   (a_fReadMask      & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
            && !(fVmcsFieldsRead & HMVMX_READ_GUEST_PENDING_DBG_XCPTS))
        {
            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,      &pVmxTransient->uGuestPendingDbgXcpts);
            AssertRC(rc);
        }

        pVmxTransient->fVmcsFieldsRead |= a_fReadMask;
    }
}


/**
 * Reads VMCS fields into the VMXTRANSIENT structure.
 *
 * This optimizes for the case where none of @a a_fReadMask has been read yet,
 * generating an optimized read sequences w/o any conditionals between in
 * non-strict builds.
 *
 * @tparam  a_fReadMask     The fields to read.  One or more of the
 *                          HMVMX_READ_XXX fields ORed together.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 */
template<uint32_t const a_fReadMask>
DECLINLINE(void) vmxHCReadToTransient(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    AssertCompile((a_fReadMask & ~(  HMVMX_READ_EXIT_QUALIFICATION
                                   | HMVMX_READ_EXIT_INSTR_LEN
                                   | HMVMX_READ_EXIT_INSTR_INFO
                                   | HMVMX_READ_IDT_VECTORING_INFO
                                   | HMVMX_READ_IDT_VECTORING_ERROR_CODE
                                   | HMVMX_READ_EXIT_INTERRUPTION_INFO
                                   | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
                                   | HMVMX_READ_GUEST_LINEAR_ADDR
                                   | HMVMX_READ_GUEST_PHYSICAL_ADDR
                                   | HMVMX_READ_GUEST_PENDING_DBG_XCPTS
                                   )) == 0);

    if (RT_LIKELY(!(pVmxTransient->fVmcsFieldsRead & a_fReadMask)))
    {
        if (a_fReadMask & HMVMX_READ_EXIT_QUALIFICATION)
        {
            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION,          &pVmxTransient->uExitQual);
            AssertRC(rc);
        }
        if (a_fReadMask & HMVMX_READ_EXIT_INSTR_LEN)
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH,         &pVmxTransient->cbExitInstr);
            AssertRC(rc);
        }
        if (a_fReadMask & HMVMX_READ_EXIT_INSTR_INFO)
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO,           &pVmxTransient->ExitInstrInfo.u);
            AssertRC(rc);
        }
        if (a_fReadMask & HMVMX_READ_IDT_VECTORING_INFO)
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO,        &pVmxTransient->uIdtVectoringInfo);
            AssertRC(rc);
        }
        if (a_fReadMask & HMVMX_READ_IDT_VECTORING_ERROR_CODE)
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,  &pVmxTransient->uIdtVectoringErrorCode);
            AssertRC(rc);
        }
        if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_INFO)
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,    &pVmxTransient->uExitIntInfo);
            AssertRC(rc);
        }
        if (a_fReadMask & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE)
        {
            int const rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
            AssertRC(rc);
        }
        if (a_fReadMask & HMVMX_READ_GUEST_LINEAR_ADDR)
        {
            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR,           &pVmxTransient->uGuestLinearAddr);
            AssertRC(rc);
        }
        if (a_fReadMask & HMVMX_READ_GUEST_PHYSICAL_ADDR)
        {
            int const rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,      &pVmxTransient->uGuestPhysicalAddr);
            AssertRC(rc);
        }
        if (a_fReadMask & HMVMX_READ_GUEST_PENDING_DBG_XCPTS)
        {
            int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,      &pVmxTransient->uGuestPendingDbgXcpts);
            AssertRC(rc);
        }

        pVmxTransient->fVmcsFieldsRead |= a_fReadMask;
    }
    else
    {
        STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatReadToTransientFallback);
        Log11Func(("a_fReadMask=%#x fVmcsFieldsRead=%#x => %#x - Taking inefficient code path!\n",
                   a_fReadMask, pVmxTransient->fVmcsFieldsRead, a_fReadMask & pVmxTransient->fVmcsFieldsRead));
        vmxHCReadToTransientSlow<a_fReadMask>(pVCpu, pVmxTransient);
    }
}


#ifdef HMVMX_ALWAYS_SAVE_RO_GUEST_STATE
/**
 * Reads all relevant read-only VMCS fields into the VMX transient structure.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 */
static void vmxHCReadAllRoFieldsVmcs(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_EXIT_QUALIFICATION,             &pVmxTransient->uExitQual);
    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_LENGTH,            &pVmxTransient->cbExitInstr);
    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INSTR_INFO,              &pVmxTransient->ExitInstrInfo.u);
    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_INFO,           &pVmxTransient->uIdtVectoringInfo);
    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE,     &pVmxTransient->uIdtVectoringErrorCode);
    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO,       &pVmxTransient->uExitIntInfo);
    rc    |= VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
    rc    |= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_RO_GUEST_LINEAR_ADDR,              &pVmxTransient->uGuestLinearAddr);
    rc    |= VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL,         &pVmxTransient->uGuestPhysicalAddr);
    AssertRC(rc);
    pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION
                                   |  HMVMX_READ_EXIT_INSTR_LEN
                                   |  HMVMX_READ_EXIT_INSTR_INFO
                                   |  HMVMX_READ_IDT_VECTORING_INFO
                                   |  HMVMX_READ_IDT_VECTORING_ERROR_CODE
                                   |  HMVMX_READ_EXIT_INTERRUPTION_INFO
                                   |  HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
                                   |  HMVMX_READ_GUEST_LINEAR_ADDR
                                   |  HMVMX_READ_GUEST_PHYSICAL_ADDR;
}
#endif

/**
 * Verifies that our cached values of the VMCS fields are all consistent with
 * what's actually present in the VMCS.
 *
 * @returns VBox status code.
 * @retval  VINF_SUCCESS if all our caches match their respective VMCS fields.
 * @retval  VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
 *                                            VMCS content. HMCPU error-field is
 *                                            updated, see VMX_VCI_XXX.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmcsInfo       The VMCS info. object.
 * @param   fIsNstGstVmcs   Whether this is a nested-guest VMCS.
 */
static int vmxHCCheckCachedVmcsCtls(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
{
    const char * const pcszVmcs = fIsNstGstVmcs ? "Nested-guest VMCS" : "VMCS";

    uint32_t u32Val;
    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
    AssertRC(rc);
    AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
                        ("%s entry controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32EntryCtls, u32Val),
                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_ENTRY,
                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);

    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXIT, &u32Val);
    AssertRC(rc);
    AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
                        ("%s exit controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ExitCtls, u32Val),
                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_EXIT,
                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);

    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
    AssertRC(rc);
    AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
                        ("%s pin controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32PinCtls, u32Val),
                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PIN_EXEC,
                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);

    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
    AssertRC(rc);
    AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
                        ("%s proc controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls, u32Val),
                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC,
                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);

    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
    {
        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
        AssertRC(rc);
        AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
                            ("%s proc2 controls mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32ProcCtls2, u32Val),
                            VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
                            VERR_VMX_VMCS_FIELD_CACHE_INVALID);
    }

    uint64_t u64Val;
    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TERTIARY_CTLS)
    {
        rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_PROC_EXEC3_FULL, &u64Val);
        AssertRC(rc);
        AssertMsgReturnStmt(pVmcsInfo->u64ProcCtls3 == u64Val,
                            ("%s proc3 controls mismatch: Cache=%#RX32 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64ProcCtls3, u64Val),
                            VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_PROC_EXEC3,
                            VERR_VMX_VMCS_FIELD_CACHE_INVALID);
    }

    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
    AssertRC(rc);
    AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
                        ("%s exception bitmap mismatch: Cache=%#RX32 VMCS=%#RX32\n", pcszVmcs, pVmcsInfo->u32XcptBitmap, u32Val),
                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);

    rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
    AssertRC(rc);
    AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
                        ("%s TSC offset mismatch: Cache=%#RX64 VMCS=%#RX64\n", pcszVmcs, pVmcsInfo->u64TscOffset, u64Val),
                        VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
                        VERR_VMX_VMCS_FIELD_CACHE_INVALID);

    NOREF(pcszVmcs);
    return VINF_SUCCESS;
}


/**
 * Exports the guest state with appropriate VM-entry and VM-exit controls in the
 * VMCS.
 *
 * This is typically required when the guest changes paging mode.
 *
 * @returns VBox status code.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 *
 * @remarks Requires EFER.
 * @remarks No-long-jump zone!!!
 */
static int vmxHCExportGuestEntryExitCtls(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
{
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
    {
        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
        PVMXVMCSINFO pVmcsInfo      = pVmxTransient->pVmcsInfo;

        /*
         * VM-entry controls.
         */
        {
            uint32_t       fVal = g_HmMsrs.u.vmx.EntryCtls.n.allowed0;    /* Bits set here must be set in the VMCS. */
            uint32_t const fZap = g_HmMsrs.u.vmx.EntryCtls.n.allowed1;    /* Bits cleared here must be cleared in the VMCS. */

            /*
             * Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
             * The first VT-x capable CPUs only supported the 1-setting of this bit.
             *
             * For nested-guests, this is a mandatory VM-entry control. It's also
             * required because we do not want to leak host bits to the nested-guest.
             */
            fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;

            /*
             * Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
             *
             * For nested-guests, the "IA-32e mode guest" control we initialize with what is
             * required to get the nested-guest working with hardware-assisted VMX execution.
             * It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested hypervisor
             * can skip intercepting changes to the EFER MSR. This is why it needs to be done
             * here rather than while merging the guest VMCS controls.
             */
            if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
            {
                Assert(pVCpu->cpum.GstCtx.msrEFER & MSR_K6_EFER_LME);
                fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
            }
            else
                Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));

            /*
             * If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
             *
             * For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
             * regardless of whether the nested-guest VMCS specifies it because we are free to
             * load whatever MSRs we require and we do not need to modify the guest visible copy
             * of the VM-entry MSR load area.
             */
            if (   g_fHmVmxSupportsVmcsEfer
#ifndef IN_NEM_DARWIN
                && hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient)
#endif
                )
                fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
            else
                Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));

            /*
             * The following should -not- be set (since we're not in SMM mode):
             * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
             * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
             */

            /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
             *        VMX_ENTRY_CTLS_LOAD_PAT_MSR. */

            if ((fVal & fZap) == fVal)
            { /* likely */ }
            else
            {
                Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
                          g_HmMsrs.u.vmx.EntryCtls.n.allowed0, fVal, fZap));
                VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_ENTRY;
                return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
            }

            /* Commit it to the VMCS. */
            if (pVmcsInfo->u32EntryCtls != fVal)
            {
                int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, fVal);
                AssertRC(rc);
                pVmcsInfo->u32EntryCtls = fVal;
            }
        }

        /*
         * VM-exit controls.
         */
        {
            uint32_t       fVal = g_HmMsrs.u.vmx.ExitCtls.n.allowed0;     /* Bits set here must be set in the VMCS. */
            uint32_t const fZap = g_HmMsrs.u.vmx.ExitCtls.n.allowed1;     /* Bits cleared here must be cleared in the VMCS. */

            /*
             * Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
             * supported the 1-setting of this bit.
             *
             * For nested-guests, we set the "save debug controls" as the converse
             * "load debug controls" is mandatory for nested-guests anyway.
             */
            fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;

            /*
             * Set the host long mode active (EFER.LMA) bit (which Intel calls
             * "Host address-space size") if necessary. On VM-exit, VT-x sets both the
             * host EFER.LMA and EFER.LME bit to this value. See assertion in
             * vmxHCExportHostMsrs().
             *
             * For nested-guests, we always set this bit as we do not support 32-bit
             * hosts.
             */
            fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;

#ifndef IN_NEM_DARWIN
            /*
             * If the VMCS EFER MSR fields are supported by the hardware, we use it.
             *
             * For nested-guests, we should use the "save IA32_EFER" control if we also
             * used the "load IA32_EFER" control while exporting VM-entry controls.
             */
            if (   g_fHmVmxSupportsVmcsEfer
                && hmR0VmxShouldSwapEferMsr(pVCpu, pVmxTransient))
            {
                fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
                     |  VMX_EXIT_CTLS_LOAD_EFER_MSR;
            }
#endif

            /*
             * Enable saving of the VMX-preemption timer value on VM-exit.
             * For nested-guests, currently not exposed/used.
             */
            /** @todo r=bird: Measure performance hit because of this vs. always rewriting
             *        the timer value. */
            if (VM_IS_VMX_PREEMPT_TIMER_USED(pVM))
            {
                Assert(g_HmMsrs.u.vmx.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER);
                fVal |= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
            }

            /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
            Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));

            /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
             *        VMX_EXIT_CTLS_SAVE_PAT_MSR,
             *        VMX_EXIT_CTLS_LOAD_PAT_MSR. */

            if ((fVal & fZap) == fVal)
            { /* likely */ }
            else
            {
                Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
                          g_HmMsrs.u.vmx.ExitCtls.n.allowed0, fVal, fZap));
                VCPU_2_VMXSTATE(pVCpu).u32HMError = VMX_UFC_CTRL_EXIT;
                return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
            }

            /* Commit it to the VMCS. */
            if (pVmcsInfo->u32ExitCtls != fVal)
            {
                int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXIT, fVal);
                AssertRC(rc);
                pVmcsInfo->u32ExitCtls = fVal;
            }
        }

        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
    }
    return VINF_SUCCESS;
}


/**
 * Sets the TPR threshold in the VMCS.
 *
 * @param   pVCpu               The cross context virtual CPU structure.
 * @param   pVmcsInfo           The VMCS info. object.
 * @param   u32TprThreshold     The TPR threshold (task-priority class only).
 */
DECLINLINE(void) vmxHCApicSetTprThreshold(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
{
    Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK));         /* Bits 31:4 MBZ. */
    Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
    RT_NOREF(pVmcsInfo);
    int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
    AssertRC(rc);
}


/**
 * Exports the guest APIC TPR state into the VMCS.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 *
 * @remarks No-long-jump zone!!!
 */
static void vmxHCExportGuestApicTpr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
{
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
    {
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);

        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
        if (!pVmxTransient->fIsNestedGuest)
        {
            if (   PDMHasApic(pVCpu->CTX_SUFF(pVM))
                && APICIsEnabled(pVCpu))
            {
                /*
                 * Setup TPR shadowing.
                 */
                if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
                {
                    bool    fPendingIntr  = false;
                    uint8_t u8Tpr         = 0;
                    uint8_t u8PendingIntr = 0;
                    int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
                    AssertRC(rc);

                    /*
                     * If there are interrupts pending but masked by the TPR, instruct VT-x to
                     * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
                     * priority of the pending interrupt so we can deliver the interrupt. If there
                     * are no interrupts pending, set threshold to 0 to not cause any
                     * TPR-below-threshold VM-exits.
                     */
                    uint32_t u32TprThreshold = 0;
                    if (fPendingIntr)
                    {
                        /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
                           (which is the Task-Priority Class). */
                        const uint8_t u8PendingPriority = u8PendingIntr >> 4;
                        const uint8_t u8TprPriority     = u8Tpr >> 4;
                        if (u8PendingPriority <= u8TprPriority)
                            u32TprThreshold = u8PendingPriority;
                    }

                    vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
                }
            }
        }
        /* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
    }
}


/**
 * Gets the guest interruptibility-state and updates related force-flags.
 *
 * @returns Guest's interruptibility-state.
 * @param   pVCpu           The cross context virtual CPU structure.
 *
 * @remarks No-long-jump zone!!!
 */
static uint32_t vmxHCGetGuestIntrStateAndUpdateFFs(PVMCPUCC pVCpu)
{
    uint32_t fIntrState;

    /*
     * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
     */
    if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx))
        fIntrState = 0;
    else
    {
        /* If inhibition is active, RIP should've been imported from the VMCS already. */
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);

        if (CPUMIsInInterruptShadowAfterSs(&pVCpu->cpum.GstCtx))
            fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
        else
        {
            fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;

            /* Block-by-STI must not be set when interrupts are disabled. */
            AssertStmt(pVCpu->cpum.GstCtx.eflags.Bits.u1IF, fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
        }
    }

    /*
     * Check if we should inhibit NMI delivery.
     */
    if (!CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx))
    { /* likely */ }
    else
        fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;

    /*
     * Validate.
     */
    /* We don't support block-by-SMI yet.*/
    Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));

    return fIntrState;
}


/**
 * Exports the exception intercepts required for guest execution in the VMCS.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 *
 * @remarks No-long-jump zone!!!
 */
static void vmxHCExportGuestXcptIntercepts(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
{
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_VMX_XCPT_INTERCEPTS)
    {
        /* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
        if (   !pVmxTransient->fIsNestedGuest
            &&  VCPU_2_VMXSTATE(pVCpu).fGIMTrapXcptUD)
            vmxHCAddXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
        else
            vmxHCRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);

        /* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_VMX_XCPT_INTERCEPTS);
    }
}


/**
 * Exports the guest's RIP into the guest-state area in the VMCS.
 *
 * @param   pVCpu   The cross context virtual CPU structure.
 *
 * @remarks No-long-jump zone!!!
 */
static void vmxHCExportGuestRip(PVMCPUCC pVCpu)
{
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RIP)
    {
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);

        int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
        AssertRC(rc);

        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RIP);
        Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
    }
}


/**
 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 *
 * @remarks No-long-jump zone!!!
 */
static void vmxHCExportGuestRflags(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
{
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
    {
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);

        /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits
           of RFLAGS are reserved (MBZ).  We use bits 63:24 for internal purposes, so no need
           to assert this, the CPUMX86EFLAGS/CPUMX86RFLAGS union masks these off for us.
           Use 32-bit VMWRITE. */
        uint32_t fEFlags = pVCpu->cpum.GstCtx.eflags.u;
        Assert((fEFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK);
        AssertMsg(!(fEFlags & ~(X86_EFL_LIVE_MASK | X86_EFL_RA1_MASK)), ("%#x\n", fEFlags));

#ifndef IN_NEM_DARWIN
        /*
         * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
         * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
         * can run the real-mode guest code under Virtual 8086 mode.
         */
        PVMXVMCSINFOSHARED pVmcsInfo = pVmxTransient->pVmcsInfo->pShared;
        if (pVmcsInfo->RealMode.fRealOnV86Active)
        {
            Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
            Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
            Assert(!pVmxTransient->fIsNestedGuest);
            pVmcsInfo->RealMode.Eflags.u32 = fEFlags;        /* Save the original eflags of the real-mode guest. */
            fEFlags |= X86_EFL_VM;                           /* Set the Virtual 8086 mode bit. */
            fEFlags &= ~X86_EFL_IOPL;                        /* Change IOPL to 0, otherwise certain instructions won't fault. */
        }
#else
        RT_NOREF(pVmxTransient);
#endif

        int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, fEFlags);
        AssertRC(rc);

        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
        Log4Func(("eflags=%#RX32\n", fEFlags));
    }
}


#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
/**
 * Copies the nested-guest VMCS to the shadow VMCS.
 *
 * @returns VBox status code.
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks No-long-jump zone!!!
 */
static int vmxHCCopyNstGstToShadowVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
{
    PVMCC      const pVM         = pVCpu->CTX_SUFF(pVM);
    PCVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;

    /*
     * Disable interrupts so we don't get preempted while the shadow VMCS is the
     * current VMCS, as we may try saving guest lazy MSRs.
     *
     * Strictly speaking the lazy MSRs are not in the VMCS, but I'd rather not risk
     * calling the import VMCS code which is currently performing the guest MSR reads
     * (on 64-bit hosts) and accessing the auto-load/store MSR area on 32-bit hosts
     * and the rest of the VMX leave session machinery.
     */
    RTCCUINTREG const fEFlags = ASMIntDisableFlags();

    int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
    if (RT_SUCCESS(rc))
    {
        /*
         * Copy all guest read/write VMCS fields.
         *
         * We don't check for VMWRITE failures here for performance reasons and
         * because they are not expected to fail, barring irrecoverable conditions
         * like hardware errors.
         */
        uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
        for (uint32_t i = 0; i < cShadowVmcsFields; i++)
        {
            uint64_t       u64Val;
            uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
            IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
            VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
        }

        /*
         * If the host CPU supports writing all VMCS fields, copy the guest read-only
         * VMCS fields, so the guest can VMREAD them without causing a VM-exit.
         */
        if (g_HmMsrs.u.vmx.u64Misc & VMX_MISC_VMWRITE_ALL)
        {
            uint32_t const cShadowVmcsRoFields = pVM->hmr0.s.vmx.cShadowVmcsRoFields;
            for (uint32_t i = 0; i < cShadowVmcsRoFields; i++)
            {
                uint64_t       u64Val;
                uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsRoFields[i];
                IEMReadVmxVmcsField(pVmcsNstGst, uVmcsField, &u64Val);
                VMX_VMCS_WRITE_64(pVCpu, uVmcsField, u64Val);
            }
        }

        rc  = vmxHCClearShadowVmcs(pVmcsInfo);
        rc |= hmR0VmxLoadVmcs(pVmcsInfo);
    }

    ASMSetFlags(fEFlags);
    return rc;
}


/**
 * Copies the shadow VMCS to the nested-guest VMCS.
 *
 * @returns VBox status code.
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks Called with interrupts disabled.
 */
static int vmxHCCopyShadowToNstGstVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
{
    Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
    PVMCC const     pVM         = pVCpu->CTX_SUFF(pVM);
    PVMXVVMCS const pVmcsNstGst = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;

    int rc = vmxHCLoadShadowVmcs(pVmcsInfo);
    if (RT_SUCCESS(rc))
    {
        /*
         * Copy guest read/write fields from the shadow VMCS.
         * Guest read-only fields cannot be modified, so no need to copy them.
         *
         * We don't check for VMREAD failures here for performance reasons and
         * because they are not expected to fail, barring irrecoverable conditions
         * like hardware errors.
         */
        uint32_t const cShadowVmcsFields = pVM->hmr0.s.vmx.cShadowVmcsFields;
        for (uint32_t i = 0; i < cShadowVmcsFields; i++)
        {
            uint64_t       u64Val;
            uint32_t const uVmcsField = pVM->hmr0.s.vmx.paShadowVmcsFields[i];
            VMX_VMCS_READ_64(pVCpu, uVmcsField, &u64Val);
            IEMWriteVmxVmcsField(pVmcsNstGst, uVmcsField, u64Val);
        }

        rc  = vmxHCClearShadowVmcs(pVmcsInfo);
        rc |= hmR0VmxLoadVmcs(pVmcsInfo);
    }
    return rc;
}


/**
 * Enables VMCS shadowing for the given VMCS info. object.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks No-long-jump zone!!!
 */
static void vmxHCEnableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
{
    uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
    if (!(uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING))
    {
        Assert(pVmcsInfo->HCPhysShadowVmcs != 0 && pVmcsInfo->HCPhysShadowVmcs != NIL_RTHCPHYS);
        uProcCtls2 |= VMX_PROC_CTLS2_VMCS_SHADOWING;
        int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);                            AssertRC(rc);
        rc     = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, pVmcsInfo->HCPhysShadowVmcs);  AssertRC(rc);
        pVmcsInfo->u32ProcCtls2   = uProcCtls2;
        pVmcsInfo->u64VmcsLinkPtr = pVmcsInfo->HCPhysShadowVmcs;
        Log4Func(("Enabled\n"));
    }
}


/**
 * Disables VMCS shadowing for the given VMCS info. object.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks No-long-jump zone!!!
 */
static void vmxHCDisableVmcsShadowing(PCVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
{
    /*
     * We want all VMREAD and VMWRITE instructions to cause VM-exits, so we clear the
     * VMCS shadowing control. However, VM-entry requires the shadow VMCS indicator bit
     * to match the VMCS shadowing control if the VMCS link pointer is not NIL_RTHCPHYS.
     * Hence, we must also reset the VMCS link pointer to ensure VM-entry does not fail.
     *
     * See Intel spec. 26.2.1.1 "VM-Execution Control Fields".
     * See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
     */
    uint32_t uProcCtls2 = pVmcsInfo->u32ProcCtls2;
    if (uProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
    {
        uProcCtls2 &= ~VMX_PROC_CTLS2_VMCS_SHADOWING;
        int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, uProcCtls2);                AssertRC(rc);
        rc     = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, NIL_RTHCPHYS);     AssertRC(rc);
        pVmcsInfo->u32ProcCtls2   = uProcCtls2;
        pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
        Log4Func(("Disabled\n"));
    }
}
#endif


/**
 * Exports the guest CR0 control register into the guest-state area in the VMCS.
 *
 * The guest FPU state is always pre-loaded hence we don't need to bother about
 * sharing FPU related CR0 bits between the guest and host.
 *
 * @returns VBox status code.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 *
 * @remarks No-long-jump zone!!!
 */
static int vmxHCExportGuestCR0(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
{
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR0)
    {
        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;

        uint64_t       fSetCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed0;
        uint64_t const fZapCr0 = g_HmMsrs.u.vmx.u64Cr0Fixed1;
        if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
            fSetCr0 &= ~(uint64_t)(X86_CR0_PE | X86_CR0_PG);
        else
            Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));

        if (!pVmxTransient->fIsNestedGuest)
        {
            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
            uint64_t       u64GuestCr0  = pVCpu->cpum.GstCtx.cr0;
            uint64_t const u64ShadowCr0 = u64GuestCr0;
            Assert(!RT_HI_U32(u64GuestCr0));

            /*
             * Setup VT-x's view of the guest CR0.
             */
            uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
            if (VM_IS_VMX_NESTED_PAGING(pVM))
            {
#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
                if (CPUMIsGuestPagingEnabled(pVCpu))
                {
                    /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
                    uProcCtls &= ~(  VMX_PROC_CTLS_CR3_LOAD_EXIT
                                   | VMX_PROC_CTLS_CR3_STORE_EXIT);
                }
                else
                {
                    /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
                    uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
                              |  VMX_PROC_CTLS_CR3_STORE_EXIT;
                }

                /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
                if (VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
                    uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
#endif
            }
            else
            {
                /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
                u64GuestCr0 |= X86_CR0_WP;
            }

            /*
             * Guest FPU bits.
             *
             * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
             * using CR0.TS.
             *
             * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
             * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
             */
            u64GuestCr0 |= X86_CR0_NE;

            /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
            bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);

            /*
             * Update exception intercepts.
             */
            uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
#ifndef IN_NEM_DARWIN
            if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
            {
                Assert(PDMVmmDevHeapIsEnabled(pVM));
                Assert(pVM->hm.s.vmx.pRealModeTSS);
                uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
            }
            else
#endif
            {
                /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
                uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
                if (fInterceptMF)
                    uXcptBitmap |= RT_BIT(X86_XCPT_MF);
            }

            /* Additional intercepts for debugging, define these yourself explicitly. */
#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
            uXcptBitmap |= 0
                        |  RT_BIT(X86_XCPT_BP)
                        |  RT_BIT(X86_XCPT_DE)
                        |  RT_BIT(X86_XCPT_NM)
                        |  RT_BIT(X86_XCPT_TS)
                        |  RT_BIT(X86_XCPT_UD)
                        |  RT_BIT(X86_XCPT_NP)
                        |  RT_BIT(X86_XCPT_SS)
                        |  RT_BIT(X86_XCPT_GP)
                        |  RT_BIT(X86_XCPT_PF)
                        |  RT_BIT(X86_XCPT_MF)
                        ;
#elif defined(HMVMX_ALWAYS_TRAP_PF)
            uXcptBitmap |= RT_BIT(X86_XCPT_PF);
#endif
            if (VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv)
                uXcptBitmap |= RT_BIT(X86_XCPT_GP);
            if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE)
                uXcptBitmap |= RT_BIT(X86_XCPT_DE);
            Assert(VM_IS_VMX_NESTED_PAGING(pVM) || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));

            /* Apply the hardware specified CR0 fixed bits and enable caching. */
            u64GuestCr0 |= fSetCr0;
            u64GuestCr0 &= fZapCr0;
            u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD | X86_CR0_NW);

            Assert(!RT_HI_U32(u64GuestCr0));
            Assert(u64GuestCr0 & X86_CR0_NE);

            /* Commit the CR0 and related fields to the guest VMCS. */
            int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0);               AssertRC(rc);
            rc     = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);   AssertRC(rc);
            if (uProcCtls != pVmcsInfo->u32ProcCtls)
            {
                rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
                AssertRC(rc);
            }
            if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
            {
                rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
                AssertRC(rc);
            }

            /* Update our caches. */
            pVmcsInfo->u32ProcCtls   = uProcCtls;
            pVmcsInfo->u32XcptBitmap = uXcptBitmap;

            Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
        }
        else
        {
            /*
             * With nested-guests, we may have extended the guest/host mask here since we
             * merged in the outer guest's mask. Thus, the merged mask can include more bits
             * (to read from the nested-guest CR0 read-shadow) than the nested hypervisor
             * originally supplied. We must copy those bits from the nested-guest CR0 into
             * the nested-guest CR0 read-shadow.
             */
            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
            uint64_t       u64GuestCr0  = pVCpu->cpum.GstCtx.cr0;
            uint64_t const u64ShadowCr0 = CPUMGetGuestVmxMaskedCr0(&pVCpu->cpum.GstCtx, pVmcsInfo->u64Cr0Mask);

            /* Apply the hardware specified CR0 fixed bits and enable caching. */
            u64GuestCr0 |= fSetCr0;
            u64GuestCr0 &= fZapCr0;
            u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD | X86_CR0_NW);

            Assert(!RT_HI_U32(u64GuestCr0));
            Assert(u64GuestCr0 & X86_CR0_NE);

            /* Commit the CR0 and CR0 read-shadow to the nested-guest VMCS. */
            int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR0, u64GuestCr0);               AssertRC(rc);
            rc     = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);   AssertRC(rc);

            Log4Func(("cr0=%#RX64 shadow=%#RX64 vmcs_read_shw=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr0, u64ShadowCr0,
                      pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs.u64Cr0ReadShadow.u, fSetCr0, fZapCr0));
        }

        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR0);
    }

    return VINF_SUCCESS;
}


/**
 * Exports the guest control registers (CR3, CR4) into the guest-state area
 * in the VMCS.
 *
 * @returns VBox strict status code.
 * @retval  VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
 *          without unrestricted guest access and the VMMDev is not presently
 *          mapped (e.g. EFI32).
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 *
 * @remarks No-long-jump zone!!!
 */
static VBOXSTRICTRC vmxHCExportGuestCR3AndCR4(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
{
    int rc  = VINF_SUCCESS;
    PVMCC pVM = pVCpu->CTX_SUFF(pVM);

    /*
     * Guest CR2.
     * It's always loaded in the assembler code. Nothing to do here.
     */

    /*
     * Guest CR3.
     */
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR3)
    {
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);

        if (VM_IS_VMX_NESTED_PAGING(pVM))
        {
#ifndef IN_NEM_DARWIN
            PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
            pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);

            /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
            Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
            Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
            Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));

            /* VMX_EPT_MEMTYPE_WB support is already checked in vmxHCSetupTaggedTlb(). */
            pVmcsInfo->HCPhysEPTP |= RT_BF_MAKE(VMX_BF_EPTP_MEMTYPE,          VMX_EPTP_MEMTYPE_WB)
                                  |  RT_BF_MAKE(VMX_BF_EPTP_PAGE_WALK_LENGTH, VMX_EPTP_PAGE_WALK_LENGTH_4);

            /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
            AssertMsg(   ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3      /* Bits 3:5 (EPT page walk length - 1) must be 3. */
                      && ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0,     /* Bits 7:11 MBZ. */
                         ("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
            AssertMsg(  !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01)           /* Bit 6 (EPT accessed & dirty bit). */
                      || (g_HmMsrs.u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_ACCESS_DIRTY),
                         ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));

            rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
            AssertRC(rc);
#endif

            PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
            uint64_t  u64GuestCr3 = pCtx->cr3;
            if (   VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
                || CPUMIsGuestPagingEnabledEx(pCtx))
            {
                /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
                if (CPUMIsGuestInPAEModeEx(pCtx))
                {
                    rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, pCtx->aPaePdpes[0].u);     AssertRC(rc);
                    rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, pCtx->aPaePdpes[1].u);     AssertRC(rc);
                    rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, pCtx->aPaePdpes[2].u);     AssertRC(rc);
                    rc = VMX_VMCS_WRITE_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, pCtx->aPaePdpes[3].u);     AssertRC(rc);
                }

                /*
                 * The guest's view of its CR3 is unblemished with nested paging when the
                 * guest is using paging or we have unrestricted guest execution to handle
                 * the guest when it's not using paging.
                 */
            }
#ifndef IN_NEM_DARWIN
            else
            {
                /*
                 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
                 * thinks it accesses physical memory directly, we use our identity-mapped
                 * page table to map guest-linear to guest-physical addresses. EPT takes care
                 * of translating it to host-physical addresses.
                 */
                RTGCPHYS GCPhys;
                Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);

                /* We obtain it here every time as the guest could have relocated this PCI region. */
                rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
                if (RT_SUCCESS(rc))
                { /* likely */ }
                else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
                {
                    Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
                    return VINF_EM_RESCHEDULE_REM;  /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
                }
                else
                    AssertMsgFailedReturn(("%Rrc\n",  rc), rc);

                u64GuestCr3 = GCPhys;
            }
#endif

            Log4Func(("guest_cr3=%#RX64 (GstN)\n", u64GuestCr3));
            rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, u64GuestCr3);
            AssertRC(rc);
        }
        else
        {
            Assert(!pVmxTransient->fIsNestedGuest);
            /* Non-nested paging case, just use the hypervisor's CR3. */
            RTHCPHYS const HCPhysGuestCr3 = PGMGetHyperCR3(pVCpu);

            Log4Func(("guest_cr3=%#RX64 (HstN)\n", HCPhysGuestCr3));
            rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR3, HCPhysGuestCr3);
            AssertRC(rc);
        }

        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR3);
    }

    /*
     * Guest CR4.
     * ASSUMES this is done everytime we get in from ring-3! (XCR0)
     */
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CR4)
    {
        PCPUMCTX     pCtx      = &pVCpu->cpum.GstCtx;
        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;

        uint64_t const fSetCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed0;
        uint64_t const fZapCr4 = g_HmMsrs.u.vmx.u64Cr4Fixed1;

        /*
         * With nested-guests, we may have extended the guest/host mask here (since we
         * merged in the outer guest's mask, see hmR0VmxMergeVmcsNested). This means, the
         * mask can include more bits (to read from the nested-guest CR4 read-shadow) than
         * the nested hypervisor originally supplied. Thus, we should, in essence, copy
         * those bits from the nested-guest CR4 into the nested-guest CR4 read-shadow.
         */
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
        uint64_t       u64GuestCr4  = pCtx->cr4;
        uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest
                                    ? pCtx->cr4
                                    : CPUMGetGuestVmxMaskedCr4(pCtx, pVmcsInfo->u64Cr4Mask);
        Assert(!RT_HI_U32(u64GuestCr4));

#ifndef IN_NEM_DARWIN
        /*
         * Setup VT-x's view of the guest CR4.
         *
         * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
         * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
         * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
         *
         * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
         */
        if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
        {
            Assert(pVM->hm.s.vmx.pRealModeTSS);
            Assert(PDMVmmDevHeapIsEnabled(pVM));
            u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
        }
#endif

        if (VM_IS_VMX_NESTED_PAGING(pVM))
        {
            if (   !CPUMIsGuestPagingEnabledEx(pCtx)
                && !VM_IS_VMX_UNRESTRICTED_GUEST(pVM))
            {
                /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
                u64GuestCr4 |= X86_CR4_PSE;
                /* Our identity mapping is a 32-bit page directory. */
                u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
            }
            /* else use guest CR4.*/
        }
        else
        {
            Assert(!pVmxTransient->fIsNestedGuest);

            /*
             * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
             * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
             */
            switch (VCPU_2_VMXSTATE(pVCpu).enmShadowMode)
            {
                case PGMMODE_REAL:              /* Real-mode. */
                case PGMMODE_PROTECTED:         /* Protected mode without paging. */
                case PGMMODE_32_BIT:            /* 32-bit paging. */
                {
                    u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
                    break;
                }

                case PGMMODE_PAE:               /* PAE paging. */
                case PGMMODE_PAE_NX:            /* PAE paging with NX. */
                {
                    u64GuestCr4 |= X86_CR4_PAE;
                    break;
                }

                case PGMMODE_AMD64:             /* 64-bit AMD paging (long mode). */
                case PGMMODE_AMD64_NX:          /* 64-bit AMD paging (long mode) with NX enabled. */
                {
#ifdef VBOX_WITH_64_BITS_GUESTS
                    /* For our assumption in vmxHCShouldSwapEferMsr. */
                    Assert(u64GuestCr4 & X86_CR4_PAE);
                    break;
#endif
                }
                default:
                    AssertFailed();
                    return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
            }
        }

        /* Apply the hardware specified CR4 fixed bits (mainly CR4.VMXE). */
        u64GuestCr4 |= fSetCr4;
        u64GuestCr4 &= fZapCr4;

        Assert(!RT_HI_U32(u64GuestCr4));
        Assert(u64GuestCr4 & X86_CR4_VMXE);

        /* Commit the CR4 and CR4 read-shadow to the guest VMCS. */
        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_CR4, u64GuestCr4);               AssertRC(rc);
        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);   AssertRC(rc);

#ifndef IN_NEM_DARWIN
        /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
        bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
        if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
        {
            pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
            hmR0VmxUpdateStartVmFunction(pVCpu);
        }
#endif

        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CR4);

        Log4Func(("cr4=%#RX64 shadow=%#RX64 (set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
    }
    return rc;
}


#ifdef VBOX_STRICT
/**
 * Strict function to validate segment registers.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks Will import guest CR0 on strict builds during validation of
 *          segments.
 */
static void vmxHCValidateSegmentRegs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
{
    /*
     * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
     *
     * The reason we check for attribute value 0 in this function and not just the unusable bit is
     * because vmxHCExportGuestSegReg() only updates the VMCS' copy of the value with the
     * unusable bit and doesn't change the guest-context value.
     */
    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
    vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
    if (   !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
        && (   !CPUMIsGuestInRealModeEx(pCtx)
            && !CPUMIsGuestInV86ModeEx(pCtx)))
    {
        /* Protected mode checks */
        /* CS */
        Assert(pCtx->cs.Attr.n.u1Present);
        Assert(!(pCtx->cs.Attr.u & 0xf00));
        Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
        Assert(   (pCtx->cs.u32Limit & 0xfff) == 0xfff
               || !(pCtx->cs.Attr.n.u1Granularity));
        Assert(   !(pCtx->cs.u32Limit & 0xfff00000)
               || (pCtx->cs.Attr.n.u1Granularity));
        /* CS cannot be loaded with NULL in protected mode. */
        Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
        if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
            Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
        else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
            Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
        else
            AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
        /* SS */
        Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
        Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
        if (   !(pCtx->cr0 & X86_CR0_PE)
            || pCtx->cs.Attr.n.u4Type == 3)
        {
            Assert(!pCtx->ss.Attr.n.u2Dpl);
        }
        if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
        {
            Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
            Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
            Assert(pCtx->ss.Attr.n.u1Present);
            Assert(!(pCtx->ss.Attr.u & 0xf00));
            Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
            Assert(   (pCtx->ss.u32Limit & 0xfff) == 0xfff
                   || !(pCtx->ss.Attr.n.u1Granularity));
            Assert(   !(pCtx->ss.u32Limit & 0xfff00000)
                   || (pCtx->ss.Attr.n.u1Granularity));
        }
        /* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSegReg(). */
        if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
        {
            Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
            Assert(pCtx->ds.Attr.n.u1Present);
            Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
            Assert(!(pCtx->ds.Attr.u & 0xf00));
            Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
            Assert(   (pCtx->ds.u32Limit & 0xfff) == 0xfff
                   || !(pCtx->ds.Attr.n.u1Granularity));
            Assert(   !(pCtx->ds.u32Limit & 0xfff00000)
                   || (pCtx->ds.Attr.n.u1Granularity));
            Assert(   !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
                   || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
        }
        if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
        {
            Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
            Assert(pCtx->es.Attr.n.u1Present);
            Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
            Assert(!(pCtx->es.Attr.u & 0xf00));
            Assert(!(pCtx->es.Attr.u & 0xfffe0000));
            Assert(   (pCtx->es.u32Limit & 0xfff) == 0xfff
                   || !(pCtx->es.Attr.n.u1Granularity));
            Assert(   !(pCtx->es.u32Limit & 0xfff00000)
                   || (pCtx->es.Attr.n.u1Granularity));
            Assert(   !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
                   || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
        }
        if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
        {
            Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
            Assert(pCtx->fs.Attr.n.u1Present);
            Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
            Assert(!(pCtx->fs.Attr.u & 0xf00));
            Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
            Assert(   (pCtx->fs.u32Limit & 0xfff) == 0xfff
                   || !(pCtx->fs.Attr.n.u1Granularity));
            Assert(   !(pCtx->fs.u32Limit & 0xfff00000)
                   || (pCtx->fs.Attr.n.u1Granularity));
            Assert(   !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
                   || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
        }
        if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
        {
            Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
            Assert(pCtx->gs.Attr.n.u1Present);
            Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
            Assert(!(pCtx->gs.Attr.u & 0xf00));
            Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
            Assert(   (pCtx->gs.u32Limit & 0xfff) == 0xfff
                   || !(pCtx->gs.Attr.n.u1Granularity));
            Assert(   !(pCtx->gs.u32Limit & 0xfff00000)
                   || (pCtx->gs.Attr.n.u1Granularity));
            Assert(   !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
                   || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
        }
        /* 64-bit capable CPUs. */
        Assert(!RT_HI_U32(pCtx->cs.u64Base));
        Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
        Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
        Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
    }
    else if (   CPUMIsGuestInV86ModeEx(pCtx)
             || (   CPUMIsGuestInRealModeEx(pCtx)
                 && !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)))
    {
        /* Real and v86 mode checks. */
        /* vmxHCExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
        uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
#ifndef IN_NEM_DARWIN
        if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
        {
            u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
            u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
        }
        else
#endif
        {
            u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
            u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
        }

        /* CS */
        AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
        Assert(pCtx->cs.u32Limit == 0xffff);
        AssertMsg(u32CSAttr == 0xf3, ("cs=%#x %#x ", pCtx->cs.Sel, u32CSAttr));
        /* SS */
        Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
        Assert(pCtx->ss.u32Limit == 0xffff);
        Assert(u32SSAttr == 0xf3);
        /* DS */
        Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
        Assert(pCtx->ds.u32Limit == 0xffff);
        Assert(u32DSAttr == 0xf3);
        /* ES */
        Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
        Assert(pCtx->es.u32Limit == 0xffff);
        Assert(u32ESAttr == 0xf3);
        /* FS */
        Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
        Assert(pCtx->fs.u32Limit == 0xffff);
        Assert(u32FSAttr == 0xf3);
        /* GS */
        Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
        Assert(pCtx->gs.u32Limit == 0xffff);
        Assert(u32GSAttr == 0xf3);
        /* 64-bit capable CPUs. */
        Assert(!RT_HI_U32(pCtx->cs.u64Base));
        Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
        Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
        Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
    }
}
#endif /* VBOX_STRICT */


/**
 * Exports a guest segment register into the guest-state area in the VMCS.
 *
 * @returns VBox status code.
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 * @param   iSegReg     The segment register number (X86_SREG_XXX).
 * @param   pSelReg     Pointer to the segment selector.
 *
 * @remarks No-long-jump zone!!!
 */
static int vmxHCExportGuestSegReg(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t iSegReg, PCCPUMSELREG pSelReg)
{
    Assert(iSegReg < X86_SREG_COUNT);

    uint32_t u32Access = pSelReg->Attr.u;
#ifndef IN_NEM_DARWIN
    if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
#endif
    {
        /*
         * The way to differentiate between whether this is really a null selector or was just
         * a selector loaded with 0 in real-mode is using the segment attributes. A selector
         * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
         * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
         * NULL selectors loaded in protected-mode have their attribute as 0.
         */
        if (u32Access)
        { }
        else
            u32Access = X86DESCATTR_UNUSABLE;
    }
#ifndef IN_NEM_DARWIN
    else
    {
        /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
        u32Access = 0xf3;
        Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
        Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
        RT_NOREF_PV(pVCpu);
    }
#else
    RT_NOREF(pVmcsInfo);
#endif

    /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
    AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
              ("Access bit not set for usable segment. %.2s sel=%#x attr %#x\n", "ESCSSSDSFSGS" + iSegReg * 2, pSelReg, pSelReg->Attr.u));

    /*
     * Commit it to the VMCS.
     */
    int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(iSegReg),           pSelReg->Sel);      AssertRC(rc);
    rc     = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(iSegReg),         pSelReg->u32Limit); AssertRC(rc);
    rc     = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(iSegReg),            pSelReg->u64Base);  AssertRC(rc);
    rc     = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(iSegReg), u32Access);         AssertRC(rc);
    return VINF_SUCCESS;
}


/**
 * Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
 * area in the VMCS.
 *
 * @returns VBox status code.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 *
 * @remarks Will import guest CR0 on strict builds during validation of
 *          segments.
 * @remarks No-long-jump zone!!!
 */
static int vmxHCExportGuestSegRegsXdtr(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient)
{
    int                 rc              = VERR_INTERNAL_ERROR_5;
#ifndef IN_NEM_DARWIN
    PVMCC               pVM             = pVCpu->CTX_SUFF(pVM);
#endif
    PCCPUMCTX           pCtx            = &pVCpu->cpum.GstCtx;
    PVMXVMCSINFO        pVmcsInfo       = pVmxTransient->pVmcsInfo;
#ifndef IN_NEM_DARWIN
    PVMXVMCSINFOSHARED  pVmcsInfoShared = pVmcsInfo->pShared;
#endif

    /*
     * Guest Segment registers: CS, SS, DS, ES, FS, GS.
     */
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
    {
        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_CS)
        {
            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
#ifndef IN_NEM_DARWIN
            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
                pVmcsInfoShared->RealMode.AttrCS.u = pCtx->cs.Attr.u;
#endif
            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
            AssertRC(rc);
            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_CS);
        }

        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_SS)
        {
            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
#ifndef IN_NEM_DARWIN
            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
                pVmcsInfoShared->RealMode.AttrSS.u = pCtx->ss.Attr.u;
#endif
            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
            AssertRC(rc);
            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_SS);
        }

        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_DS)
        {
            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
#ifndef IN_NEM_DARWIN
            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
                pVmcsInfoShared->RealMode.AttrDS.u = pCtx->ds.Attr.u;
#endif
            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
            AssertRC(rc);
            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_DS);
        }

        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_ES)
        {
            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
#ifndef IN_NEM_DARWIN
            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
                pVmcsInfoShared->RealMode.AttrES.u = pCtx->es.Attr.u;
#endif
            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
            AssertRC(rc);
            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_ES);
        }

        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_FS)
        {
            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
#ifndef IN_NEM_DARWIN
            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
                pVmcsInfoShared->RealMode.AttrFS.u = pCtx->fs.Attr.u;
#endif
            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
            AssertRC(rc);
            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_FS);
        }

        if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GS)
        {
            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
#ifndef IN_NEM_DARWIN
            if (pVmcsInfoShared->RealMode.fRealOnV86Active)
                pVmcsInfoShared->RealMode.AttrGS.u = pCtx->gs.Attr.u;
#endif
            rc = vmxHCExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
            AssertRC(rc);
            ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GS);
        }

#ifdef VBOX_STRICT
        vmxHCValidateSegmentRegs(pVCpu, pVmcsInfo);
#endif
        Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
                  pCtx->cs.Attr.u));
    }

    /*
     * Guest TR.
     */
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_TR)
    {
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);

        /*
         * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
         * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
         * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
         */
        uint16_t u16Sel;
        uint32_t u32Limit;
        uint64_t u64Base;
        uint32_t u32AccessRights;
#ifndef IN_NEM_DARWIN
        if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
#endif
        {
            u16Sel          = pCtx->tr.Sel;
            u32Limit        = pCtx->tr.u32Limit;
            u64Base         = pCtx->tr.u64Base;
            u32AccessRights = pCtx->tr.Attr.u;
        }
#ifndef IN_NEM_DARWIN
        else
        {
            Assert(!pVmxTransient->fIsNestedGuest);
            Assert(pVM->hm.s.vmx.pRealModeTSS);
            Assert(PDMVmmDevHeapIsEnabled(pVM));    /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */

            /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
            RTGCPHYS GCPhys;
            rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
            AssertRCReturn(rc, rc);

            X86DESCATTR DescAttr;
            DescAttr.u           = 0;
            DescAttr.n.u1Present = 1;
            DescAttr.n.u4Type    = X86_SEL_TYPE_SYS_386_TSS_BUSY;

            u16Sel          = 0;
            u32Limit        = HM_VTX_TSS_SIZE;
            u64Base         = GCPhys;
            u32AccessRights = DescAttr.u;
        }
#endif

        /* Validate. */
        Assert(!(u16Sel & RT_BIT(2)));
        AssertMsg(   (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
                  || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
        AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
        Assert(!(u32AccessRights & RT_BIT(4)));                 /* System MBZ.*/
        Assert(u32AccessRights & RT_BIT(7));                    /* Present MB1.*/
        Assert(!(u32AccessRights & 0xf00));                     /* 11:8 MBZ. */
        Assert(!(u32AccessRights & 0xfffe0000));                /* 31:17 MBZ. */
        Assert(   (u32Limit & 0xfff) == 0xfff
               || !(u32AccessRights & RT_BIT(15)));             /* Granularity MBZ. */
        Assert(   !(pCtx->tr.u32Limit & 0xfff00000)
               || (u32AccessRights & RT_BIT(15)));              /* Granularity MB1. */

        rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL,           u16Sel);             AssertRC(rc);
        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT,         u32Limit);           AssertRC(rc);
        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);    AssertRC(rc);
        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE,            u64Base);            AssertRC(rc);

        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_TR);
        Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
    }

    /*
     * Guest GDTR.
     */
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_GDTR)
    {
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);

        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);     AssertRC(rc);
        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE,  pCtx->gdtr.pGdt);        AssertRC(rc);

        /* Validate. */
        Assert(!(pCtx->gdtr.cbGdt & 0xffff0000));          /* Bits 31:16 MBZ. */

        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
        Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
    }

    /*
     * Guest LDTR.
     */
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_LDTR)
    {
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);

        /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
        uint32_t u32Access;
        if (   !pVmxTransient->fIsNestedGuest
            && !pCtx->ldtr.Attr.u)
            u32Access = X86DESCATTR_UNUSABLE;
        else
            u32Access = pCtx->ldtr.Attr.u;

        rc = VMX_VMCS_WRITE_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL,           pCtx->ldtr.Sel);       AssertRC(rc);
        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT,         pCtx->ldtr.u32Limit);  AssertRC(rc);
        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);            AssertRC(rc);
        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE,            pCtx->ldtr.u64Base);   AssertRC(rc);

        /* Validate. */
        if (!(u32Access & X86DESCATTR_UNUSABLE))
        {
            Assert(!(pCtx->ldtr.Sel & RT_BIT(2)));              /* TI MBZ. */
            Assert(pCtx->ldtr.Attr.n.u4Type == 2);              /* Type MB2 (LDT). */
            Assert(!pCtx->ldtr.Attr.n.u1DescType);              /* System MBZ. */
            Assert(pCtx->ldtr.Attr.n.u1Present == 1);           /* Present MB1. */
            Assert(!pCtx->ldtr.Attr.n.u4LimitHigh);             /* 11:8 MBZ. */
            Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000));          /* 31:17 MBZ. */
            Assert(   (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
                   || !pCtx->ldtr.Attr.n.u1Granularity);        /* Granularity MBZ. */
            Assert(   !(pCtx->ldtr.u32Limit & 0xfff00000)
                   || pCtx->ldtr.Attr.n.u1Granularity);         /* Granularity MB1. */
        }

        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
        Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
    }

    /*
     * Guest IDTR.
     */
    if (ASMAtomicUoReadU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged) & HM_CHANGED_GUEST_IDTR)
    {
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);

        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);     AssertRC(rc);
        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE,  pCtx->idtr.pIdt);        AssertRC(rc);

        /* Validate. */
        Assert(!(pCtx->idtr.cbIdt & 0xffff0000));          /* Bits 31:16 MBZ. */

        ASMAtomicUoAndU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
        Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
    }

    return VINF_SUCCESS;
}


/**
 * Gets the IEM exception flags for the specified vector and IDT vectoring /
 * VM-exit interruption info type.
 *
 * @returns The IEM exception flags.
 * @param   uVector         The event vector.
 * @param   uVmxEventType   The VMX event type.
 *
 * @remarks This function currently only constructs flags required for
 *          IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
 *          and CR2 aspects of an exception are not included).
 */
static uint32_t vmxHCGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
{
    uint32_t fIemXcptFlags;
    switch (uVmxEventType)
    {
        case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
        case VMX_IDT_VECTORING_INFO_TYPE_NMI:
            fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
            break;

        case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
            fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
            break;

        case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
            fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
            break;

        case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
        {
            fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
            if (uVector == X86_XCPT_BP)
                fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
            else if (uVector == X86_XCPT_OF)
                fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
            else
            {
                fIemXcptFlags = 0;
                AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
            }
            break;
        }

        case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
            fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
            break;

        default:
            fIemXcptFlags = 0;
            AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
            break;
    }
    return fIemXcptFlags;
}


/**
 * Sets an event as a pending event to be injected into the guest.
 *
 * @param   pVCpu               The cross context virtual CPU structure.
 * @param   u32IntInfo          The VM-entry interruption-information field.
 * @param   cbInstr             The VM-entry instruction length in bytes (for
 *                              software interrupts, exceptions and privileged
 *                              software exceptions).
 * @param   u32ErrCode          The VM-entry exception error code.
 * @param   GCPtrFaultAddress   The fault-address (CR2) in case it's a
 *                              page-fault.
 */
DECLINLINE(void) vmxHCSetPendingEvent(PVMCPUCC pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
                                      RTGCUINTPTR GCPtrFaultAddress)
{
    Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
    VCPU_2_VMXSTATE(pVCpu).Event.fPending          = true;
    VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo        = u32IntInfo;
    VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode        = u32ErrCode;
    VCPU_2_VMXSTATE(pVCpu).Event.cbInstr           = cbInstr;
    VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress = GCPtrFaultAddress;
}


/**
 * Sets an external interrupt as pending-for-injection into the VM.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   u8Interrupt     The external interrupt vector.
 */
DECLINLINE(void) vmxHCSetPendingExtInt(PVMCPUCC pVCpu, uint8_t u8Interrupt)
{
    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR,          u8Interrupt)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
}


/**
 * Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
 *
 * @param   pVCpu   The cross context virtual CPU structure.
 */
DECLINLINE(void) vmxHCSetPendingXcptNmi(PVMCPUCC pVCpu)
{
    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_NMI)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_ENTRY_INT_INFO_TYPE_NMI)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
}


/**
 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
 *
 * @param   pVCpu   The cross context virtual CPU structure.
 */
DECLINLINE(void) vmxHCSetPendingXcptDF(PVMCPUCC pVCpu)
{
    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_DF)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
}


/**
 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
 *
 * @param   pVCpu   The cross context virtual CPU structure.
 */
DECLINLINE(void) vmxHCSetPendingXcptUD(PVMCPUCC pVCpu)
{
    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_UD)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
}


/**
 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
 *
 * @param   pVCpu   The cross context virtual CPU structure.
 */
DECLINLINE(void) vmxHCSetPendingXcptDB(PVMCPUCC pVCpu)
{
    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_DB)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
}


#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
/**
 * Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   u32ErrCode  The error code for the general-protection exception.
 */
DECLINLINE(void) vmxHCSetPendingXcptGP(PVMCPUCC pVCpu, uint32_t u32ErrCode)
{
    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_GP)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
}


/**
 * Sets a stack (\#SS) exception as pending-for-injection into the VM.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   u32ErrCode  The error code for the stack exception.
 */
DECLINLINE(void) vmxHCSetPendingXcptSS(PVMCPUCC pVCpu, uint32_t u32ErrCode)
{
    uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR,         X86_XCPT_SS)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,           VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
                              | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1);
    vmxHCSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrCode, 0 /* GCPtrFaultAddress */);
}
#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */


/**
 * Fixes up attributes for the specified segment register.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pSelReg     The segment register that needs fixing.
 * @param   pszRegName  The register name (for logging and assertions).
 */
static void vmxHCFixUnusableSegRegAttr(PVMCPUCC pVCpu, PCPUMSELREG pSelReg, const char *pszRegName)
{
    Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);

    /*
     * If VT-x marks the segment as unusable, most other bits remain undefined:
     *   - For CS the L, D and G bits have meaning.
     *   - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
     *   - For the remaining data segments no bits are defined.
     *
     * The present bit and the unusable bit has been observed to be set at the
     * same time (the selector was supposed to be invalid as we started executing
     * a V8086 interrupt in ring-0).
     *
     * What should be important for the rest of the VBox code, is that the P bit is
     * cleared.  Some of the other VBox code recognizes the unusable bit, but
     * AMD-V certainly don't, and REM doesn't really either.  So, to be on the
     * safe side here, we'll strip off P and other bits we don't care about.  If
     * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
     *
     * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
     */
#ifdef VBOX_STRICT
    uint32_t const uAttr = pSelReg->Attr.u;
#endif

    /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
    pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L    | X86DESCATTR_D  | X86DESCATTR_G
                     | X86DESCATTR_DPL      | X86DESCATTR_TYPE | X86DESCATTR_DT;

#ifdef VBOX_STRICT
# ifndef IN_NEM_DARWIN
    VMMRZCallRing3Disable(pVCpu);
# endif
    Log4Func(("Unusable %s: sel=%#x attr=%#x -> %#x\n", pszRegName, pSelReg->Sel, uAttr, pSelReg->Attr.u));
# ifdef DEBUG_bird
    AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
              ("%s: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
               pszRegName, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
# endif
# ifndef IN_NEM_DARWIN
    VMMRZCallRing3Enable(pVCpu);
# endif
    NOREF(uAttr);
#endif
    RT_NOREF2(pVCpu, pszRegName);
}


/**
 * Imports a guest segment register from the current VMCS into the guest-CPU
 * context.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @tparam  a_iSegReg   The segment register number (X86_SREG_XXX).
 *
 * @remarks Called with interrupts and/or preemption disabled.
 */
template<uint32_t const a_iSegReg>
DECLINLINE(void) vmxHCImportGuestSegReg(PVMCPUCC pVCpu)
{
    AssertCompile(a_iSegReg < X86_SREG_COUNT);
    /* Check that the macros we depend upon here and in the export parenter function works: */
#define MY_SEG_VMCS_FIELD(a_FieldPrefix, a_FieldSuff) \
        (  a_iSegReg == X86_SREG_ES ? a_FieldPrefix ## ES ## a_FieldSuff \
         : a_iSegReg == X86_SREG_CS ? a_FieldPrefix ## CS ## a_FieldSuff \
         : a_iSegReg == X86_SREG_SS ? a_FieldPrefix ## SS ## a_FieldSuff \
         : a_iSegReg == X86_SREG_DS ? a_FieldPrefix ## DS ## a_FieldSuff \
         : a_iSegReg == X86_SREG_FS ? a_FieldPrefix ## FS ## a_FieldSuff \
         : a_iSegReg == X86_SREG_GS ? a_FieldPrefix ## GS ## a_FieldSuff : 0)
    AssertCompile(VMX_VMCS_GUEST_SEG_BASE(a_iSegReg)             == MY_SEG_VMCS_FIELD(VMX_VMCS_GUEST_,_BASE));
    AssertCompile(VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg)            == MY_SEG_VMCS_FIELD(VMX_VMCS16_GUEST_,_SEL));
    AssertCompile(VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg)          == MY_SEG_VMCS_FIELD(VMX_VMCS32_GUEST_,_LIMIT));
    AssertCompile(VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg)  == MY_SEG_VMCS_FIELD(VMX_VMCS32_GUEST_,_ACCESS_RIGHTS));

    PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[a_iSegReg];

    uint16_t u16Sel;
    int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_SEG_SEL(a_iSegReg), &u16Sel);   AssertRC(rc);
    pSelReg->Sel      = u16Sel;
    pSelReg->ValidSel = u16Sel;

    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_LIMIT(a_iSegReg), &pSelReg->u32Limit); AssertRC(rc);
    rc     = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SEG_BASE(a_iSegReg), &pSelReg->u64Base);     AssertRC(rc);

    uint32_t u32Attr;
    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SEG_ACCESS_RIGHTS(a_iSegReg), &u32Attr);   AssertRC(rc);
    pSelReg->Attr.u   = u32Attr;
    if (u32Attr & X86DESCATTR_UNUSABLE)
        vmxHCFixUnusableSegRegAttr(pVCpu, pSelReg, "ES\0CS\0SS\0DS\0FS\0GS" + a_iSegReg * 3);

    pSelReg->fFlags   = CPUMSELREG_FLAGS_VALID;
}


/**
 * Imports the guest LDTR from the current VMCS into the guest-CPU context.
 *
 * @param   pVCpu   The cross context virtual CPU structure.
 *
 * @remarks Called with interrupts and/or preemption disabled.
 */
DECLINLINE(void) vmxHCImportGuestLdtr(PVMCPUCC pVCpu)
{
    uint16_t u16Sel;
    uint64_t u64Base;
    uint32_t u32Limit, u32Attr;
    int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_LDTR_SEL,           &u16Sel);       AssertRC(rc);
    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_LIMIT,         &u32Limit);     AssertRC(rc);
    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);      AssertRC(rc);
    rc     = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_LDTR_BASE,            &u64Base);      AssertRC(rc);

    pVCpu->cpum.GstCtx.ldtr.Sel      = u16Sel;
    pVCpu->cpum.GstCtx.ldtr.ValidSel = u16Sel;
    pVCpu->cpum.GstCtx.ldtr.fFlags   = CPUMSELREG_FLAGS_VALID;
    pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
    pVCpu->cpum.GstCtx.ldtr.u64Base  = u64Base;
    pVCpu->cpum.GstCtx.ldtr.Attr.u   = u32Attr;
    if (u32Attr & X86DESCATTR_UNUSABLE)
        vmxHCFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, "LDTR");
}


/**
 * Imports the guest TR from the current VMCS into the guest-CPU context.
 *
 * @param   pVCpu   The cross context virtual CPU structure.
 *
 * @remarks Called with interrupts and/or preemption disabled.
 */
DECLINLINE(void) vmxHCImportGuestTr(PVMCPUCC pVCpu)
{
    uint16_t u16Sel;
    uint64_t u64Base;
    uint32_t u32Limit, u32Attr;
    int rc = VMX_VMCS_READ_16(pVCpu, VMX_VMCS16_GUEST_TR_SEL,           &u16Sel);     AssertRC(rc);
    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_LIMIT,         &u32Limit);   AssertRC(rc);
    rc     = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);    AssertRC(rc);
    rc     = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_TR_BASE,            &u64Base);    AssertRC(rc);

    pVCpu->cpum.GstCtx.tr.Sel      = u16Sel;
    pVCpu->cpum.GstCtx.tr.ValidSel = u16Sel;
    pVCpu->cpum.GstCtx.tr.fFlags   = CPUMSELREG_FLAGS_VALID;
    pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
    pVCpu->cpum.GstCtx.tr.u64Base  = u64Base;
    pVCpu->cpum.GstCtx.tr.Attr.u   = u32Attr;
    /* TR is the only selector that can never be unusable. */
    Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
}


/**
 * Core: Imports the guest RIP from the VMCS back into the guest-CPU context.
 *
 * @returns The RIP value.
 * @param   pVCpu               The cross context virtual CPU structure.
 *
 * @remarks Called with interrupts and/or preemption disabled, should not assert!
 * @remarks Do -not- call this function directly!
 */
DECL_FORCE_INLINE(uint64_t) vmxHCImportGuestCoreRip(PVMCPUCC pVCpu)
{
    uint64_t u64Val;
    int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
    AssertRC(rc);

    pVCpu->cpum.GstCtx.rip = u64Val;

    return u64Val;
}


/**
 * Imports the guest RIP from the VMCS back into the guest-CPU context.
 *
 * @param   pVCpu   The cross context virtual CPU structure.
 *
 * @remarks Called with interrupts and/or preemption disabled, should not assert!
 * @remarks Do -not- call this function directly, use vmxHCImportGuestState()
 *          instead!!!
 */
DECLINLINE(void) vmxHCImportGuestRip(PVMCPUCC pVCpu)
{
    if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP)
    {
        EMHistoryUpdatePC(pVCpu, vmxHCImportGuestCoreRip(pVCpu), false);
        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
    }
}


/**
 * Core: Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks Called with interrupts and/or preemption disabled, should not assert!
 * @remarks Do -not- call this function directly!
 */
DECL_FORCE_INLINE(void) vmxHCImportGuestCoreRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
{
    uint64_t fRFlags;
    int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &fRFlags);
    AssertRC(rc);

    Assert((fRFlags & X86_EFL_RA1_MASK) == X86_EFL_RA1_MASK);
    Assert((fRFlags & ~(uint64_t)(X86_EFL_1 | X86_EFL_LIVE_MASK)) == 0);

    pVCpu->cpum.GstCtx.rflags.u = fRFlags;
#ifndef IN_NEM_DARWIN
    PCVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
    if (!pVmcsInfoShared->RealMode.fRealOnV86Active)
    { /* mostly likely */ }
    else
    {
        pVCpu->cpum.GstCtx.eflags.Bits.u1VM   = 0;
        pVCpu->cpum.GstCtx.eflags.Bits.u2IOPL = pVmcsInfoShared->RealMode.Eflags.Bits.u2IOPL;
    }
#else
    RT_NOREF(pVmcsInfo);
#endif
}


/**
 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks Called with interrupts and/or preemption disabled, should not assert!
 * @remarks Do -not- call this function directly, use vmxHCImportGuestState()
 *          instead!!!
 */
DECLINLINE(void) vmxHCImportGuestRFlags(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
{
    if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RFLAGS)
    {
        vmxHCImportGuestCoreRFlags(pVCpu, pVmcsInfo);
        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
    }
}


/**
 * Worker for vmxHCImportGuestIntrState that handles the case where any of the
 * relevant VMX_VMCS32_GUEST_INT_STATE bits are set.
 */
DECL_NO_INLINE(static,void) vmxHCImportGuestIntrStateSlow(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo, uint32_t fGstIntState)
{
    /*
     * We must import RIP here to set our EM interrupt-inhibited state.
     * We also import RFLAGS as our code that evaluates pending interrupts
     * before VM-entry requires it.
     */
    vmxHCImportGuestRip(pVCpu);
    vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);

    CPUMUpdateInterruptShadowSsStiEx(&pVCpu->cpum.GstCtx,
                                     RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
                                     RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
                                     pVCpu->cpum.GstCtx.rip);
    CPUMUpdateInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx, RT_BOOL(fGstIntState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
}


/**
 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
 * context.
 *
 * @note    May import RIP and RFLAGS if interrupt or NMI are blocked.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
 *          do not log!
 * @remarks Do -not- call this function directly, use vmxHCImportGuestState()
 *          instead!!!
 */
DECLINLINE(void) vmxHCImportGuestIntrState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
{
    uint32_t u32Val;
    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32Val);    AssertRC(rc);
    if (!u32Val)
    {
        CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);
        CPUMClearInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
    }
    else
        vmxHCImportGuestIntrStateSlow(pVCpu, pVmcsInfo, u32Val);
}


/**
 * Worker for VMXR0ImportStateOnDemand.
 *
 * @returns VBox status code.
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 * @param   fWhat       What to import, CPUMCTX_EXTRN_XXX.
 */
static int vmxHCImportGuestStateEx(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
{
    int      rc   = VINF_SUCCESS;
    PVMCC    pVM  = pVCpu->CTX_SUFF(pVM);
    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
    uint32_t u32Val;

    /*
     * Note! This is hack to workaround a mysterious BSOD observed with release builds
     *       on Windows 10 64-bit hosts. Profile and debug builds are not affected and
     *       neither are other host platforms.
     *
     *       Committing this temporarily as it prevents BSOD.
     *
     * Update: This is very likely a compiler optimization bug, see @bugref{9180}.
     */
#ifdef RT_OS_WINDOWS
    if (pVM == 0 || pVM == (void *)(uintptr_t)-1)
        return VERR_HM_IPE_1;
#endif

    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);

#ifndef IN_NEM_DARWIN
    /*
     * We disable interrupts to make the updating of the state and in particular
     * the fExtrn modification atomic wrt to preemption hooks.
     */
    RTCCUINTREG const fEFlags = ASMIntDisableFlags();
#endif

    fWhat &= pCtx->fExtrn;
    if (fWhat)
    {
        do
        {
            if (fWhat & CPUMCTX_EXTRN_RIP)
                vmxHCImportGuestRip(pVCpu);

            if (fWhat & CPUMCTX_EXTRN_RFLAGS)
                vmxHCImportGuestRFlags(pVCpu, pVmcsInfo);

            /* Note! vmxHCImportGuestIntrState may also include RIP and RFLAGS and update fExtrn. */
            if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
                vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);

            if (fWhat & CPUMCTX_EXTRN_RSP)
            {
                rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pCtx->rsp);
                AssertRC(rc);
            }

            if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
            {
                PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
#ifndef IN_NEM_DARWIN
                bool const fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
#else
                bool const fRealOnV86Active = false; /* HV supports only unrestricted guest execution. */
#endif
                if (fWhat & CPUMCTX_EXTRN_CS)
                {
                    vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
                    vmxHCImportGuestRip(pVCpu); /** @todo WTF? */
                    if (fRealOnV86Active)
                        pCtx->cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
                    EMHistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
                }
                if (fWhat & CPUMCTX_EXTRN_SS)
                {
                    vmxHCImportGuestSegReg<X86_SREG_SS>(pVCpu);
                    if (fRealOnV86Active)
                        pCtx->ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
                }
                if (fWhat & CPUMCTX_EXTRN_DS)
                {
                    vmxHCImportGuestSegReg<X86_SREG_DS>(pVCpu);
                    if (fRealOnV86Active)
                        pCtx->ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
                }
                if (fWhat & CPUMCTX_EXTRN_ES)
                {
                    vmxHCImportGuestSegReg<X86_SREG_ES>(pVCpu);
                    if (fRealOnV86Active)
                        pCtx->es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
                }
                if (fWhat & CPUMCTX_EXTRN_FS)
                {
                    vmxHCImportGuestSegReg<X86_SREG_FS>(pVCpu);
                    if (fRealOnV86Active)
                        pCtx->fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
                }
                if (fWhat & CPUMCTX_EXTRN_GS)
                {
                    vmxHCImportGuestSegReg<X86_SREG_GS>(pVCpu);
                    if (fRealOnV86Active)
                        pCtx->gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
                }
            }

            if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
            {
                if (fWhat & CPUMCTX_EXTRN_LDTR)
                    vmxHCImportGuestLdtr(pVCpu);

                if (fWhat & CPUMCTX_EXTRN_GDTR)
                {
                    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE,    &pCtx->gdtr.pGdt);  AssertRC(rc);
                    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);           AssertRC(rc);
                    pCtx->gdtr.cbGdt = u32Val;
                }

                /* Guest IDTR. */
                if (fWhat & CPUMCTX_EXTRN_IDTR)
                {
                    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE,    &pCtx->idtr.pIdt);  AssertRC(rc);
                    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);           AssertRC(rc);
                    pCtx->idtr.cbIdt = u32Val;
                }

                /* Guest TR. */
                if (fWhat & CPUMCTX_EXTRN_TR)
                {
#ifndef IN_NEM_DARWIN
                    /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
                       don't need to import that one. */
                    if (!pVmcsInfo->pShared->RealMode.fRealOnV86Active)
#endif
                        vmxHCImportGuestTr(pVCpu);
                }
            }

            if (fWhat & CPUMCTX_EXTRN_DR7)
            {
#ifndef IN_NEM_DARWIN
                if (!pVCpu->hmr0.s.fUsingHyperDR7)
#endif
                {
                    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pCtx->dr[7]);
                    AssertRC(rc);
                }
            }

            if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
            {
                rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP,  &pCtx->SysEnter.eip);  AssertRC(rc);
                rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP,  &pCtx->SysEnter.esp);  AssertRC(rc);
                rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);              AssertRC(rc);
                pCtx->SysEnter.cs = u32Val;
            }

#ifndef IN_NEM_DARWIN
            if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
            {
                if (   pVM->hmr0.s.fAllow64BitGuests
                    && (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
                    pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
            }

            if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
            {
                if (   pVM->hmr0.s.fAllow64BitGuests
                    && (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
                {
                    pCtx->msrLSTAR  = ASMRdMsr(MSR_K8_LSTAR);
                    pCtx->msrSTAR   = ASMRdMsr(MSR_K6_STAR);
                    pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
                }
            }

            if (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
            {
                PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
                PCVMXAUTOMSR       pMsrs           = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
                uint32_t const     cMsrs           = pVmcsInfo->cExitMsrStore;
                Assert(pMsrs);
                Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
                Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE);
                for (uint32_t i = 0; i < cMsrs; i++)
                {
                    uint32_t const idMsr = pMsrs[i].u32Msr;
                    switch (idMsr)
                    {
                        case MSR_K8_TSC_AUX:        CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value);     break;
                        case MSR_IA32_SPEC_CTRL:    CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value);   break;
                        case MSR_K6_EFER:           /* Can't be changed without causing a VM-exit */  break;
                        default:
                        {
                            uint32_t idxLbrMsr;
                            if (VM_IS_VMX_LBR(pVM))
                            {
                                if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
                                {
                                    Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
                                    pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
                                    break;
                                }
                                if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
                                {
                                    Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
                                    pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
                                    break;
                                }
                                if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
                                {
                                    pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
                                    break;
                                }
                                /* Fallthru (no break) */
                            }
                            pCtx->fExtrn = 0;
                            VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr;
                            ASMSetFlags(fEFlags);
                            AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
                            return VERR_HM_UNEXPECTED_LD_ST_MSR;
                        }
                    }
                }
            }
#endif

            if (fWhat & CPUMCTX_EXTRN_CR_MASK)
            {
                if (fWhat & CPUMCTX_EXTRN_CR0)
                {
                    uint64_t u64Cr0;
                    uint64_t u64Shadow;
                    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0,            &u64Cr0);       AssertRC(rc);
                    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);    AssertRC(rc);
#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
                    u64Cr0 = (u64Cr0    & ~pVmcsInfo->u64Cr0Mask)
                           | (u64Shadow &  pVmcsInfo->u64Cr0Mask);
#else
                    if (!CPUMIsGuestInVmxNonRootMode(pCtx))
                    {
                        u64Cr0 = (u64Cr0    & ~pVmcsInfo->u64Cr0Mask)
                               | (u64Shadow &  pVmcsInfo->u64Cr0Mask);
                    }
                    else
                    {
                        /*
                         * We've merged the guest and nested-guest's CR0 guest/host mask while executing
                         * the nested-guest using hardware-assisted VMX. Accordingly we need to
                         * re-construct CR0. See @bugref{9180#c95} for details.
                         */
                        PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
                        PVMXVVMCS const     pVmcsNstGst  = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
                        u64Cr0 = (u64Cr0                     & ~(pVmcsInfoGst->u64Cr0Mask & pVmcsNstGst->u64Cr0Mask.u))
                               | (pVmcsNstGst->u64GuestCr0.u &   pVmcsNstGst->u64Cr0Mask.u)
                               | (u64Shadow                  &  (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
                        Assert(u64Cr0 & X86_CR0_NE);
                    }
#endif
#ifndef IN_NEM_DARWIN
                    VMMRZCallRing3Disable(pVCpu);   /* May call into PGM which has Log statements. */
#endif
                    CPUMSetGuestCR0(pVCpu, u64Cr0);
#ifndef IN_NEM_DARWIN
                    VMMRZCallRing3Enable(pVCpu);
#endif
                }

                if (fWhat & CPUMCTX_EXTRN_CR4)
                {
                    uint64_t u64Cr4;
                    uint64_t u64Shadow;
                    rc  = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4,            &u64Cr4);      AssertRC(rc);
                    rc |= VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);   AssertRC(rc);
#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
                    u64Cr4 = (u64Cr4    & ~pVmcsInfo->u64Cr4Mask)
                           | (u64Shadow &  pVmcsInfo->u64Cr4Mask);
#else
                    if (!CPUMIsGuestInVmxNonRootMode(pCtx))
                    {
                        u64Cr4 = (u64Cr4    & ~pVmcsInfo->u64Cr4Mask)
                               | (u64Shadow &  pVmcsInfo->u64Cr4Mask);
                    }
                    else
                    {
                        /*
                         * We've merged the guest and nested-guest's CR4 guest/host mask while executing
                         * the nested-guest using hardware-assisted VMX. Accordingly we need to
                         * re-construct CR4. See @bugref{9180#c95} for details.
                         */
                        PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
                        PVMXVVMCS const     pVmcsNstGst  = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
                        u64Cr4 = (u64Cr4                     & ~(pVmcsInfo->u64Cr4Mask & pVmcsNstGst->u64Cr4Mask.u))
                               | (pVmcsNstGst->u64GuestCr4.u &   pVmcsNstGst->u64Cr4Mask.u)
                               | (u64Shadow                  &  (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
                        Assert(u64Cr4 & X86_CR4_VMXE);
                    }
#endif
                    pCtx->cr4 = u64Cr4;
                }

                if (fWhat & CPUMCTX_EXTRN_CR3)
                {
                    /* CR0.PG bit changes are always intercepted, so it's up to date. */
                    if (   VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
                        || (   VM_IS_VMX_NESTED_PAGING(pVM)
                            && CPUMIsGuestPagingEnabledEx(pCtx)))
                    {
                        uint64_t u64Cr3;
                        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3);  AssertRC(rc);
                        if (pCtx->cr3 != u64Cr3)
                        {
                            pCtx->cr3 = u64Cr3;
                            VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
                        }

                        /*
                         * If the guest is in PAE mode, sync back the PDPE's into the guest state.
                         * CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
                         */
                        if (CPUMIsGuestInPAEModeEx(pCtx))
                        {
                            X86PDPE aPaePdpes[4];
                            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u);     AssertRC(rc);
                            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u);     AssertRC(rc);
                            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u);     AssertRC(rc);
                            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u);     AssertRC(rc);
                            if (memcmp(&aPaePdpes[0], &pCtx->aPaePdpes[0], sizeof(aPaePdpes)))
                            {
                                memcpy(&pCtx->aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
                                /* PGM now updates PAE PDPTEs while updating CR3. */
                                VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
                            }
                        }
                    }
                }
            }

#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
            if (fWhat & CPUMCTX_EXTRN_HWVIRT)
            {
                if (   (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
                    && !CPUMIsGuestInVmxNonRootMode(pCtx))
                {
                    Assert(CPUMIsGuestInVmxRootMode(pCtx));
                    rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
                    if (RT_SUCCESS(rc))
                    { /* likely */ }
                    else
                        break;
                }
            }
#endif
        } while (0);

        if (RT_SUCCESS(rc))
        {
            /* Update fExtrn. */
            pCtx->fExtrn &= ~fWhat;

            /* If everything has been imported, clear the HM keeper bit. */
            if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
            {
#ifndef IN_NEM_DARWIN
                pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
#else
                pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
#endif
                Assert(!pCtx->fExtrn);
            }
        }
    }
#ifndef IN_NEM_DARWIN
    else
        AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));

    /*
     * Restore interrupts.
     */
    ASMSetFlags(fEFlags);
#endif

    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);

    if (RT_SUCCESS(rc))
    { /* likely */ }
    else
        return rc;

    /*
     * Honor any pending CR3 updates.
     *
     * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
     * -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
     * -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
     *
     * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
     * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
     * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
     * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
     *
     * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
     *
     * The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
     */
    if (   VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3)
#ifndef IN_NEM_DARWIN
        && VMMRZCallRing3IsEnabled(pVCpu)
#endif
        )
    {
        Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
        PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
        Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
    }

    return VINF_SUCCESS;
}


/**
 * Internal state fetcher, inner version where we fetch all of a_fWhat.
 *
 * @returns VBox status code.
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 * @param   fEFlags     Saved EFLAGS for restoring the interrupt flag. Ignored
 *                      in NEM/darwin context.
 * @tparam  a_fWhat     What to import, zero or more bits from
 *                      HMVMX_CPUMCTX_EXTRN_ALL.
 */
template<uint64_t const a_fWhat>
static int vmxHCImportGuestStateInner(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t fEFlags)
{
    Assert(a_fWhat != 0); /* No AssertCompile as the assertion probably kicks in before the compiler (clang) discards it. */
    AssertCompile(!(a_fWhat & ~HMVMX_CPUMCTX_EXTRN_ALL));
    Assert(   (pVCpu->cpum.GstCtx.fExtrn & a_fWhat) == a_fWhat
           || (pVCpu->cpum.GstCtx.fExtrn & a_fWhat) == (a_fWhat & ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)));

    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);

    PVMCC const pVM  = pVCpu->CTX_SUFF(pVM);

    /* RIP and RFLAGS may have been imported already by the post exit code
       together with the CPUMCTX_EXTRN_INHIBIT_INT/NMI state, so this part
       of the code is skipping this part of the code. */
    if (   (a_fWhat & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS))
        && pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS))
    {
        if (a_fWhat & CPUMCTX_EXTRN_RFLAGS)
            vmxHCImportGuestCoreRFlags(pVCpu, pVmcsInfo);

        if (a_fWhat & CPUMCTX_EXTRN_RIP)
        {
            if (!(a_fWhat & CPUMCTX_EXTRN_CS))
                EMHistoryUpdatePC(pVCpu, vmxHCImportGuestCoreRip(pVCpu), false);
            else
                vmxHCImportGuestCoreRip(pVCpu);
        }
    }

    /* Note! vmxHCImportGuestIntrState may also include RIP and RFLAGS and update fExtrn. */
    if (a_fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
        vmxHCImportGuestIntrState(pVCpu, pVmcsInfo);

    if (a_fWhat & (CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TR))
    {
        if (a_fWhat & CPUMCTX_EXTRN_CS)
        {
            vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);
            /** @todo try get rid of this carp, it smells and is probably never ever
             *        used: */
            if (   !(a_fWhat & CPUMCTX_EXTRN_RIP)
                && (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP))
            {
                vmxHCImportGuestCoreRip(pVCpu);
                pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_RIP;
            }
            EMHistoryUpdatePC(pVCpu, pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, true /* fFlattened */);
        }
        if (a_fWhat & CPUMCTX_EXTRN_SS)
            vmxHCImportGuestSegReg<X86_SREG_SS>(pVCpu);
        if (a_fWhat & CPUMCTX_EXTRN_DS)
            vmxHCImportGuestSegReg<X86_SREG_DS>(pVCpu);
        if (a_fWhat & CPUMCTX_EXTRN_ES)
            vmxHCImportGuestSegReg<X86_SREG_ES>(pVCpu);
        if (a_fWhat & CPUMCTX_EXTRN_FS)
            vmxHCImportGuestSegReg<X86_SREG_FS>(pVCpu);
        if (a_fWhat & CPUMCTX_EXTRN_GS)
            vmxHCImportGuestSegReg<X86_SREG_GS>(pVCpu);

        /* Guest TR.
           Real-mode emulation using virtual-8086 mode has the fake TSS
           (pRealModeTSS) in TR, don't need to import that one. */
#ifndef IN_NEM_DARWIN
        PVMXVMCSINFOSHARED const pVmcsInfoShared  = pVmcsInfo->pShared;
        bool const               fRealOnV86Active = pVmcsInfoShared->RealMode.fRealOnV86Active;
        if ((a_fWhat & CPUMCTX_EXTRN_TR) && !fRealOnV86Active)
#else
        if (a_fWhat & CPUMCTX_EXTRN_TR)
#endif
            vmxHCImportGuestTr(pVCpu);

#ifndef IN_NEM_DARWIN /* NEM/Darwin: HV supports only unrestricted guest execution. */
        if (fRealOnV86Active)
        {
            if (a_fWhat & CPUMCTX_EXTRN_CS)
                pVCpu->cpum.GstCtx.cs.Attr.u = pVmcsInfoShared->RealMode.AttrCS.u;
            if (a_fWhat & CPUMCTX_EXTRN_SS)
                pVCpu->cpum.GstCtx.ss.Attr.u = pVmcsInfoShared->RealMode.AttrSS.u;
            if (a_fWhat & CPUMCTX_EXTRN_DS)
                pVCpu->cpum.GstCtx.ds.Attr.u = pVmcsInfoShared->RealMode.AttrDS.u;
            if (a_fWhat & CPUMCTX_EXTRN_ES)
                pVCpu->cpum.GstCtx.es.Attr.u = pVmcsInfoShared->RealMode.AttrES.u;
            if (a_fWhat & CPUMCTX_EXTRN_FS)
                pVCpu->cpum.GstCtx.fs.Attr.u = pVmcsInfoShared->RealMode.AttrFS.u;
            if (a_fWhat & CPUMCTX_EXTRN_GS)
                pVCpu->cpum.GstCtx.gs.Attr.u = pVmcsInfoShared->RealMode.AttrGS.u;
        }
#endif
    }

    if (a_fWhat & CPUMCTX_EXTRN_RSP)
    {
        int const rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RSP, &pVCpu->cpum.GstCtx.rsp);
        AssertRC(rc);
    }

    if (a_fWhat & CPUMCTX_EXTRN_LDTR)
        vmxHCImportGuestLdtr(pVCpu);

    if (a_fWhat & CPUMCTX_EXTRN_GDTR)
    {
        int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE,    &pVCpu->cpum.GstCtx.gdtr.pGdt); AssertRC(rc1);
        uint32_t u32Val;
        int const rc2 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRC(rc2);
        pVCpu->cpum.GstCtx.gdtr.cbGdt = (uint16_t)u32Val;
    }

    /* Guest IDTR. */
    if (a_fWhat & CPUMCTX_EXTRN_IDTR)
    {
        int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE,    &pVCpu->cpum.GstCtx.idtr.pIdt); AssertRC(rc1);
        uint32_t u32Val;
        int const rc2 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRC(rc2);
        pVCpu->cpum.GstCtx.idtr.cbIdt = (uint64_t)u32Val;
    }

    if (a_fWhat & CPUMCTX_EXTRN_DR7)
    {
#ifndef IN_NEM_DARWIN
        if (!pVCpu->hmr0.s.fUsingHyperDR7)
#endif
        {
            int rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_DR7, &pVCpu->cpum.GstCtx.dr[7]);
            AssertRC(rc);
        }
    }

    if (a_fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
    {
        int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_EIP,  &pVCpu->cpum.GstCtx.SysEnter.eip); AssertRC(rc1);
        int const rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_SYSENTER_ESP,  &pVCpu->cpum.GstCtx.SysEnter.esp); AssertRC(rc2);
        uint32_t u32Val;
        int const rc3 = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRC(rc3);
        pVCpu->cpum.GstCtx.SysEnter.cs = u32Val;
    }

#ifndef IN_NEM_DARWIN
    if (a_fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
    {
        if (   (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
            && pVM->hmr0.s.fAllow64BitGuests)
            pVCpu->cpum.GstCtx.msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
    }

    if (a_fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
    {
        if (   (pVCpu->hmr0.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
            && pVM->hmr0.s.fAllow64BitGuests)
        {
            pVCpu->cpum.GstCtx.msrLSTAR  = ASMRdMsr(MSR_K8_LSTAR);
            pVCpu->cpum.GstCtx.msrSTAR   = ASMRdMsr(MSR_K6_STAR);
            pVCpu->cpum.GstCtx.msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
        }
    }

    if (a_fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
    {
        PVMXVMCSINFOSHARED pVmcsInfoShared = pVmcsInfo->pShared;
        PCVMXAUTOMSR       pMsrs           = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
        uint32_t const     cMsrs           = pVmcsInfo->cExitMsrStore;
        Assert(pMsrs);
        Assert(cMsrs <= VMX_MISC_MAX_MSRS(g_HmMsrs.u.vmx.u64Misc));
        Assert(sizeof(*pMsrs) * cMsrs <= X86_PAGE_4K_SIZE);
        for (uint32_t i = 0; i < cMsrs; i++)
        {
            uint32_t const idMsr = pMsrs[i].u32Msr;
            switch (idMsr)
            {
                case MSR_K8_TSC_AUX:        CPUMSetGuestTscAux(pVCpu, pMsrs[i].u64Value);     break;
                case MSR_IA32_SPEC_CTRL:    CPUMSetGuestSpecCtrl(pVCpu, pMsrs[i].u64Value);   break;
                case MSR_K6_EFER:           /* Can't be changed without causing a VM-exit */  break;
                default:
                {
                    uint32_t idxLbrMsr;
                    if (VM_IS_VMX_LBR(pVM))
                    {
                        if (hmR0VmxIsLbrBranchFromMsr(pVM, idMsr, &idxLbrMsr))
                        {
                            Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
                            pVmcsInfoShared->au64LbrFromIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
                            break;
                        }
                        if (hmR0VmxIsLbrBranchToMsr(pVM, idMsr, &idxLbrMsr))
                        {
                            Assert(idxLbrMsr < RT_ELEMENTS(pVmcsInfoShared->au64LbrFromIpMsr));
                            pVmcsInfoShared->au64LbrToIpMsr[idxLbrMsr] = pMsrs[i].u64Value;
                            break;
                        }
                        if (idMsr == pVM->hmr0.s.vmx.idLbrTosMsr)
                        {
                            pVmcsInfoShared->u64LbrTosMsr = pMsrs[i].u64Value;
                            break;
                        }
                    }
                    pVCpu->cpum.GstCtx.fExtrn = 0;
                    VCPU_2_VMXSTATE(pVCpu).u32HMError = pMsrs->u32Msr;
                    ASMSetFlags(fEFlags);
                    AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", idMsr, cMsrs));
                    return VERR_HM_UNEXPECTED_LD_ST_MSR;
                }
            }
        }
    }
#endif

    if (a_fWhat & CPUMCTX_EXTRN_CR0)
    {
        uint64_t u64Cr0;
        uint64_t u64Shadow;
        int const rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0,            &u64Cr0);    AssertRC(rc1);
        int const rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow); AssertRC(rc2);
#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
        u64Cr0 = (u64Cr0    & ~pVmcsInfo->u64Cr0Mask)
               | (u64Shadow &  pVmcsInfo->u64Cr0Mask);
#else
        if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
            u64Cr0 = (u64Cr0    & ~pVmcsInfo->u64Cr0Mask)
                   | (u64Shadow &  pVmcsInfo->u64Cr0Mask);
        else
        {
            /*
             * We've merged the guest and nested-guest's CR0 guest/host mask while executing
             * the nested-guest using hardware-assisted VMX. Accordingly we need to
             * re-construct CR0. See @bugref{9180#c95} for details.
             */
            PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
            PVMXVVMCS const     pVmcsNstGst  = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
            u64Cr0 = (u64Cr0                     & ~(pVmcsInfoGst->u64Cr0Mask & pVmcsNstGst->u64Cr0Mask.u))
                   | (pVmcsNstGst->u64GuestCr0.u &   pVmcsNstGst->u64Cr0Mask.u)
                   | (u64Shadow                  &  (pVmcsInfoGst->u64Cr0Mask & ~pVmcsNstGst->u64Cr0Mask.u));
            Assert(u64Cr0 & X86_CR0_NE);
        }
#endif
#ifndef IN_NEM_DARWIN
        VMMRZCallRing3Disable(pVCpu);   /* May call into PGM which has Log statements. */
#endif
        CPUMSetGuestCR0(pVCpu, u64Cr0);
#ifndef IN_NEM_DARWIN
        VMMRZCallRing3Enable(pVCpu);
#endif
    }

    if (a_fWhat & CPUMCTX_EXTRN_CR4)
    {
        uint64_t u64Cr4;
        uint64_t u64Shadow;
        int rc1 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4,            &u64Cr4);    AssertRC(rc1);
        int rc2 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow); AssertRC(rc2);
#ifndef VBOX_WITH_NESTED_HWVIRT_VMX
        u64Cr4 = (u64Cr4    & ~pVmcsInfo->u64Cr4Mask)
               | (u64Shadow &  pVmcsInfo->u64Cr4Mask);
#else
        if (!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
            u64Cr4 = (u64Cr4    & ~pVmcsInfo->u64Cr4Mask)
                   | (u64Shadow &  pVmcsInfo->u64Cr4Mask);
        else
        {
            /*
             * We've merged the guest and nested-guest's CR4 guest/host mask while executing
             * the nested-guest using hardware-assisted VMX. Accordingly we need to
             * re-construct CR4. See @bugref{9180#c95} for details.
             */
            PCVMXVMCSINFO const pVmcsInfoGst = &pVCpu->hmr0.s.vmx.VmcsInfo;
            PVMXVVMCS const     pVmcsNstGst  = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
            u64Cr4 = (u64Cr4                     & ~(pVmcsInfo->u64Cr4Mask & pVmcsNstGst->u64Cr4Mask.u))
                   | (pVmcsNstGst->u64GuestCr4.u &   pVmcsNstGst->u64Cr4Mask.u)
                   | (u64Shadow                  &  (pVmcsInfoGst->u64Cr4Mask & ~pVmcsNstGst->u64Cr4Mask.u));
            Assert(u64Cr4 & X86_CR4_VMXE);
        }
#endif
        pVCpu->cpum.GstCtx.cr4 = u64Cr4;
    }

    if (a_fWhat & CPUMCTX_EXTRN_CR3)
    {
        /* CR0.PG bit changes are always intercepted, so it's up to date. */
        if (   VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
            || (   VM_IS_VMX_NESTED_PAGING(pVM)
                && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)))
        {
            uint64_t u64Cr3;
            int const rc0 = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR3, &u64Cr3);  AssertRC(rc0);
            if (pVCpu->cpum.GstCtx.cr3 != u64Cr3)
            {
                pVCpu->cpum.GstCtx.cr3 = u64Cr3;
                VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
            }

            /*
             * If the guest is in PAE mode, sync back the PDPE's into the guest state.
             * CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date.
             */
            if (CPUMIsGuestInPAEModeEx(&pVCpu->cpum.GstCtx))
            {
                X86PDPE aPaePdpes[4];
                int const rc1 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &aPaePdpes[0].u); AssertRC(rc1);
                int const rc2 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &aPaePdpes[1].u); AssertRC(rc2);
                int const rc3 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &aPaePdpes[2].u); AssertRC(rc3);
                int const rc4 = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &aPaePdpes[3].u); AssertRC(rc4);
                if (memcmp(&aPaePdpes[0], &pVCpu->cpum.GstCtx.aPaePdpes[0], sizeof(aPaePdpes)))
                {
                    memcpy(&pVCpu->cpum.GstCtx.aPaePdpes[0], &aPaePdpes[0], sizeof(aPaePdpes));
                    /* PGM now updates PAE PDPTEs while updating CR3. */
                    VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
                }
            }
        }
    }

#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
    if (a_fWhat & CPUMCTX_EXTRN_HWVIRT)
    {
        if (   (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING)
            && !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
        {
            Assert(CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx));
            int const rc = vmxHCCopyShadowToNstGstVmcs(pVCpu, pVmcsInfo);
            AssertRCReturn(rc, rc);
        }
    }
#endif

    /* Update fExtrn. */
    pVCpu->cpum.GstCtx.fExtrn &= ~a_fWhat;

    /* If everything has been imported, clear the HM keeper bit. */
    if (!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
    {
#ifndef IN_NEM_DARWIN
        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
#else
        pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_KEEPER_NEM;
#endif
        Assert(!pVCpu->cpum.GstCtx.fExtrn);
    }

    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestState, x);

    /*
     * Honor any pending CR3 updates.
     *
     * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> VMXR0CallRing3Callback()
     * -> VMMRZCallRing3Disable() -> vmxHCImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
     * -> continue with VM-exit handling -> vmxHCImportGuestState() and here we are.
     *
     * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
     * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
     * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
     * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
     *
     * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
     *
     * The force-flag is checked first as it's cheaper for potential superfluous calls to this function.
     */
#ifndef IN_NEM_DARWIN
    if (!(a_fWhat & CPUMCTX_EXTRN_CR3)
        ? RT_LIKELY(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) || !VMMRZCallRing3IsEnabled(pVCpu))
        :           !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) || !VMMRZCallRing3IsEnabled(pVCpu) )
        return VINF_SUCCESS;
    ASMSetFlags(fEFlags);
#else
    if (!(a_fWhat & CPUMCTX_EXTRN_CR3)
        ? RT_LIKELY(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
        :           !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3) )
        return VINF_SUCCESS;
    RT_NOREF_PV(fEFlags);
#endif

    Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
    PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
    Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
    return VINF_SUCCESS;
}


/**
 * Internal state fetcher.
 *
 * @returns VBox status code.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmcsInfo       The VMCS info. object.
 * @param   pszCaller       For logging.
 * @tparam  a_fWhat         What needs to be imported, CPUMCTX_EXTRN_XXX.
 * @tparam  a_fDoneLocal    What's ASSUMED to have been retrieved locally
 *                          already.  This is ORed together with @a a_fWhat when
 *                          calculating what needs fetching (just for safety).
 * @tparam  a_fDonePostExit What's ASSUMED to been been retrieved by
 *                          hmR0VmxPostRunGuest()/nemR3DarwinHandleExitCommon()
 *                          already.  This is ORed together with @a a_fWhat when
 *                          calculating what needs fetching (just for safety).
 */
template<uint64_t const a_fWhat,
    uint64_t const a_fDoneLocal = 0,
    uint64_t const a_fDonePostExit = 0
#ifndef IN_NEM_DARWIN
                                   | CPUMCTX_EXTRN_INHIBIT_INT
                                   | CPUMCTX_EXTRN_INHIBIT_NMI
# if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
                                   | HMVMX_CPUMCTX_EXTRN_ALL
# elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
                                   | CPUMCTX_EXTRN_RFLAGS
# endif
#else  /* IN_NEM_DARWIN */
                                   | CPUMCTX_EXTRN_ALL /** @todo optimize */
#endif /* IN_NEM_DARWIN */
>
DECLINLINE(int) vmxHCImportGuestState(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, const char *pszCaller)
{
    RT_NOREF_PV(pszCaller);
    if ((a_fWhat | a_fDoneLocal | a_fDonePostExit) & HMVMX_CPUMCTX_EXTRN_ALL)
    {
#ifndef IN_NEM_DARWIN
        /*
         * We disable interrupts to make the updating of the state and in particular
         * the fExtrn modification atomic wrt to preemption hooks.
         */
        RTCCUINTREG const fEFlags   = ASMIntDisableFlags();
#else
        RTCCUINTREG const fEFlags   = 0;
#endif

        /*
         * We combine all three parameters and take the (probably) inlined optimized
         * code path for the new things specified in a_fWhat.
         *
         * As a tweak to deal with exits that have INHIBIT_INT/NMI active, causing
         * vmxHCImportGuestIntrState to automatically fetch both RIP & RFLAGS, we
         * also take the streamlined path when both of these are cleared in fExtrn
         * already. vmxHCImportGuestStateInner checks fExtrn before fetching.  This
         * helps with MWAIT and HLT exits that always inhibit IRQs on many platforms.
         */
        uint64_t const    fWhatToDo = pVCpu->cpum.GstCtx.fExtrn
                                    & ((a_fWhat | a_fDoneLocal | a_fDonePostExit) & HMVMX_CPUMCTX_EXTRN_ALL);
        if (RT_LIKELY(   (   fWhatToDo ==   (a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal | a_fDonePostExit))
                          || fWhatToDo == (  a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal | a_fDonePostExit)
                                           & ~(CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)) /* fetch with INHIBIT_INT/NMI */))
            && (a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL & ~(a_fDoneLocal | a_fDonePostExit)) != 0 /* just in case */)
        {
            int const rc = vmxHCImportGuestStateInner<  a_fWhat
                                                      & HMVMX_CPUMCTX_EXTRN_ALL
                                                      & ~(a_fDoneLocal | a_fDonePostExit)>(pVCpu, pVmcsInfo, fEFlags);
#ifndef IN_NEM_DARWIN
            ASMSetFlags(fEFlags);
#endif
            return rc;
        }

#ifndef IN_NEM_DARWIN
        ASMSetFlags(fEFlags);
#endif

        /*
         * We shouldn't normally get here, but it may happen when executing
         * in the debug run-loops.  Typically, everything should already have
         * been fetched then.  Otherwise call the fallback state import function.
         */
        if (fWhatToDo == 0)
        { /* hope the cause was the debug loop or something similar */ }
        else
        {
            STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatImportGuestStateFallback);
            Log11Func(("a_fWhat=%#RX64/%#RX64/%#RX64 fExtrn=%#RX64 => %#RX64 - Taking inefficient code path from %s!\n",
                       a_fWhat & HMVMX_CPUMCTX_EXTRN_ALL, a_fDoneLocal & HMVMX_CPUMCTX_EXTRN_ALL,
                       a_fDonePostExit & HMVMX_CPUMCTX_EXTRN_ALL, pVCpu->cpum.GstCtx.fExtrn, fWhatToDo, pszCaller));
            return vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, a_fWhat | a_fDoneLocal | a_fDonePostExit);
        }
    }
    return VINF_SUCCESS;
}


/**
 * Check per-VM and per-VCPU force flag actions that require us to go back to
 * ring-3 for one reason or another.
 *
 * @returns Strict VBox status code (i.e. informational status codes too)
 * @retval VINF_SUCCESS if we don't have any actions that require going back to
 *         ring-3.
 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
 *         interrupts)
 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
 *         all EMTs to be in ring-3.
 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
 *         to the EM loop.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   fIsNestedGuest  Flag whether this is for a for a pending nested guest event.
 * @param   fStepping       Whether we are single-stepping the guest using the
 *                          hypervisor debugger.
 *
 * @remarks This might cause nested-guest VM-exits, caller must check if the guest
 *          is no longer in VMX non-root mode.
 */
static VBOXSTRICTRC vmxHCCheckForceFlags(PVMCPUCC pVCpu, bool fIsNestedGuest, bool fStepping)
{
#ifndef IN_NEM_DARWIN
    Assert(VMMRZCallRing3IsEnabled(pVCpu));
#endif

    /*
     * Update pending interrupts into the APIC's IRR.
     */
    if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
        APICUpdatePendingInterrupts(pVCpu);

    /*
     * Anything pending?  Should be more likely than not if we're doing a good job.
     */
    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
    if (  !fStepping
        ?    !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
          && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
        :    !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
          && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
        return VINF_SUCCESS;

    /* Pending PGM C3 sync. */
    if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
    {
        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
        Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
        VBOXSTRICTRC rcStrict = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
                                           VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
        if (rcStrict != VINF_SUCCESS)
        {
            AssertRC(VBOXSTRICTRC_VAL(rcStrict));
            Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
            return rcStrict;
        }
    }

    /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
    if (   VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
        || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
    {
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHmToR3FF);
        int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
        Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d (fVM=%#RX64 fCpu=%#RX64)\n",
                  rc, pVM->fGlobalForcedActions, pVCpu->fLocalForcedActions));
        return rc;
    }

    /* Pending VM request packets, such as hardware interrupts. */
    if (   VM_FF_IS_SET(pVM, VM_FF_REQUEST)
        || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
    {
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchVmReq);
        Log4Func(("Pending VM request forcing us back to ring-3\n"));
        return VINF_EM_PENDING_REQUEST;
    }

    /* Pending PGM pool flushes. */
    if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
    {
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchPgmPoolFlush);
        Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
        return VINF_PGM_POOL_FLUSH_PENDING;
    }

    /* Pending DMA requests. */
    if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
    {
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchDma);
        Log4Func(("Pending DMA request forcing us back to ring-3\n"));
        return VINF_EM_RAW_TO_R3;
    }

#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
    /*
     * Pending nested-guest events.
     *
     * Please note the priority of these events are specified and important.
     * See Intel spec. 29.4.3.2 "APIC-Write Emulation".
     * See Intel spec. 6.9 "Priority Among Simultaneous Exceptions And Interrupts".
     */
    if (fIsNestedGuest)
    {
        /* Pending nested-guest APIC-write. */
        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_APIC_WRITE))
        {
            Log4Func(("Pending nested-guest APIC-write\n"));
            VBOXSTRICTRC rcStrict = IEMExecVmxVmexitApicWrite(pVCpu);
            Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
            return rcStrict;
        }

        /* Pending nested-guest monitor-trap flag (MTF). */
        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_MTF))
        {
            Log4Func(("Pending nested-guest MTF\n"));
            VBOXSTRICTRC rcStrict = IEMExecVmxVmexit(pVCpu, VMX_EXIT_MTF, 0 /* uExitQual */);
            Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
            return rcStrict;
        }

        /* Pending nested-guest VMX-preemption timer expired. */
        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_VMX_PREEMPT_TIMER))
        {
            Log4Func(("Pending nested-guest preempt timer\n"));
            VBOXSTRICTRC rcStrict = IEMExecVmxVmexitPreemptTimer(pVCpu);
            Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
            return rcStrict;
        }
    }
#else
    NOREF(fIsNestedGuest);
#endif

    return VINF_SUCCESS;
}


/**
 * Converts any TRPM trap into a pending HM event. This is typically used when
 * entering from ring-3 (not longjmp returns).
 *
 * @param   pVCpu   The cross context virtual CPU structure.
 */
static void vmxHCTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
{
    Assert(TRPMHasTrap(pVCpu));
    Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);

    uint8_t     uVector;
    TRPMEVENT   enmTrpmEvent;
    uint32_t    uErrCode;
    RTGCUINTPTR GCPtrFaultAddress;
    uint8_t     cbInstr;
    bool        fIcebp;

    int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, &fIcebp);
    AssertRC(rc);

    uint32_t u32IntInfo;
    u32IntInfo  = uVector | VMX_IDT_VECTORING_INFO_VALID;
    u32IntInfo |= HMTrpmEventTypeToVmxEventType(uVector, enmTrpmEvent, fIcebp);

    rc = TRPMResetTrap(pVCpu);
    AssertRC(rc);
    Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
          u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));

    vmxHCSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
}


/**
 * Converts the pending HM event into a TRPM trap.
 *
 * @param   pVCpu   The cross context virtual CPU structure.
 */
static void vmxHCPendingEventToTrpmTrap(PVMCPUCC pVCpu)
{
    Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);

    /* If a trap was already pending, we did something wrong! */
    Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);

    uint32_t const  u32IntInfo  = VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo;
    uint32_t const  uVector     = VMX_IDT_VECTORING_INFO_VECTOR(u32IntInfo);
    TRPMEVENT const enmTrapType = HMVmxEventTypeToTrpmEventType(u32IntInfo);

    Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));

    int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
    AssertRC(rc);

    if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
        TRPMSetErrorCode(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode);

    if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(u32IntInfo))
        TRPMSetFaultAddress(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.GCPtrFaultAddress);
    else
    {
        uint8_t const uVectorType = VMX_IDT_VECTORING_INFO_TYPE(u32IntInfo);
        switch (uVectorType)
        {
            case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
                TRPMSetTrapDueToIcebp(pVCpu);
                RT_FALL_THRU();
            case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
            case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
            {
                AssertMsg(   uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
                          || (   uVector == X86_XCPT_BP /* INT3 */
                              || uVector == X86_XCPT_OF /* INTO */
                              || uVector == X86_XCPT_DB /* INT1 (ICEBP) */),
                          ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
                TRPMSetInstrLength(pVCpu, VCPU_2_VMXSTATE(pVCpu).Event.cbInstr);
                break;
            }
        }
    }

    /* We're now done converting the pending event. */
    VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;
}


/**
 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 */
static void vmxHCSetIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
{
    if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
    {
        if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
        {
            pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
            int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
            AssertRC(rc);
        }
    } /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
}


/**
 * Clears the interrupt-window exiting control in the VMCS.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 */
DECLINLINE(void) vmxHCClearIntWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
{
    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
    {
        pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
        int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
        AssertRC(rc);
    }
}


/**
 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 */
static void vmxHCSetNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
{
    if (g_HmMsrs.u.vmx.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
    {
        if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
        {
            pVmcsInfo->u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
            int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
            AssertRC(rc);
            Log4Func(("Setup NMI-window exiting\n"));
        }
    } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
}


/**
 * Clears the NMI-window exiting control in the VMCS.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 */
DECLINLINE(void) vmxHCClearNmiWindowExitVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo)
{
    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
    {
        pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
        int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
        AssertRC(rc);
    }
}


/**
 * Injects an event into the guest upon VM-entry by updating the relevant fields
 * in the VM-entry area in the VMCS.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @retval  VINF_SUCCESS if the event is successfully injected into the VMCS.
 * @retval  VINF_EM_RESET if event injection resulted in a triple-fault.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmcsInfo       The VMCS info object.
 * @param   fIsNestedGuest  Flag whether this is for a for a pending nested guest event.
 * @param   pEvent          The event being injected.
 * @param   pfIntrState     Pointer to the VT-x guest-interruptibility-state. This
 *                          will be updated if necessary. This cannot not be NULL.
 * @param   fStepping       Whether we're single-stepping guest execution and should
 *                          return VINF_EM_DBG_STEPPED if the event is injected
 *                          directly (registers modified by us, not by hardware on
 *                          VM-entry).
 */
static VBOXSTRICTRC vmxHCInjectEventVmcs(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, PCHMEVENT pEvent,
                                         bool fStepping, uint32_t *pfIntrState)
{
    /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
    AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
    Assert(pfIntrState);

#ifdef IN_NEM_DARWIN
    RT_NOREF(fIsNestedGuest, fStepping, pfIntrState);
#endif

    PCPUMCTX          pCtx       = &pVCpu->cpum.GstCtx;
    uint32_t          u32IntInfo = pEvent->u64IntInfo;
    uint32_t const    u32ErrCode = pEvent->u32ErrCode;
    uint32_t const    cbInstr    = pEvent->cbInstr;
    RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
    uint8_t const     uVector    = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
    uint32_t const    uIntType   = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);

#ifdef VBOX_STRICT
    /*
     * Validate the error-code-valid bit for hardware exceptions.
     * No error codes for exceptions in real-mode.
     *
     * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
     */
    if (   uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
        && !CPUMIsGuestInRealModeEx(pCtx))
    {
        switch (uVector)
        {
            case X86_XCPT_PF:
            case X86_XCPT_DF:
            case X86_XCPT_TS:
            case X86_XCPT_NP:
            case X86_XCPT_SS:
            case X86_XCPT_GP:
            case X86_XCPT_AC:
                AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
                          ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
                RT_FALL_THRU();
            default:
                break;
        }
    }

    /* Cannot inject an NMI when block-by-MOV SS is in effect. */
    Assert(   uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
           || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
#endif

    RT_NOREF(uVector);
    if (   uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
        || uIntType == VMX_EXIT_INT_INFO_TYPE_NMI
        || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT
        || uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
    {
        Assert(uVector <= X86_XCPT_LAST);
        Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_NMI          || uVector == X86_XCPT_NMI);
        Assert(uIntType != VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT || uVector == X86_XCPT_DB);
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedXcpts[uVector]);
    }
    else
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).aStatInjectedIrqs[uVector & MASK_INJECT_IRQ_STAT]);

    /*
     * Hardware interrupts & exceptions cannot be delivered through the software interrupt
     * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
     * interrupt handler in the (real-mode) guest.
     *
     * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
     * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
     */
    if (CPUMIsGuestInRealModeEx(pCtx))     /* CR0.PE bit changes are always intercepted, so it's up to date. */
    {
#ifndef IN_NEM_DARWIN
        if (pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest)
#endif
        {
            /*
             * For CPUs with unrestricted guest execution enabled and with the guest
             * in real-mode, we must not set the deliver-error-code bit.
             *
             * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
             */
            u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
        }
#ifndef IN_NEM_DARWIN
        else
        {
            PVMCC pVM = pVCpu->CTX_SUFF(pVM);
            Assert(PDMVmmDevHeapIsEnabled(pVM));
            Assert(pVM->hm.s.vmx.pRealModeTSS);
            Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));

            /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
            int rc2 = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK
                                                                | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
            AssertRCReturn(rc2, rc2);

            /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
            size_t const cbIdtEntry = sizeof(X86IDTR16);
            if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
            {
                /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
                if (uVector == X86_XCPT_DF)
                    return VINF_EM_RESET;

                /* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
                   No error codes for exceptions in real-mode. */
                if (uVector == X86_XCPT_GP)
                {
                    static HMEVENT const s_EventXcptDf
                        = HMEVENT_INIT_ONLY_INT_INFO(  RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
                                                     | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,   VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
                                                     | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
                                                     | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1));
                    return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptDf, fStepping, pfIntrState);
                }

                /*
                 * If we're injecting an event with no valid IDT entry, inject a #GP.
                 * No error codes for exceptions in real-mode.
                 *
                 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
                 */
                static HMEVENT const s_EventXcptGp
                    = HMEVENT_INIT_ONLY_INT_INFO(  RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
                                                 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE,   VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
                                                 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
                                                 | RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID,          1));
                return vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &s_EventXcptGp, fStepping, pfIntrState);
            }

            /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
            uint16_t uGuestIp = pCtx->ip;
            if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
            {
                Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
                /* #BP and #OF are both benign traps, we need to resume the next instruction. */
                uGuestIp = pCtx->ip + (uint16_t)cbInstr;
            }
            else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
                uGuestIp = pCtx->ip + (uint16_t)cbInstr;

            /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
            X86IDTR16 IdtEntry;
            RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
            rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
            AssertRCReturn(rc2, rc2);

            /* Construct the stack frame for the interrupt/exception handler. */
            VBOXSTRICTRC rcStrict;
            rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, (uint16_t)pCtx->eflags.u);
            if (rcStrict == VINF_SUCCESS)
            {
                rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
                if (rcStrict == VINF_SUCCESS)
                    rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
            }

            /* Clear the required eflag bits and jump to the interrupt/exception handler. */
            if (rcStrict == VINF_SUCCESS)
            {
                pCtx->eflags.u   &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
                pCtx->rip         = IdtEntry.offSel;
                pCtx->cs.Sel      = IdtEntry.uSel;
                pCtx->cs.ValidSel = IdtEntry.uSel;
                pCtx->cs.u64Base  = IdtEntry.uSel << cbIdtEntry;
                if (   uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
                    && uVector  == X86_XCPT_PF)
                    pCtx->cr2 = GCPtrFault;

                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CS  | HM_CHANGED_GUEST_CR2
                                                         | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
                                                         | HM_CHANGED_GUEST_RSP);

                /*
                 * If we delivered a hardware exception (other than an NMI) and if there was
                 * block-by-STI in effect, we should clear it.
                 */
                if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
                {
                    Assert(   uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
                           && uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
                    Log4Func(("Clearing inhibition due to STI\n"));
                    *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
                }

                Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
                      u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));

                /*
                 * The event has been truly dispatched to the guest. Mark it as no longer pending so
                 * we don't attempt to undo it if we are returning to ring-3 before executing guest code.
                 */
                VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;

                /*
                 * If we eventually support nested-guest execution without unrestricted guest execution,
                 * we should set fInterceptEvents here.
                 */
                Assert(!fIsNestedGuest);

                /* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
                if (fStepping)
                    rcStrict = VINF_EM_DBG_STEPPED;
            }
            AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
                      ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
            return rcStrict;
        }
#else
        RT_NOREF(pVmcsInfo);
#endif
    }

    /*
     * Validate.
     */
    Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo));                     /* Bit 31 (Valid bit) must be set by caller. */
    Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK));       /* Bits 30:12 MBZ. */

    /*
     * Inject the event into the VMCS.
     */
    int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
    if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
        rc |= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
    rc |= VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
    AssertRC(rc);

    /*
     * Update guest CR2 if this is a page-fault.
     */
    if (VMX_ENTRY_INT_INFO_IS_XCPT_PF(u32IntInfo))
        pCtx->cr2 = GCPtrFault;

    Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
    return VINF_SUCCESS;
}


/**
 * Evaluates the event to be delivered to the guest and sets it as the pending
 * event.
 *
 * Toggling of interrupt force-flags here is safe since we update TRPM on premature
 * exits to ring-3 before executing guest code, see vmxHCExitToRing3(). We must
 * NOT restore these force-flags.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmcsInfo       The VMCS information structure.
 * @param   fIsNestedGuest  Flag whether the evaluation happens for a nested guest.
 * @param   pfIntrState     Where to store the VT-x guest-interruptibility state.
 */
static VBOXSTRICTRC vmxHCEvaluatePendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest, uint32_t *pfIntrState)
{
    Assert(pfIntrState);
    Assert(!TRPMHasTrap(pVCpu));

    /*
     * Compute/update guest-interruptibility state related FFs.
     * The FFs will be used below while evaluating events to be injected.
     */
    *pfIntrState = vmxHCGetGuestIntrStateAndUpdateFFs(pVCpu);

    /*
     * Evaluate if a new event needs to be injected.
     * An event that's already pending has already performed all necessary checks.
     */
    if (   !VCPU_2_VMXSTATE(pVCpu).Event.fPending
        && !CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx))
    {
        /** @todo SMI. SMIs take priority over NMIs. */

        /*
         * NMIs.
         * NMIs take priority over external interrupts.
         */
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
        PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
#endif
        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
        {
            /*
             * For a guest, the FF always indicates the guest's ability to receive an NMI.
             *
             * For a nested-guest, the FF always indicates the outer guest's ability to
             * receive an NMI while the guest-interruptibility state bit depends on whether
             * the nested-hypervisor is using virtual-NMIs.
             */
            if (!CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx))
            {
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
                if (   fIsNestedGuest
                    && CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_NMI_EXIT))
                    return IEMExecVmxVmexitXcptNmi(pVCpu);
#endif
                vmxHCSetPendingXcptNmi(pVCpu);
                VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
                Log4Func(("NMI pending injection\n"));

                /* We've injected the NMI, bail. */
                return VINF_SUCCESS;
            }
            if (!fIsNestedGuest)
                vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
        }

        /*
         * External interrupts (PIC/APIC).
         * Once PDMGetInterrupt() returns a valid interrupt we -must- deliver it.
         * We cannot re-request the interrupt from the controller again.
         */
        if (    VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
            && !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
        {
            Assert(!DBGFIsStepping(pVCpu));
            int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
            AssertRC(rc);

            /*
             * We must not check EFLAGS directly when executing a nested-guest, use
             * CPUMIsGuestPhysIntrEnabled() instead as EFLAGS.IF does not control the blocking of
             * external interrupts when "External interrupt exiting" is set. This fixes a nasty
             * SMP hang while executing nested-guest VCPUs on spinlocks which aren't rescued by
             * other VM-exits (like a preemption timer), see @bugref{9562#c18}.
             *
             * See Intel spec. 25.4.1 "Event Blocking".
             */
            if (CPUMIsGuestPhysIntrEnabled(pVCpu))
            {
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
                if (    fIsNestedGuest
                    &&  CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
                {
                    VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0 /* uVector */, true /* fIntPending */);
                    if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
                        return rcStrict;
                }
#endif
                uint8_t u8Interrupt;
                rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
                if (RT_SUCCESS(rc))
                {
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
                    if (   fIsNestedGuest
                        && CPUMIsGuestVmxPinCtlsSet(pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
                    {
                        VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
                        Assert(rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE);
                        return rcStrict;
                    }
#endif
                    vmxHCSetPendingExtInt(pVCpu, u8Interrupt);
                    Log4Func(("External interrupt (%#x) pending injection\n", u8Interrupt));
                }
                else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
                {
                    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchTprMaskedIrq);

                    if (   !fIsNestedGuest
                        && (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
                        vmxHCApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
                    /* else: for nested-guests, TPR threshold is picked up while merging VMCS controls. */

                    /*
                     * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
                     * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
                     * need to re-set this force-flag here.
                     */
                }
                else
                    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchGuestIrq);

                /* We've injected the interrupt or taken necessary action, bail. */
                return VINF_SUCCESS;
            }
            if (!fIsNestedGuest)
                vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
        }
    }
    else if (!fIsNestedGuest)
    {
        /*
         * An event is being injected or we are in an interrupt shadow. Check if another event is
         * pending. If so, instruct VT-x to cause a VM-exit as soon as the guest is ready to accept
         * the pending event.
         */
        if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
            vmxHCSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
        else if (   VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
                 && !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
            vmxHCSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
    }
    /* else: for nested-guests, NMI/interrupt-window exiting will be picked up when merging VMCS controls. */

    return VINF_SUCCESS;
}


/**
 * Injects any pending events into the guest if the guest is in a state to
 * receive them.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmcsInfo       The VMCS information structure.
 * @param   fIsNestedGuest  Flag whether the event injection happens for a nested guest.
 * @param   fIntrState      The VT-x guest-interruptibility state.
 * @param   fStepping       Whether we are single-stepping the guest using the
 *                          hypervisor debugger and should return
 *                          VINF_EM_DBG_STEPPED if the event was dispatched
 *                          directly.
 */
static VBOXSTRICTRC vmxHCInjectPendingEvent(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNestedGuest,
                                            uint32_t fIntrState, bool fStepping)
{
    HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
#ifndef IN_NEM_DARWIN
    Assert(VMMRZCallRing3IsEnabled(pVCpu));
#endif

#ifdef VBOX_STRICT
    /*
     * Verify guest-interruptibility state.
     *
     * We put this in a scoped block so we do not accidentally use fBlockSti or fBlockMovSS,
     * since injecting an event may modify the interruptibility state and we must thus always
     * use fIntrState.
     */
    {
        bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
        bool const fBlockSti   = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
        Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
        Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF);     /* Cannot set block-by-STI when interrupts are disabled. */
        Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI));    /* We don't support block-by-SMI yet.*/
        Assert(!TRPMHasTrap(pVCpu));
        NOREF(fBlockMovSS); NOREF(fBlockSti);
    }
#endif

    VBOXSTRICTRC rcStrict = VINF_SUCCESS;
    if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
    {
        /*
         * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
         * pending even while injecting an event and in this case, we want a VM-exit as soon as
         * the guest is ready for the next interrupt, see @bugref{6208#c45}.
         *
         * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
         */
        uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo);
#ifdef VBOX_STRICT
        if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
        {
            Assert(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_IF);
            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
        }
        else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
        {
            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI));
            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI));
            Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
        }
#endif
        Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
              uIntType));

        /*
         * Inject the event and get any changes to the guest-interruptibility state.
         *
         * The guest-interruptibility state may need to be updated if we inject the event
         * into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
         */
        rcStrict = vmxHCInjectEventVmcs(pVCpu, pVmcsInfo, fIsNestedGuest, &VCPU_2_VMXSTATE(pVCpu).Event, fStepping, &fIntrState);
        AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);

        if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterrupt);
        else
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectXcpt);
    }

    /*
     * Deliver any pending debug exceptions if the guest is single-stepping using EFLAGS.TF and
     * is an interrupt shadow (block-by-STI or block-by-MOV SS).
     */
    if (   (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
        && !fIsNestedGuest)
    {
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);

        if (!VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
        {
            /*
             * Set or clear the BS bit depending on whether the trap flag is active or not. We need
             * to do both since we clear the BS bit from the VMCS while exiting to ring-3.
             */
            Assert(!DBGFIsStepping(pVCpu));
            uint8_t const fTrapFlag = !!(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_TF);
            int rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS,
                                       fTrapFlag << VMX_BF_VMCS_PENDING_DBG_XCPT_BS_SHIFT);
            AssertRC(rc);
        }
        else
        {
            /*
             * We must not deliver a debug exception when single-stepping over STI/Mov-SS in the
             * hypervisor debugger using EFLAGS.TF but rather clear interrupt inhibition. However,
             * we take care of this case in vmxHCExportSharedDebugState and also the case if
             * we use MTF, so just make sure it's called before executing guest-code.
             */
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
        }
    }
    /* else: for nested-guest currently handling while merging controls. */

    /*
     * Finally, update the guest-interruptibility state.
     *
     * This is required for the real-on-v86 software interrupt injection, for
     * pending debug exceptions as well as updates to the guest state from ring-3 (IEM).
     */
    int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
    AssertRC(rc);

    /*
     * There's no need to clear the VM-entry interruption-information field here if we're not
     * injecting anything. VT-x clears the valid bit on every VM-exit.
     *
     * See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
     */

    Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
    return rcStrict;
}


/**
 * Tries to determine what part of the guest-state VT-x has deemed as invalid
 * and update error record fields accordingly.
 *
 * @returns VMX_IGS_* error codes.
 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
 *         wrong with the guest state.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   pVmcsInfo   The VMCS info. object.
 *
 * @remarks This function assumes our cache of the VMCS controls
 *          are valid, i.e. vmxHCCheckCachedVmcsCtls() succeeded.
 */
static uint32_t vmxHCCheckGuestState(PVMCPUCC pVCpu, PCVMXVMCSINFO pVmcsInfo)
{
#define HMVMX_ERROR_BREAK(err)              { uError = (err); break; }
#define HMVMX_CHECK_BREAK(expr, err)        if (!(expr)) { uError = (err); break; } else do { } while (0)

    PCPUMCTX pCtx   = &pVCpu->cpum.GstCtx;
    uint32_t uError = VMX_IGS_ERROR;
    uint32_t u32IntrState = 0;
#ifndef IN_NEM_DARWIN
    PVMCC    pVM    = pVCpu->CTX_SUFF(pVM);
    bool const fUnrestrictedGuest = VM_IS_VMX_UNRESTRICTED_GUEST(pVM);
#else
    bool const fUnrestrictedGuest = true;
#endif
    do
    {
        int rc;

        /*
         * Guest-interruptibility state.
         *
         * Read this first so that any check that fails prior to those that actually
         * require the guest-interruptibility state would still reflect the correct
         * VMCS value and avoids causing further confusion.
         */
        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
        AssertRC(rc);

        uint32_t u32Val;
        uint64_t u64Val;

        /*
         * CR0.
         */
        /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
        uint64_t       fSetCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 & g_HmMsrs.u.vmx.u64Cr0Fixed1);
        uint64_t const fZapCr0 = (g_HmMsrs.u.vmx.u64Cr0Fixed0 | g_HmMsrs.u.vmx.u64Cr0Fixed1);
        /* Exceptions for unrestricted guest execution for CR0 fixed bits (PE, PG).
           See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
        if (fUnrestrictedGuest)
            fSetCr0 &= ~(uint64_t)(X86_CR0_PE | X86_CR0_PG);

        uint64_t u64GuestCr0;
        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64GuestCr0);
        AssertRC(rc);
        HMVMX_CHECK_BREAK((u64GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
        HMVMX_CHECK_BREAK(!(u64GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
        if (   !fUnrestrictedGuest
            &&  (u64GuestCr0 & X86_CR0_PG)
            && !(u64GuestCr0 & X86_CR0_PE))
            HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);

        /*
         * CR4.
         */
        /** @todo Why do we need to OR and AND the fixed-0 and fixed-1 bits below? */
        uint64_t const fSetCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 & g_HmMsrs.u.vmx.u64Cr4Fixed1);
        uint64_t const fZapCr4 = (g_HmMsrs.u.vmx.u64Cr4Fixed0 | g_HmMsrs.u.vmx.u64Cr4Fixed1);

        uint64_t u64GuestCr4;
        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR4, &u64GuestCr4);
        AssertRC(rc);
        HMVMX_CHECK_BREAK((u64GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
        HMVMX_CHECK_BREAK(!(u64GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);

        /*
         * IA32_DEBUGCTL MSR.
         */
        rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
        AssertRC(rc);
        if (   (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
            && (u64Val & 0xfffffe3c))                           /* Bits 31:9, bits 5:2 MBZ. */
        {
            HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
        }
        uint64_t u64DebugCtlMsr = u64Val;

#ifdef VBOX_STRICT
        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY, &u32Val);
        AssertRC(rc);
        Assert(u32Val == pVmcsInfo->u32EntryCtls);
#endif
        bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);

        /*
         * RIP and RFLAGS.
         */
        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RIP, &u64Val);
        AssertRC(rc);
        /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
        if (   !fLongModeGuest
            || !pCtx->cs.Attr.n.u1Long)
        {
            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
        }
        /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
         *        must be identical if the "IA-32e mode guest" VM-entry
         *        control is 1 and CS.L is 1. No check applies if the
         *        CPU supports 64 linear-address bits. */

        /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_RFLAGS, &u64Val);
        AssertRC(rc);
        HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)),                     /* Bit 63:22, Bit 15, 5, 3 MBZ. */
                          VMX_IGS_RFLAGS_RESERVED);
        HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1);       /* Bit 1 MB1. */
        uint32_t const u32Eflags = u64Val;

        if (   fLongModeGuest
            || (   fUnrestrictedGuest
                && !(u64GuestCr0 & X86_CR0_PE)))
        {
            HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
        }

        uint32_t u32EntryInfo;
        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
        AssertRC(rc);
        if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
        {
            HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
        }

        /*
         * 64-bit checks.
         */
        if (fLongModeGuest)
        {
            HMVMX_CHECK_BREAK(u64GuestCr0 & X86_CR0_PG,  VMX_IGS_CR0_PG_LONGMODE);
            HMVMX_CHECK_BREAK(u64GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
        }

        if (   !fLongModeGuest
            && (u64GuestCr4 & X86_CR4_PCIDE))
            HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);

        /** @todo CR3 field must be such that bits 63:52 and bits in the range
         *        51:32 beyond the processor's physical-address width are 0. */

        if (   (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
            && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
            HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);

#ifndef IN_NEM_DARWIN
        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
        AssertRC(rc);
        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);

        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
        AssertRC(rc);
        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
#endif

        /*
         * PERF_GLOBAL MSR.
         */
        if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
        {
            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
            AssertRC(rc);
            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
                              VMX_IGS_PERF_GLOBAL_MSR_RESERVED);        /* Bits 63:35, bits 31:2 MBZ. */
        }

        /*
         * PAT MSR.
         */
        if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
        {
            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
            AssertRC(rc);
            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
            for (unsigned i = 0; i < 8; i++)
            {
                uint8_t u8Val = (u64Val & 0xff);
                if (   u8Val != 0 /* UC */
                    && u8Val != 1 /* WC */
                    && u8Val != 4 /* WT */
                    && u8Val != 5 /* WP */
                    && u8Val != 6 /* WB */
                    && u8Val != 7 /* UC- */)
                    HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
                u64Val >>= 8;
            }
        }

        /*
         * EFER MSR.
         */
        if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
        {
            Assert(g_fHmVmxSupportsVmcsEfer);
            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
            AssertRC(rc);
            HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
                              VMX_IGS_EFER_MSR_RESERVED);               /* Bits 63:12, bit 9, bits 7:1 MBZ. */
            HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(  pVmcsInfo->u32EntryCtls
                                                                           & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
                              VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
            /** @todo r=ramshankar: Unrestricted check here is probably wrong, see
             *        iemVmxVmentryCheckGuestState(). */
            HMVMX_CHECK_BREAK(   fUnrestrictedGuest
                              || !(u64GuestCr0 & X86_CR0_PG)
                              || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
                              VMX_IGS_EFER_LMA_LME_MISMATCH);
        }

        /*
         * Segment registers.
         */
        HMVMX_CHECK_BREAK(   (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
                          || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
        if (!(u32Eflags & X86_EFL_VM))
        {
            /* CS */
            HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
            HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
            HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
            HMVMX_CHECK_BREAK(   (pCtx->cs.u32Limit & 0xfff) == 0xfff
                              || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
            HMVMX_CHECK_BREAK(   !(pCtx->cs.u32Limit & 0xfff00000)
                              || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
            /* CS cannot be loaded with NULL in protected mode. */
            HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
            HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
            if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
                HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
            else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
                HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
            else if (fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
                HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
            else
                HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);

            /* SS */
            HMVMX_CHECK_BREAK(   fUnrestrictedGuest
                              || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
            HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
            if (   !(pCtx->cr0 & X86_CR0_PE)
                || pCtx->cs.Attr.n.u4Type == 3)
            {
                HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
            }

            if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
            {
                HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
                HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
                HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
                HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
                HMVMX_CHECK_BREAK(   (pCtx->ss.u32Limit & 0xfff) == 0xfff
                                  || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
                HMVMX_CHECK_BREAK(   !(pCtx->ss.u32Limit & 0xfff00000)
                                  || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
            }

            /* DS, ES, FS, GS - only check for usable selectors, see vmxHCExportGuestSReg(). */
            if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
            {
                HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
                HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
                HMVMX_CHECK_BREAK(   fUnrestrictedGuest
                                  || pCtx->ds.Attr.n.u4Type > 11
                                  || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
                HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
                HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
                HMVMX_CHECK_BREAK(   (pCtx->ds.u32Limit & 0xfff) == 0xfff
                                  || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
                HMVMX_CHECK_BREAK(   !(pCtx->ds.u32Limit & 0xfff00000)
                                  || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
                HMVMX_CHECK_BREAK(   !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
                                  || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
            }
            if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
            {
                HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
                HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
                HMVMX_CHECK_BREAK(   fUnrestrictedGuest
                                  || pCtx->es.Attr.n.u4Type > 11
                                  || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
                HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
                HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
                HMVMX_CHECK_BREAK(   (pCtx->es.u32Limit & 0xfff) == 0xfff
                                  || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
                HMVMX_CHECK_BREAK(   !(pCtx->es.u32Limit & 0xfff00000)
                                  || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
                HMVMX_CHECK_BREAK(   !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
                                  || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
            }
            if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
            {
                HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
                HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
                HMVMX_CHECK_BREAK(   fUnrestrictedGuest
                                  || pCtx->fs.Attr.n.u4Type > 11
                                  || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
                HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
                HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
                HMVMX_CHECK_BREAK(   (pCtx->fs.u32Limit & 0xfff) == 0xfff
                                  || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
                HMVMX_CHECK_BREAK(   !(pCtx->fs.u32Limit & 0xfff00000)
                                  || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
                HMVMX_CHECK_BREAK(   !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
                                  || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
            }
            if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
            {
                HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
                HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
                HMVMX_CHECK_BREAK(   fUnrestrictedGuest
                                  || pCtx->gs.Attr.n.u4Type > 11
                                  || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
                HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
                HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
                HMVMX_CHECK_BREAK(   (pCtx->gs.u32Limit & 0xfff) == 0xfff
                                  || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
                HMVMX_CHECK_BREAK(   !(pCtx->gs.u32Limit & 0xfff00000)
                                  || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
                HMVMX_CHECK_BREAK(   !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
                                  || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
            }
            /* 64-bit capable CPUs. */
            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
            HMVMX_CHECK_BREAK(   (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
                              || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
            HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
            HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
                              VMX_IGS_LONGMODE_SS_BASE_INVALID);
            HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
                              VMX_IGS_LONGMODE_DS_BASE_INVALID);
            HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
                              VMX_IGS_LONGMODE_ES_BASE_INVALID);
        }
        else
        {
            /* V86 mode checks. */
            uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
            if (pVmcsInfo->pShared->RealMode.fRealOnV86Active)
            {
                u32CSAttr = 0xf3;   u32SSAttr = 0xf3;
                u32DSAttr = 0xf3;   u32ESAttr = 0xf3;
                u32FSAttr = 0xf3;   u32GSAttr = 0xf3;
            }
            else
            {
                u32CSAttr = pCtx->cs.Attr.u;   u32SSAttr = pCtx->ss.Attr.u;
                u32DSAttr = pCtx->ds.Attr.u;   u32ESAttr = pCtx->es.Attr.u;
                u32FSAttr = pCtx->fs.Attr.u;   u32GSAttr = pCtx->gs.Attr.u;
            }

            /* CS */
            HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
            HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
            HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
            /* SS */
            HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
            HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
            HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
            /* DS */
            HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
            HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
            HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
            /* ES */
            HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
            HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
            HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
            /* FS */
            HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
            HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
            HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
            /* GS */
            HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
            HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
            HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
            /* 64-bit capable CPUs. */
            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
            HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
            HMVMX_CHECK_BREAK(   (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
                              || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
            HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
            HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
                              VMX_IGS_LONGMODE_SS_BASE_INVALID);
            HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
                              VMX_IGS_LONGMODE_DS_BASE_INVALID);
            HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
                              VMX_IGS_LONGMODE_ES_BASE_INVALID);
        }

        /*
         * TR.
         */
        HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
        /* 64-bit capable CPUs. */
        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
        if (fLongModeGuest)
            HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11,           /* 64-bit busy TSS. */
                              VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
        else
            HMVMX_CHECK_BREAK(   pCtx->tr.Attr.n.u4Type == 3          /* 16-bit busy TSS. */
                              || pCtx->tr.Attr.n.u4Type == 11,        /* 32-bit busy TSS.*/
                              VMX_IGS_TR_ATTR_TYPE_INVALID);
        HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
        HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
        HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED);   /* Bits 11:8 MBZ. */
        HMVMX_CHECK_BREAK(   (pCtx->tr.u32Limit & 0xfff) == 0xfff
                          || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
        HMVMX_CHECK_BREAK(   !(pCtx->tr.u32Limit & 0xfff00000)
                          || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
        HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);

        /*
         * GDTR and IDTR (64-bit capable checks).
         */
        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
        AssertRC(rc);
        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);

        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
        AssertRC(rc);
        HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);

        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
        AssertRC(rc);
        HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID);      /* Bits 31:16 MBZ. */

        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
        AssertRC(rc);
        HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID);      /* Bits 31:16 MBZ. */

        /*
         * Guest Non-Register State.
         */
        /* Activity State. */
        uint32_t u32ActivityState;
        rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
        AssertRC(rc);
        HMVMX_CHECK_BREAK(   !u32ActivityState
                          || (u32ActivityState & RT_BF_GET(g_HmMsrs.u.vmx.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
                             VMX_IGS_ACTIVITY_STATE_INVALID);
        HMVMX_CHECK_BREAK(   !(pCtx->ss.Attr.n.u2Dpl)
                          || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);

        if (   u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
            || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
        {
            HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
        }

        /** @todo Activity state and injecting interrupts. Left as a todo since we
         *        currently don't use activity states but ACTIVE. */

        HMVMX_CHECK_BREAK(   !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
                          || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);

        /* Guest interruptibility-state. */
        HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
        HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
                                       != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
                          VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
        HMVMX_CHECK_BREAK(   (u32Eflags & X86_EFL_IF)
                          || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
                          VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
        if (VMX_ENTRY_INT_INFO_IS_EXT_INT(u32EntryInfo))
        {
            HMVMX_CHECK_BREAK(   !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
                              && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
                              VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
        }
        else if (VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
        {
            HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
                              VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
            HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
                              VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
        }
        /** @todo Assumes the processor is not in SMM. */
        HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
                          VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
        HMVMX_CHECK_BREAK(   !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
                          || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
                             VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
        if (   (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
            && VMX_ENTRY_INT_INFO_IS_XCPT_NMI(u32EntryInfo))
        {
            HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI), VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
        }

        /* Pending debug exceptions. */
        rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
        AssertRC(rc);
        /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
        HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
        u32Val = u64Val;    /* For pending debug exceptions checks below. */

        if (   (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
            || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
            || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
        {
            if (   (u32Eflags & X86_EFL_TF)
                && !(u64DebugCtlMsr & RT_BIT_64(1)))    /* Bit 1 is IA32_DEBUGCTL.BTF. */
            {
                /* Bit 14 is PendingDebug.BS. */
                HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
            }
            if (   !(u32Eflags & X86_EFL_TF)
                || (u64DebugCtlMsr & RT_BIT_64(1)))     /* Bit 1 is IA32_DEBUGCTL.BTF. */
            {
                /* Bit 14 is PendingDebug.BS. */
                HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
            }
        }

#ifndef IN_NEM_DARWIN
        /* VMCS link pointer. */
        rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
        AssertRC(rc);
        if (u64Val != UINT64_C(0xffffffffffffffff))
        {
            HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
            /** @todo Bits beyond the processor's physical-address width MBZ. */
            /** @todo SMM checks. */
            Assert(pVmcsInfo->HCPhysShadowVmcs == u64Val);
            Assert(pVmcsInfo->pvShadowVmcs);
            VMXVMCSREVID VmcsRevId;
            VmcsRevId.u = *(uint32_t *)pVmcsInfo->pvShadowVmcs;
            HMVMX_CHECK_BREAK(VmcsRevId.n.u31RevisionId == RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_ID),
                              VMX_IGS_VMCS_LINK_PTR_SHADOW_VMCS_ID_INVALID);
            HMVMX_CHECK_BREAK(VmcsRevId.n.fIsShadowVmcs == (uint32_t)!!(pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VMCS_SHADOWING),
                              VMX_IGS_VMCS_LINK_PTR_NOT_SHADOW);
        }

        /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
         *        not using nested paging? */
        if (   VM_IS_VMX_NESTED_PAGING(pVM)
            && !fLongModeGuest
            && CPUMIsGuestInPAEModeEx(pCtx))
        {
            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
            AssertRC(rc);
            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);

            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
            AssertRC(rc);
            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);

            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
            AssertRC(rc);
            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);

            rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
            AssertRC(rc);
            HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
        }
#endif

        /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
        if (uError == VMX_IGS_ERROR)
            uError = VMX_IGS_REASON_NOT_FOUND;
    } while (0);

    VCPU_2_VMXSTATE(pVCpu).u32HMError = uError;
    VCPU_2_VMXSTATE(pVCpu).vmx.LastError.u32GuestIntrState = u32IntrState;
    return uError;

#undef HMVMX_ERROR_BREAK
#undef HMVMX_CHECK_BREAK
}


#ifndef HMVMX_USE_FUNCTION_TABLE
/**
 * Handles a guest VM-exit from hardware-assisted VMX execution.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 */
DECLINLINE(VBOXSTRICTRC) vmxHCHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
#ifdef DEBUG_ramshankar
# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
       do { \
            if (a_fSave != 0) \
                vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__); \
            VBOXSTRICTRC rcStrict = a_CallExpr; \
            if (a_fSave != 0) \
                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST); \
            return rcStrict; \
        } while (0)
#else
# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
#endif
    uint32_t const uExitReason = pVmxTransient->uExitReason;
    switch (uExitReason)
    {
        case VMX_EXIT_EPT_MISCONFIG:           VMEXIT_CALL_RET(0, vmxHCExitEptMisconfig(pVCpu, pVmxTransient));
        case VMX_EXIT_EPT_VIOLATION:           VMEXIT_CALL_RET(0, vmxHCExitEptViolation(pVCpu, pVmxTransient));
        case VMX_EXIT_IO_INSTR:                VMEXIT_CALL_RET(0, vmxHCExitIoInstr(pVCpu, pVmxTransient));
        case VMX_EXIT_CPUID:                   VMEXIT_CALL_RET(0, vmxHCExitCpuid(pVCpu, pVmxTransient));
        case VMX_EXIT_RDTSC:                   VMEXIT_CALL_RET(0, vmxHCExitRdtsc(pVCpu, pVmxTransient));
        case VMX_EXIT_RDTSCP:                  VMEXIT_CALL_RET(0, vmxHCExitRdtscp(pVCpu, pVmxTransient));
        case VMX_EXIT_APIC_ACCESS:             VMEXIT_CALL_RET(0, vmxHCExitApicAccess(pVCpu, pVmxTransient));
        case VMX_EXIT_XCPT_OR_NMI:             VMEXIT_CALL_RET(0, vmxHCExitXcptOrNmi(pVCpu, pVmxTransient));
        case VMX_EXIT_MOV_CRX:                 VMEXIT_CALL_RET(0, vmxHCExitMovCRx(pVCpu, pVmxTransient));
        case VMX_EXIT_EXT_INT:                 VMEXIT_CALL_RET(0, vmxHCExitExtInt(pVCpu, pVmxTransient));
        case VMX_EXIT_INT_WINDOW:              VMEXIT_CALL_RET(0, vmxHCExitIntWindow(pVCpu, pVmxTransient));
        case VMX_EXIT_TPR_BELOW_THRESHOLD:     VMEXIT_CALL_RET(0, vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient));
        case VMX_EXIT_MWAIT:                   VMEXIT_CALL_RET(0, vmxHCExitMwait(pVCpu, pVmxTransient));
        case VMX_EXIT_MONITOR:                 VMEXIT_CALL_RET(0, vmxHCExitMonitor(pVCpu, pVmxTransient));
        case VMX_EXIT_TASK_SWITCH:             VMEXIT_CALL_RET(0, vmxHCExitTaskSwitch(pVCpu, pVmxTransient));
        case VMX_EXIT_PREEMPT_TIMER:           VMEXIT_CALL_RET(0, vmxHCExitPreemptTimer(pVCpu, pVmxTransient));
        case VMX_EXIT_RDMSR:                   VMEXIT_CALL_RET(0, vmxHCExitRdmsr(pVCpu, pVmxTransient));
        case VMX_EXIT_WRMSR:                   VMEXIT_CALL_RET(0, vmxHCExitWrmsr(pVCpu, pVmxTransient));
        case VMX_EXIT_VMCALL:                  VMEXIT_CALL_RET(0, vmxHCExitVmcall(pVCpu, pVmxTransient));
        case VMX_EXIT_MOV_DRX:                 VMEXIT_CALL_RET(0, vmxHCExitMovDRx(pVCpu, pVmxTransient));
        case VMX_EXIT_HLT:                     VMEXIT_CALL_RET(0, vmxHCExitHlt(pVCpu, pVmxTransient));
        case VMX_EXIT_INVD:                    VMEXIT_CALL_RET(0, vmxHCExitInvd(pVCpu, pVmxTransient));
        case VMX_EXIT_INVLPG:                  VMEXIT_CALL_RET(0, vmxHCExitInvlpg(pVCpu, pVmxTransient));
        case VMX_EXIT_MTF:                     VMEXIT_CALL_RET(0, vmxHCExitMtf(pVCpu, pVmxTransient));
        case VMX_EXIT_PAUSE:                   VMEXIT_CALL_RET(0, vmxHCExitPause(pVCpu, pVmxTransient));
        case VMX_EXIT_WBINVD:                  VMEXIT_CALL_RET(0, vmxHCExitWbinvd(pVCpu, pVmxTransient));
        case VMX_EXIT_XSETBV:                  VMEXIT_CALL_RET(0, vmxHCExitXsetbv(pVCpu, pVmxTransient));
        case VMX_EXIT_INVPCID:                 VMEXIT_CALL_RET(0, vmxHCExitInvpcid(pVCpu, pVmxTransient));
        case VMX_EXIT_GETSEC:                  VMEXIT_CALL_RET(0, vmxHCExitGetsec(pVCpu, pVmxTransient));
        case VMX_EXIT_RDPMC:                   VMEXIT_CALL_RET(0, vmxHCExitRdpmc(pVCpu, pVmxTransient));
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
        case VMX_EXIT_VMCLEAR:                 VMEXIT_CALL_RET(0, vmxHCExitVmclear(pVCpu, pVmxTransient));
        case VMX_EXIT_VMLAUNCH:                VMEXIT_CALL_RET(0, vmxHCExitVmlaunch(pVCpu, pVmxTransient));
        case VMX_EXIT_VMPTRLD:                 VMEXIT_CALL_RET(0, vmxHCExitVmptrld(pVCpu, pVmxTransient));
        case VMX_EXIT_VMPTRST:                 VMEXIT_CALL_RET(0, vmxHCExitVmptrst(pVCpu, pVmxTransient));
        case VMX_EXIT_VMREAD:                  VMEXIT_CALL_RET(0, vmxHCExitVmread(pVCpu, pVmxTransient));
        case VMX_EXIT_VMRESUME:                VMEXIT_CALL_RET(0, vmxHCExitVmwrite(pVCpu, pVmxTransient));
        case VMX_EXIT_VMWRITE:                 VMEXIT_CALL_RET(0, vmxHCExitVmresume(pVCpu, pVmxTransient));
        case VMX_EXIT_VMXOFF:                  VMEXIT_CALL_RET(0, vmxHCExitVmxoff(pVCpu, pVmxTransient));
        case VMX_EXIT_VMXON:                   VMEXIT_CALL_RET(0, vmxHCExitVmxon(pVCpu, pVmxTransient));
        case VMX_EXIT_INVVPID:                 VMEXIT_CALL_RET(0, vmxHCExitInvvpid(pVCpu, pVmxTransient));
#else
        case VMX_EXIT_VMCLEAR:
        case VMX_EXIT_VMLAUNCH:
        case VMX_EXIT_VMPTRLD:
        case VMX_EXIT_VMPTRST:
        case VMX_EXIT_VMREAD:
        case VMX_EXIT_VMRESUME:
        case VMX_EXIT_VMWRITE:
        case VMX_EXIT_VMXOFF:
        case VMX_EXIT_VMXON:
        case VMX_EXIT_INVVPID:
            return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
#endif
#ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
        case VMX_EXIT_INVEPT:                  VMEXIT_CALL_RET(0, vmxHCExitInvept(pVCpu, pVmxTransient));
#else
        case VMX_EXIT_INVEPT:                  return vmxHCExitSetPendingXcptUD(pVCpu, pVmxTransient);
#endif

        case VMX_EXIT_TRIPLE_FAULT:            return vmxHCExitTripleFault(pVCpu, pVmxTransient);
        case VMX_EXIT_NMI_WINDOW:              return vmxHCExitNmiWindow(pVCpu, pVmxTransient);
        case VMX_EXIT_ERR_INVALID_GUEST_STATE: return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);

        case VMX_EXIT_INIT_SIGNAL:
        case VMX_EXIT_SIPI:
        case VMX_EXIT_IO_SMI:
        case VMX_EXIT_SMI:
        case VMX_EXIT_ERR_MSR_LOAD:
        case VMX_EXIT_ERR_MACHINE_CHECK:
        case VMX_EXIT_PML_FULL:
        case VMX_EXIT_VIRTUALIZED_EOI:
        case VMX_EXIT_GDTR_IDTR_ACCESS:
        case VMX_EXIT_LDTR_TR_ACCESS:
        case VMX_EXIT_APIC_WRITE:
        case VMX_EXIT_RDRAND:
        case VMX_EXIT_RSM:
        case VMX_EXIT_VMFUNC:
        case VMX_EXIT_ENCLS:
        case VMX_EXIT_RDSEED:
        case VMX_EXIT_XSAVES:
        case VMX_EXIT_XRSTORS:
        case VMX_EXIT_UMWAIT:
        case VMX_EXIT_TPAUSE:
        case VMX_EXIT_LOADIWKEY:
        default:
            return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
    }
#undef VMEXIT_CALL_RET
}
#endif /* !HMVMX_USE_FUNCTION_TABLE */


#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
/**
 * Handles a nested-guest VM-exit from hardware-assisted VMX execution.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 */
DECLINLINE(VBOXSTRICTRC) vmxHCHandleExitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    uint32_t const uExitReason = pVmxTransient->uExitReason;
    switch (uExitReason)
    {
# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
        case VMX_EXIT_EPT_MISCONFIG:            return vmxHCExitEptMisconfigNested(pVCpu, pVmxTransient);
        case VMX_EXIT_EPT_VIOLATION:            return vmxHCExitEptViolationNested(pVCpu, pVmxTransient);
# else
        case VMX_EXIT_EPT_MISCONFIG:            return vmxHCExitEptMisconfig(pVCpu, pVmxTransient);
        case VMX_EXIT_EPT_VIOLATION:            return vmxHCExitEptViolation(pVCpu, pVmxTransient);
# endif
        case VMX_EXIT_XCPT_OR_NMI:              return vmxHCExitXcptOrNmiNested(pVCpu, pVmxTransient);
        case VMX_EXIT_IO_INSTR:                 return vmxHCExitIoInstrNested(pVCpu, pVmxTransient);
        case VMX_EXIT_HLT:                      return vmxHCExitHltNested(pVCpu, pVmxTransient);

        /*
         * We shouldn't direct host physical interrupts to the nested-guest.
         */
        case VMX_EXIT_EXT_INT:
            return vmxHCExitExtInt(pVCpu, pVmxTransient);

        /*
         * Instructions that cause VM-exits unconditionally or the condition is
         * always taken solely from the nested hypervisor (meaning if the VM-exit
         * happens, it's guaranteed to be a nested-guest VM-exit).
         *
         *   - Provides VM-exit instruction length ONLY.
         */
        case VMX_EXIT_CPUID:              /* Unconditional. */
        case VMX_EXIT_VMCALL:
        case VMX_EXIT_GETSEC:
        case VMX_EXIT_INVD:
        case VMX_EXIT_XSETBV:
        case VMX_EXIT_VMLAUNCH:
        case VMX_EXIT_VMRESUME:
        case VMX_EXIT_VMXOFF:
        case VMX_EXIT_ENCLS:              /* Condition specified solely by nested hypervisor. */
        case VMX_EXIT_VMFUNC:
            return vmxHCExitInstrNested(pVCpu, pVmxTransient);

        /*
         * Instructions that cause VM-exits unconditionally or the condition is
         * always taken solely from the nested hypervisor (meaning if the VM-exit
         * happens, it's guaranteed to be a nested-guest VM-exit).
         *
         *   - Provides VM-exit instruction length.
         *   - Provides VM-exit information.
         *   - Optionally provides Exit qualification.
         *
         * Since Exit qualification is 0 for all VM-exits where it is not
         * applicable, reading and passing it to the guest should produce
         * defined behavior.
         *
         * See Intel spec. 27.2.1 "Basic VM-Exit Information".
         */
        case VMX_EXIT_INVEPT:             /* Unconditional. */
        case VMX_EXIT_INVVPID:
        case VMX_EXIT_VMCLEAR:
        case VMX_EXIT_VMPTRLD:
        case VMX_EXIT_VMPTRST:
        case VMX_EXIT_VMXON:
        case VMX_EXIT_GDTR_IDTR_ACCESS:   /* Condition specified solely by nested hypervisor. */
        case VMX_EXIT_LDTR_TR_ACCESS:
        case VMX_EXIT_RDRAND:
        case VMX_EXIT_RDSEED:
        case VMX_EXIT_XSAVES:
        case VMX_EXIT_XRSTORS:
        case VMX_EXIT_UMWAIT:
        case VMX_EXIT_TPAUSE:
            return vmxHCExitInstrWithInfoNested(pVCpu, pVmxTransient);

        case VMX_EXIT_RDTSC:                    return vmxHCExitRdtscNested(pVCpu, pVmxTransient);
        case VMX_EXIT_RDTSCP:                   return vmxHCExitRdtscpNested(pVCpu, pVmxTransient);
        case VMX_EXIT_RDMSR:                    return vmxHCExitRdmsrNested(pVCpu, pVmxTransient);
        case VMX_EXIT_WRMSR:                    return vmxHCExitWrmsrNested(pVCpu, pVmxTransient);
        case VMX_EXIT_INVLPG:                   return vmxHCExitInvlpgNested(pVCpu, pVmxTransient);
        case VMX_EXIT_INVPCID:                  return vmxHCExitInvpcidNested(pVCpu, pVmxTransient);
        case VMX_EXIT_TASK_SWITCH:              return vmxHCExitTaskSwitchNested(pVCpu, pVmxTransient);
        case VMX_EXIT_WBINVD:                   return vmxHCExitWbinvdNested(pVCpu, pVmxTransient);
        case VMX_EXIT_MTF:                      return vmxHCExitMtfNested(pVCpu, pVmxTransient);
        case VMX_EXIT_APIC_ACCESS:              return vmxHCExitApicAccessNested(pVCpu, pVmxTransient);
        case VMX_EXIT_APIC_WRITE:               return vmxHCExitApicWriteNested(pVCpu, pVmxTransient);
        case VMX_EXIT_VIRTUALIZED_EOI:          return vmxHCExitVirtEoiNested(pVCpu, pVmxTransient);
        case VMX_EXIT_MOV_CRX:                  return vmxHCExitMovCRxNested(pVCpu, pVmxTransient);
        case VMX_EXIT_INT_WINDOW:               return vmxHCExitIntWindowNested(pVCpu, pVmxTransient);
        case VMX_EXIT_NMI_WINDOW:               return vmxHCExitNmiWindowNested(pVCpu, pVmxTransient);
        case VMX_EXIT_TPR_BELOW_THRESHOLD:      return vmxHCExitTprBelowThresholdNested(pVCpu, pVmxTransient);
        case VMX_EXIT_MWAIT:                    return vmxHCExitMwaitNested(pVCpu, pVmxTransient);
        case VMX_EXIT_MONITOR:                  return vmxHCExitMonitorNested(pVCpu, pVmxTransient);
        case VMX_EXIT_PAUSE:                    return vmxHCExitPauseNested(pVCpu, pVmxTransient);

        case VMX_EXIT_PREEMPT_TIMER:
        {
            /** @todo NSTVMX: Preempt timer. */
            return vmxHCExitPreemptTimer(pVCpu, pVmxTransient);
        }

        case VMX_EXIT_MOV_DRX:                  return vmxHCExitMovDRxNested(pVCpu, pVmxTransient);
        case VMX_EXIT_RDPMC:                    return vmxHCExitRdpmcNested(pVCpu, pVmxTransient);

        case VMX_EXIT_VMREAD:
        case VMX_EXIT_VMWRITE:                  return vmxHCExitVmreadVmwriteNested(pVCpu, pVmxTransient);

        case VMX_EXIT_TRIPLE_FAULT:             return vmxHCExitTripleFaultNested(pVCpu, pVmxTransient);
        case VMX_EXIT_ERR_INVALID_GUEST_STATE:  return vmxHCExitErrInvalidGuestStateNested(pVCpu, pVmxTransient);

        case VMX_EXIT_INIT_SIGNAL:
        case VMX_EXIT_SIPI:
        case VMX_EXIT_IO_SMI:
        case VMX_EXIT_SMI:
        case VMX_EXIT_ERR_MSR_LOAD:
        case VMX_EXIT_ERR_MACHINE_CHECK:
        case VMX_EXIT_PML_FULL:
        case VMX_EXIT_RSM:
        default:
            return vmxHCExitErrUnexpected(pVCpu, pVmxTransient);
    }
}
#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */


/** @name VM-exit helpers.
 * @{
 */
/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit helpers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */

/** Macro for VM-exits called unexpectedly. */
#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_HmError) \
    do { \
        VCPU_2_VMXSTATE((a_pVCpu)).u32HMError = (a_HmError); \
        return VERR_VMX_UNEXPECTED_EXIT; \
    } while (0)

#ifdef VBOX_STRICT
# ifndef IN_NEM_DARWIN
/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
    RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()

# define HMVMX_ASSERT_PREEMPT_CPUID() \
    do { \
         RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
         AssertMsg(idAssertCpu == idAssertCpuNow,  ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
    } while (0)

# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
    do { \
        AssertPtr((a_pVCpu)); \
        AssertPtr((a_pVmxTransient)); \
        Assert(   (a_pVmxTransient)->fVMEntryFailed == false \
               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE \
               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MSR_LOAD \
               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MACHINE_CHECK); \
        Assert((a_pVmxTransient)->pVmcsInfo); \
        Assert(ASMIntAreEnabled()); \
        HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
        HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
        Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
        HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
        if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
            HMVMX_ASSERT_PREEMPT_CPUID(); \
        HMVMX_STOP_EXIT_DISPATCH_PROF(); \
    } while (0)
# else
# define HMVMX_ASSERT_PREEMPT_CPUID_VAR()   do { } while(0)
# define HMVMX_ASSERT_PREEMPT_CPUID()       do { } while(0)
# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
    do { \
        AssertPtr((a_pVCpu)); \
        AssertPtr((a_pVmxTransient)); \
        Assert(   (a_pVmxTransient)->fVMEntryFailed == false \
               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE \
               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MSR_LOAD \
               || (a_pVmxTransient)->uExitReason == VMX_EXIT_ERR_MACHINE_CHECK); \
        Assert((a_pVmxTransient)->pVmcsInfo); \
        Log4Func(("vcpu[%RU32]\n", (a_pVCpu)->idCpu)); \
        HMVMX_STOP_EXIT_DISPATCH_PROF(); \
    } while (0)
# endif

# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
    do { \
        HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); \
        Assert((a_pVmxTransient)->fIsNestedGuest); \
    } while (0)

# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
    do { \
        Log4Func(("\n")); \
    } while (0)
#else
# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
    do { \
        HMVMX_STOP_EXIT_DISPATCH_PROF(); \
        NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
    } while (0)

# define HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
    do { HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient); } while (0)

# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient)      do { } while (0)
#endif

#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
/** Macro that does the necessary privilege checks and intercepted VM-exits for
 *  guests that attempted to execute a VMX instruction. */
# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
    do \
    { \
        VBOXSTRICTRC rcStrictTmp = vmxHCCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
        if (rcStrictTmp == VINF_SUCCESS) \
        { /* likely */ } \
        else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
        { \
            Assert((a_pVCpu)->hm.s.Event.fPending); \
            Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
            return VINF_SUCCESS; \
        } \
        else \
        { \
            int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
            AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
        } \
    } while (0)

/** Macro that decodes a memory operand for an VM-exit caused by an instruction. */
# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
    do \
    { \
        VBOXSTRICTRC rcStrictTmp = vmxHCDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
                                                           (a_pGCPtrEffAddr)); \
        if (rcStrictTmp == VINF_SUCCESS) \
        { /* likely */ } \
        else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
        { \
            uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
            Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
            NOREF(uXcptTmp); \
            return VINF_SUCCESS; \
        } \
        else \
        { \
            Log4Func(("vmxHCDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
            return rcStrictTmp; \
        } \
    } while (0)
#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */


/**
 * Advances the guest RIP by the specified number of bytes.
 *
 * @param   pVCpu       The cross context virtual CPU structure.
 * @param   cbInstr     Number of bytes to advance the RIP by.
 *
 * @remarks No-long-jump zone!!!
 */
DECLINLINE(void) vmxHCAdvanceGuestRipBy(PVMCPUCC pVCpu, uint32_t cbInstr)
{
    CPUM_ASSERT_NOT_EXTRN(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI);

    /*
     * Advance RIP.
     *
     * The upper 32 bits are only set when in 64-bit mode, so we have to detect
     * when the addition causes a "carry" into the upper half and check whether
     * we're in 64-bit and can go on with it or wether we should zap the top
     * half. (Note! The 8086, 80186 and 80286 emulation is done exclusively in
     * IEM, so we don't need to bother with pre-386 16-bit wraparound.)
     *
     * See PC wrap around tests in bs3-cpu-weird-1.
     */
    uint64_t const uRipPrev = pVCpu->cpum.GstCtx.rip;
    uint64_t const uRipNext = uRipPrev + cbInstr;
    if (RT_LIKELY(   !((uRipNext ^ uRipPrev) & RT_BIT_64(32))
                  || CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
        pVCpu->cpum.GstCtx.rip = uRipNext;
    else
        pVCpu->cpum.GstCtx.rip = (uint32_t)uRipNext;

    /*
     * Clear RF and interrupt shadowing.
     */
    if (RT_LIKELY(!(pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_RF | X86_EFL_TF))))
        pVCpu->cpum.GstCtx.eflags.uBoth &= ~CPUMCTX_INHIBIT_SHADOW;
    else
    {
        if ((pVCpu->cpum.GstCtx.eflags.uBoth & (X86_EFL_RF | X86_EFL_TF)) == X86_EFL_TF)
        {
            /** @todo \#DB - single step. */
        }
        pVCpu->cpum.GstCtx.eflags.uBoth &= ~(X86_EFL_RF | CPUMCTX_INHIBIT_SHADOW);
    }
    AssertCompile(CPUMCTX_INHIBIT_SHADOW < UINT32_MAX);

    /* Mark both RIP and RFLAGS as updated. */
    ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
}


/**
 * Advances the guest RIP after reading it from the VMCS.
 *
 * @returns VBox status code, no informational status codes.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 *
 * @remarks No-long-jump zone!!!
 */
static int vmxHCAdvanceGuestRip(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    vmxHCReadToTransientSlow<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    /** @todo consider template here after checking callers.   */
    int rc = vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
    AssertRCReturn(rc, rc);

    vmxHCAdvanceGuestRipBy(pVCpu, pVmxTransient->cbExitInstr);
    return VINF_SUCCESS;
}


/**
 * Handle a condition that occurred while delivering an event through the guest or
 * nested-guest IDT.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @retval  VINF_SUCCESS if we should continue handling the VM-exit.
 * @retval  VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
 *          to continue execution of the guest which will delivery the \#DF.
 * @retval  VINF_EM_RESET if we detected a triple-fault condition.
 * @retval  VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 *
 * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
 *          Additionally, HMVMX_READ_EXIT_QUALIFICATION is required if the VM-exit
 *          is due to an EPT violation, PML full or SPP-related event.
 *
 * @remarks No-long-jump zone!!!
 */
static VBOXSTRICTRC vmxHCCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    Assert(!VCPU_2_VMXSTATE(pVCpu).Event.fPending);
    HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);
    if (   pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
        || pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
        || pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
        HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_EXIT_QUALIFICATION);

    VBOXSTRICTRC   rcStrict       = VINF_SUCCESS;
    PCVMXVMCSINFO  pVmcsInfo      = pVmxTransient->pVmcsInfo;
    uint32_t const uIdtVectorInfo = pVmxTransient->uIdtVectoringInfo;
    uint32_t const uExitIntInfo   = pVmxTransient->uExitIntInfo;
    if (VMX_IDT_VECTORING_INFO_IS_VALID(uIdtVectorInfo))
    {
        uint32_t const uIdtVector     = VMX_IDT_VECTORING_INFO_VECTOR(uIdtVectorInfo);
        uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(uIdtVectorInfo);

        /*
         * If the event was a software interrupt (generated with INT n) or a software exception
         * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
         * can handle the VM-exit and continue guest execution which will re-execute the
         * instruction rather than re-injecting the exception, as that can cause premature
         * trips to ring-3 before injection and involve TRPM which currently has no way of
         * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
         * the problem).
         */
        IEMXCPTRAISE     enmRaise;
        IEMXCPTRAISEINFO fRaiseInfo;
        if (   uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
            || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
            || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
        {
            enmRaise   = IEMXCPTRAISE_REEXEC_INSTR;
            fRaiseInfo = IEMXCPTRAISEINFO_NONE;
        }
        else if (VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo))
        {
            uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
            uint8_t const  uExitVector     = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
            Assert(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT);

            uint32_t const fIdtVectorFlags  = vmxHCGetIemXcptFlags(uIdtVector, uIdtVectorType);
            uint32_t const fExitVectorFlags = vmxHCGetIemXcptFlags(uExitVector, uExitVectorType);

            enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);

            /* Determine a vectoring #PF condition, see comment in vmxHCExitXcptPF(). */
            if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
            {
                pVmxTransient->fVectoringPF = true;
                enmRaise = IEMXCPTRAISE_PREV_EVENT;
            }
        }
        else
        {
            /*
             * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
             * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
             * It is sufficient to reflect the original event to the guest after handling the VM-exit.
             */
            Assert(   uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
                   || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
                   || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
            enmRaise   = IEMXCPTRAISE_PREV_EVENT;
            fRaiseInfo = IEMXCPTRAISEINFO_NONE;
        }

        /*
         * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
         * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
         * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
         * subsequent VM-entry would fail, see @bugref{7445}.
         *
         * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception".
         */
        if (   uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
            && enmRaise == IEMXCPTRAISE_PREV_EVENT
            && (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
            && CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx))
            CPUMClearInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);

        switch (enmRaise)
        {
            case IEMXCPTRAISE_CURRENT_XCPT:
            {
                Log4Func(("IDT: Pending secondary Xcpt: idtinfo=%#RX64 exitinfo=%#RX64\n", uIdtVectorInfo, uExitIntInfo));
                Assert(rcStrict == VINF_SUCCESS);
                break;
            }

            case IEMXCPTRAISE_PREV_EVENT:
            {
                uint32_t u32ErrCode;
                if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(uIdtVectorInfo))
                    u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
                else
                    u32ErrCode = 0;

                /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see vmxHCExitXcptPF(). */
                STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflect);
                vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(uIdtVectorInfo), 0 /* cbInstr */, u32ErrCode,
                                     pVCpu->cpum.GstCtx.cr2);

                Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
                          VCPU_2_VMXSTATE(pVCpu).Event.u32ErrCode));
                Assert(rcStrict == VINF_SUCCESS);
                break;
            }

            case IEMXCPTRAISE_REEXEC_INSTR:
                Assert(rcStrict == VINF_SUCCESS);
                break;

            case IEMXCPTRAISE_DOUBLE_FAULT:
            {
                /*
                 * Determine a vectoring double #PF condition. Used later, when PGM evaluates the
                 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
                 */
                if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
                {
                    pVmxTransient->fVectoringDoublePF = true;
                    Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
                          pVCpu->cpum.GstCtx.cr2));
                    rcStrict = VINF_SUCCESS;
                }
                else
                {
                    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectConvertDF);
                    vmxHCSetPendingXcptDF(pVCpu);
                    Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", VCPU_2_VMXSTATE(pVCpu).Event.u64IntInfo,
                              uIdtVector, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
                    rcStrict = VINF_HM_DOUBLE_FAULT;
                }
                break;
            }

            case IEMXCPTRAISE_TRIPLE_FAULT:
            {
                Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector,
                          VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo)));
                rcStrict = VINF_EM_RESET;
                break;
            }

            case IEMXCPTRAISE_CPU_HANG:
            {
                Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
                rcStrict = VERR_EM_GUEST_CPU_HANG;
                break;
            }

            default:
            {
                AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
                rcStrict = VERR_VMX_IPE_2;
                break;
            }
        }
    }
    else if (   (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
             && !CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx))
    {
        if (    VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo)
             && VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo) != X86_XCPT_DF
             && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(uExitIntInfo))
        {
            /*
             * Execution of IRET caused a fault when NMI blocking was in effect (i.e we're in
             * the guest or nested-guest NMI handler). We need to set the block-by-NMI field so
             * that virtual NMIs remain blocked until the IRET execution is completed.
             *
             * See Intel spec. 31.7.1.2 "Resuming Guest Software After Handling An Exception".
             */
            CPUMSetInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
            Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
        }
        else if (   pVmxTransient->uExitReason == VMX_EXIT_EPT_VIOLATION
                 || pVmxTransient->uExitReason == VMX_EXIT_PML_FULL
                 || pVmxTransient->uExitReason == VMX_EXIT_SPP_EVENT)
        {
            /*
             * Execution of IRET caused an EPT violation, page-modification log-full event or
             * SPP-related event VM-exit when NMI blocking was in effect (i.e. we're in the
             * guest or nested-guest NMI handler). We need to set the block-by-NMI field so
             * that virtual NMIs remain blocked until the IRET execution is completed.
             *
             * See Intel spec. 27.2.3 "Information about NMI unblocking due to IRET"
             */
            if (VMX_EXIT_QUAL_EPT_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitQual))
            {
                CPUMSetInterruptInhibitingByNmiEx(&pVCpu->cpum.GstCtx);
                Log4Func(("Set NMI blocking. uExitReason=%u\n", pVmxTransient->uExitReason));
            }
        }
    }

    Assert(   rcStrict == VINF_SUCCESS  || rcStrict == VINF_HM_DOUBLE_FAULT
           || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
    return rcStrict;
}


#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
/**
 * Perform the relevant VMX instruction checks for VM-exits that occurred due to the
 * guest attempting to execute a VMX instruction.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @retval  VINF_SUCCESS if we should continue handling the VM-exit.
 * @retval  VINF_HM_PENDING_XCPT if an exception was raised.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   uExitReason     The VM-exit reason.
 *
 * @todo    NSTVMX: Document other error codes when VM-exit is implemented.
 * @remarks No-long-jump zone!!!
 */
static VBOXSTRICTRC vmxHCCheckExitDueToVmxInstr(PVMCPUCC pVCpu, uint32_t uExitReason)
{
    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS
                              | CPUMCTX_EXTRN_CS  | CPUMCTX_EXTRN_EFER);

    /*
     * The physical CPU would have already checked the CPU mode/code segment.
     * We shall just assert here for paranoia.
     * See Intel spec. 25.1.1 "Relative Priority of Faults and VM Exits".
     */
    Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
    Assert(   !CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
           ||  CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx));

    if (uExitReason == VMX_EXIT_VMXON)
    {
        HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);

        /*
         * We check CR4.VMXE because it is required to be always set while in VMX operation
         * by physical CPUs and our CR4 read-shadow is only consulted when executing specific
         * instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
         * otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
         */
        if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
        {
            Log4Func(("CR4.VMXE is not set -> #UD\n"));
            vmxHCSetPendingXcptUD(pVCpu);
            return VINF_HM_PENDING_XCPT;
        }
    }
    else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
    {
        /*
         * The guest has not entered VMX operation but attempted to execute a VMX instruction
         * (other than VMXON), we need to raise a #UD.
         */
        Log4Func(("Not in VMX root mode -> #UD\n"));
        vmxHCSetPendingXcptUD(pVCpu);
        return VINF_HM_PENDING_XCPT;
    }

    /* All other checks (including VM-exit intercepts) are handled by IEM instruction emulation. */
    return VINF_SUCCESS;
}


/**
 * Decodes the memory operand of an instruction that caused a VM-exit.
 *
 * The Exit qualification field provides the displacement field for memory
 * operand instructions, if any.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @retval  VINF_SUCCESS if the operand was successfully decoded.
 * @retval  VINF_HM_PENDING_XCPT if an exception was raised while decoding the
 *          operand.
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   uExitInstrInfo  The VM-exit instruction information field.
 * @param   enmMemAccess    The memory operand's access type (read or write).
 * @param   GCPtrDisp       The instruction displacement field, if any. For
 *                          RIP-relative addressing pass RIP + displacement here.
 * @param   pGCPtrMem       Where to store the effective destination memory address.
 *
 * @remarks Warning! This function ASSUMES the instruction cannot be used in real or
 *          virtual-8086 mode hence skips those checks while verifying if the
 *          segment is valid.
 */
static VBOXSTRICTRC vmxHCDecodeMemOperand(PVMCPUCC pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
                                            PRTGCPTR pGCPtrMem)
{
    Assert(pGCPtrMem);
    Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER
                              | CPUMCTX_EXTRN_CR0);

    static uint64_t const s_auAddrSizeMasks[]   = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
    static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
    AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));

    VMXEXITINSTRINFO ExitInstrInfo;
    ExitInstrInfo.u = uExitInstrInfo;
    uint8_t const   uAddrSize     =  ExitInstrInfo.All.u3AddrSize;
    uint8_t const   iSegReg       =  ExitInstrInfo.All.iSegReg;
    bool const      fIdxRegValid  = !ExitInstrInfo.All.fIdxRegInvalid;
    uint8_t const   iIdxReg       =  ExitInstrInfo.All.iIdxReg;
    uint8_t const   uScale        =  ExitInstrInfo.All.u2Scaling;
    bool const      fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
    uint8_t const   iBaseReg      =  ExitInstrInfo.All.iBaseReg;
    bool const      fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
    bool const      fIsLongMode   =  CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);

    /*
     * Validate instruction information.
     * This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
     */
    AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
                          ("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
    AssertLogRelMsgReturn(iSegReg  < X86_SREG_COUNT,
                          ("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
    AssertLogRelMsgReturn(fIsMemOperand,
                          ("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);

    /*
     * Compute the complete effective address.
     *
     * See AMD instruction spec. 1.4.2 "SIB Byte Format"
     * See AMD spec. 4.5.2 "Segment Registers".
     */
    RTGCPTR GCPtrMem = GCPtrDisp;
    if (fBaseRegValid)
        GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
    if (fIdxRegValid)
        GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;

    RTGCPTR const GCPtrOff = GCPtrMem;
    if (   !fIsLongMode
        || iSegReg >= X86_SREG_FS)
        GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
    GCPtrMem &= s_auAddrSizeMasks[uAddrSize];

    /*
     * Validate effective address.
     * See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
     */
    uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
    Assert(cbAccess > 0);
    if (fIsLongMode)
    {
        if (X86_IS_CANONICAL(GCPtrMem))
        {
            *pGCPtrMem = GCPtrMem;
            return VINF_SUCCESS;
        }

        /** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
         *        "Data Limit Checks in 64-bit Mode". */
        Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
        vmxHCSetPendingXcptGP(pVCpu, 0);
        return VINF_HM_PENDING_XCPT;
    }

    /*
     * This is a watered down version of iemMemApplySegment().
     * Parts that are not applicable for VMX instructions like real-or-v8086 mode
     * and segment CPL/DPL checks are skipped.
     */
    RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
    RTGCPTR32 const GCPtrLast32  = GCPtrFirst32 + cbAccess - 1;
    PCCPUMSELREG    pSel         = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];

    /* Check if the segment is present and usable. */
    if (    pSel->Attr.n.u1Present
        && !pSel->Attr.n.u1Unusable)
    {
        Assert(pSel->Attr.n.u1DescType);
        if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
        {
            /* Check permissions for the data segment. */
            if (   enmMemAccess == VMXMEMACCESS_WRITE
                && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
            {
                Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
                vmxHCSetPendingXcptGP(pVCpu, iSegReg);
                return VINF_HM_PENDING_XCPT;
            }

            /* Check limits if it's a normal data segment. */
            if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
            {
                if (   GCPtrFirst32 > pSel->u32Limit
                    || GCPtrLast32  > pSel->u32Limit)
                {
                    Log4Func(("Data segment limit exceeded. "
                              "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
                              GCPtrLast32, pSel->u32Limit));
                    if (iSegReg == X86_SREG_SS)
                        vmxHCSetPendingXcptSS(pVCpu, 0);
                    else
                        vmxHCSetPendingXcptGP(pVCpu, 0);
                    return VINF_HM_PENDING_XCPT;
                }
            }
            else
            {
               /* Check limits if it's an expand-down data segment.
                  Note! The upper boundary is defined by the B bit, not the G bit! */
               if (   GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
                   || GCPtrLast32  > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
               {
                   Log4Func(("Expand-down data segment limit exceeded. "
                             "iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
                             GCPtrLast32, pSel->u32Limit));
                   if (iSegReg == X86_SREG_SS)
                       vmxHCSetPendingXcptSS(pVCpu, 0);
                   else
                       vmxHCSetPendingXcptGP(pVCpu, 0);
                   return VINF_HM_PENDING_XCPT;
               }
            }
        }
        else
        {
            /* Check permissions for the code segment. */
            if (   enmMemAccess == VMXMEMACCESS_WRITE
                || (   enmMemAccess == VMXMEMACCESS_READ
                    && !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
            {
                Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
                Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
                vmxHCSetPendingXcptGP(pVCpu, 0);
                return VINF_HM_PENDING_XCPT;
            }

            /* Check limits for the code segment (normal/expand-down not applicable for code segments). */
            if (   GCPtrFirst32 > pSel->u32Limit
                || GCPtrLast32  > pSel->u32Limit)
            {
                Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
                          GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
                if (iSegReg == X86_SREG_SS)
                    vmxHCSetPendingXcptSS(pVCpu, 0);
                else
                    vmxHCSetPendingXcptGP(pVCpu, 0);
                return VINF_HM_PENDING_XCPT;
            }
        }
    }
    else
    {
        Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
        vmxHCSetPendingXcptGP(pVCpu, 0);
        return VINF_HM_PENDING_XCPT;
    }

    *pGCPtrMem = GCPtrMem;
    return VINF_SUCCESS;
}
#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */


/**
 * VM-exit helper for LMSW.
 */
static VBOXSTRICTRC vmxHCExitLmsw(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint16_t uMsw, RTGCPTR GCPtrEffDst)
{
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedLmsw(pVCpu, cbInstr, uMsw, GCPtrEffDst);
    AssertMsg(   rcStrict == VINF_SUCCESS
              || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));

    ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
    if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }

    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitLmsw);
    Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
    return rcStrict;
}


/**
 * VM-exit helper for CLTS.
 */
static VBOXSTRICTRC vmxHCExitClts(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr)
{
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedClts(pVCpu, cbInstr);
    AssertMsg(   rcStrict == VINF_SUCCESS
              || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));

    ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
    if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }

    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitClts);
    Log4Func(("rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
    return rcStrict;
}


/**
 * VM-exit helper for MOV from CRx (CRx read).
 */
static VBOXSTRICTRC vmxHCExitMovFromCrX(PVMCPUCC pVCpu, PVMXVMCSINFO pVmcsInfo, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
{
    Assert(iCrReg < 16);
    Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));

    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
    AssertMsg(   rcStrict == VINF_SUCCESS
              || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));

    if (iGReg == X86_GREG_xSP)
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
    else
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
#ifdef VBOX_WITH_STATISTICS
    switch (iCrReg)
    {
        case 0: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Read); break;
        case 2: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Read); break;
        case 3: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Read); break;
        case 4: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Read); break;
        case 8: STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Read); break;
    }
#endif
    Log4Func(("CR%d Read access rcStrict=%Rrc\n", iCrReg, VBOXSTRICTRC_VAL(rcStrict)));
    return rcStrict;
}


/**
 * VM-exit helper for MOV to CRx (CRx write).
 */
static VBOXSTRICTRC vmxHCExitMovToCrX(PVMCPUCC pVCpu, uint8_t cbInstr, uint8_t iGReg, uint8_t iCrReg)
{
    HMVMX_CPUMCTX_ASSERT(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);

    VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
    AssertMsg(   rcStrict == VINF_SUCCESS
              || rcStrict == VINF_IEM_RAISED_XCPT
              || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));

    switch (iCrReg)
    {
        case 0:
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0
                                                     | HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR0Write);
            Log4Func(("CR0 write. rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
            break;

        case 2:
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR2Write);
            /* Nothing to do here, CR2 it's not part of the VMCS. */
            break;

        case 3:
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR3Write);
            Log4Func(("CR3 write. rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
            break;

        case 4:
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR4Write);
#ifndef IN_NEM_DARWIN
            Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
                      pVCpu->cpum.GstCtx.cr4, pVCpu->hmr0.s.fLoadSaveGuestXcr0));
#else
            Log4Func(("CR4 write. rc=%Rrc CR4=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr4));
#endif
            break;

        case 8:
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
                             HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitCR8Write);
            break;

        default:
            AssertMsgFailed(("Invalid CRx register %#x\n", iCrReg));
            break;
    }

    if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit exception handler for \#PF (Page-fault exception).
 *
 * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
 */
static VBOXSTRICTRC vmxHCExitXcptPF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);

#ifndef IN_NEM_DARWIN
    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
    if (!VM_IS_VMX_NESTED_PAGING(pVM))
    { /* likely */ }
    else
#endif
    {
#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF) && !defined(IN_NEM_DARWIN)
        Assert(pVmxTransient->fIsNestedGuest || pVCpu->hmr0.s.fUsingDebugLoop);
#endif
        VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;                  /* In case it's a contributory or vectoring #PF. */
        if (!pVmxTransient->fVectoringDoublePF)
        {
            vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
                                   pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
        }
        else
        {
            /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
            Assert(!pVmxTransient->fIsNestedGuest);
            vmxHCSetPendingXcptDF(pVCpu);
            Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
        }
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
        return VINF_SUCCESS;
    }

    Assert(!pVmxTransient->fIsNestedGuest);

    /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
       of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
    if (pVmxTransient->fVectoringPF)
    {
        Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
        return VINF_EM_RAW_INJECT_TRPM_EVENT;
    }

    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    Log4Func(("#PF: cs:rip=%#04x:%08RX64 err_code=%#RX32 exit_qual=%#RX64 cr3=%#RX64\n", pVCpu->cpum.GstCtx.cs.Sel,
              pVCpu->cpum.GstCtx.rip, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual, pVCpu->cpum.GstCtx.cr3));

    TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
    rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, &pVCpu->cpum.GstCtx, (RTGCPTR)pVmxTransient->uExitQual);

    Log4Func(("#PF: rc=%Rrc\n", rc));
    if (rc == VINF_SUCCESS)
    {
        /*
         * This is typically a shadow page table sync or a MMIO instruction. But we may have
         * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
         */
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
        TRPMResetTrap(pVCpu);
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPF);
        return rc;
    }

    if (rc == VINF_EM_RAW_GUEST_TRAP)
    {
        if (!pVmxTransient->fVectoringDoublePF)
        {
            /* It's a guest page fault and needs to be reflected to the guest. */
            uint32_t const uGstErrorCode = TRPMGetErrorCode(pVCpu);
            TRPMResetTrap(pVCpu);
            VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;                 /* In case it's a contributory #PF. */
            vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
                                   uGstErrorCode, pVmxTransient->uExitQual);
        }
        else
        {
            /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
            TRPMResetTrap(pVCpu);
            VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;     /* Clear pending #PF to replace it with #DF. */
            vmxHCSetPendingXcptDF(pVCpu);
            Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
        }

        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);
        return VINF_SUCCESS;
    }

    TRPMResetTrap(pVCpu);
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitShadowPFEM);
    return rc;
}


/**
 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
 *
 * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
 */
static VBOXSTRICTRC vmxHCExitXcptMF(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF);

    int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CR0>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
    {
        /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
        rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);

        /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
         *        provides VM-exit instruction length. If this causes problem later,
         *        disassemble the instruction like it's done on AMD-V. */
        int rc2 = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
        AssertRCReturn(rc2, rc2);
        return rc;
    }

    vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbExitInstr,
                           pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
    return VINF_SUCCESS;
}


/**
 * VM-exit exception handler for \#BP (Breakpoint exception).
 *
 * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
 */
static VBOXSTRICTRC vmxHCExitXcptBP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP);

    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict;
    if (!pVmxTransient->fIsNestedGuest)
        rcStrict = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx);
    else
        rcStrict = VINF_EM_RAW_GUEST_TRAP;

    if (rcStrict == VINF_EM_RAW_GUEST_TRAP)
    {
        vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
                               pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
        rcStrict = VINF_SUCCESS;
    }

    Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_DBG_BREAKPOINT);
    return rcStrict;
}


/**
 * VM-exit exception handler for \#AC (Alignment-check exception).
 *
 * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
 */
static VBOXSTRICTRC vmxHCExitXcptAC(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /*
     * Detect #ACs caused by host having enabled split-lock detection.
     * Emulate such instructions.
     */
#define VMX_HC_EXIT_XCPT_AC_INITIAL_REGS    (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_CS)
    int rc = vmxHCImportGuestState<VMX_HC_EXIT_XCPT_AC_INITIAL_REGS>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);
    /** @todo detect split lock in cpu feature?   */
    if (   /* 1. If 486-style alignment checks aren't enabled, then this must be a split-lock exception */
           !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_AM)
           /* 2. #AC cannot happen in rings 0-2 except for split-lock detection. */
        || CPUMGetGuestCPL(pVCpu) != 3
           /* 3. When the EFLAGS.AC != 0 this can only be a split-lock case. */
        || !(pVCpu->cpum.GstCtx.eflags.u & X86_EFL_AC) )
    {
        /*
         * Check for debug/trace events and import state accordingly.
         */
        STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestACSplitLock);
        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
        if (   !DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK)
#ifndef IN_NEM_DARWIN
            && !VBOXVMM_VMX_SPLIT_LOCK_ENABLED()
#endif
            )
        {
            if (pVM->cCpus == 1)
            {
#if 0 /** @todo r=bird: This is potentially wrong.  Might have to just do a whole state sync above and mark everything changed to be safe... */
                rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK,
                                           VMX_HC_EXIT_XCPT_AC_INITIAL_REGS>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
#else
                rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
                                           VMX_HC_EXIT_XCPT_AC_INITIAL_REGS>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
#endif
                AssertRCReturn(rc, rc);
            }
        }
        else
        {
            rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
                                       VMX_HC_EXIT_XCPT_AC_INITIAL_REGS>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
            AssertRCReturn(rc, rc);

            VBOXVMM_XCPT_DF(pVCpu, &pVCpu->cpum.GstCtx);

            if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_VMX_SPLIT_LOCK))
            {
                VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, DBGFEVENT_VMX_SPLIT_LOCK, DBGFEVENTCTX_HM, 0);
                if (rcStrict != VINF_SUCCESS)
                    return rcStrict;
            }
        }

        /*
         * Emulate the instruction.
         *
         * We have to ignore the LOCK prefix here as we must not retrigger the
         * detection on the host.  This isn't all that satisfactory, though...
         */
        if (pVM->cCpus == 1)
        {
            Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC\n", pVCpu->cpum.GstCtx.cs.Sel,
                      pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));

            /** @todo For SMP configs we should do a rendezvous here. */
            VBOXSTRICTRC rcStrict = IEMExecOneIgnoreLock(pVCpu);
            if (rcStrict == VINF_SUCCESS)
#if 0 /** @todo r=bird: This is potentially wrong.  Might have to just do a whole state sync above and mark everything changed to be safe... */
                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged,
                                   HM_CHANGED_GUEST_RIP
                                 | HM_CHANGED_GUEST_RFLAGS
                                 | HM_CHANGED_GUEST_GPRS_MASK
                                 | HM_CHANGED_GUEST_CS
                                 | HM_CHANGED_GUEST_SS);
#else
                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
#endif
            else if (rcStrict == VINF_IEM_RAISED_XCPT)
            {
                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
                rcStrict = VINF_SUCCESS;
            }
            return rcStrict;
        }
        Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 split-lock #AC -> VINF_EM_EMULATE_SPLIT_LOCK\n",
                  pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0));
        return VINF_EM_EMULATE_SPLIT_LOCK;
    }

    STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC);
    Log8Func(("cs:rip=%#04x:%08RX64 rflags=%#RX64 cr0=%#RX64 cpl=%d -> #AC\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
              pVCpu->cpum.GstCtx.rflags, pVCpu->cpum.GstCtx.cr0, CPUMGetGuestCPL(pVCpu) ));

    /* Re-inject it. We'll detect any nesting before getting here. */
    vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
                           pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
    return VINF_SUCCESS;
}


/**
 * VM-exit exception handler for \#DB (Debug exception).
 *
 * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
 */
static VBOXSTRICTRC vmxHCExitXcptDB(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB);

    /*
     * Get the DR6-like values from the Exit qualification and pass it to DBGF for processing.
     */
    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);

    /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
    uint64_t const uDR6 = X86_DR6_INIT_VAL
                        | (pVmxTransient->uExitQual & (  X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3
                                                       | X86_DR6_BD | X86_DR6_BS));
    Log6Func(("uDR6=%#RX64 uExitQual=%#RX64\n", uDR6, pVmxTransient->uExitQual));

    int rc;
    if (!pVmxTransient->fIsNestedGuest)
    {
        rc = DBGFTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, &pVCpu->cpum.GstCtx, uDR6, VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);

        /*
         * Prevents stepping twice over the same instruction when the guest is stepping using
         * EFLAGS.TF and the hypervisor debugger is stepping using MTF.
         * Testcase: DOSQEMM, break (using "ba x 1") at cs:rip 0x70:0x774 and step (using "t").
         */
        if (   rc == VINF_EM_DBG_STEPPED
            && (pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG))
        {
            Assert(VCPU_2_VMXSTATE(pVCpu).fSingleInstruction);
            rc = VINF_EM_RAW_GUEST_TRAP;
        }
    }
    else
        rc = VINF_EM_RAW_GUEST_TRAP;
    Log6Func(("rc=%Rrc\n", rc));
    if (rc == VINF_EM_RAW_GUEST_TRAP)
    {
        /*
         * The exception was for the guest.  Update DR6, DR7.GD and
         * IA32_DEBUGCTL.LBR before forwarding it.
         * See Intel spec. 27.1 "Architectural State before a VM-Exit"
         * and @sdmv3{077,622,17.2.3,Debug Status Register (DR6)}.
         */
#ifndef IN_NEM_DARWIN
        VMMRZCallRing3Disable(pVCpu);
        HM_DISABLE_PREEMPT(pVCpu);

        pVCpu->cpum.GstCtx.dr[6] &= ~X86_DR6_B_MASK;
        pVCpu->cpum.GstCtx.dr[6] |= uDR6;
        if (CPUMIsGuestDebugStateActive(pVCpu))
            ASMSetDR6(pVCpu->cpum.GstCtx.dr[6]);

        HM_RESTORE_PREEMPT();
        VMMRZCallRing3Enable(pVCpu);
#else
        /** @todo */
#endif

        rc = vmxHCImportGuestState<CPUMCTX_EXTRN_DR7>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
        AssertRCReturn(rc, rc);

        /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
        pVCpu->cpum.GstCtx.dr[7] &= ~(uint64_t)X86_DR7_GD;

        /* Paranoia. */
        pVCpu->cpum.GstCtx.dr[7] &= ~(uint64_t)X86_DR7_RAZ_MASK;
        pVCpu->cpum.GstCtx.dr[7] |= X86_DR7_RA1_MASK;

        rc = VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_GUEST_DR7, pVCpu->cpum.GstCtx.dr[7]);
        AssertRC(rc);

        /*
         * Raise #DB in the guest.
         *
         * It is important to reflect exactly what the VM-exit gave us (preserving the
         * interruption-type) rather than use vmxHCSetPendingXcptDB() as the #DB could've
         * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
         * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
         *
         * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
         * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
         */
        vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
                             pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
        return VINF_SUCCESS;
    }

    /*
     * Not a guest trap, must be a hypervisor related debug event then.
     * Update DR6 in case someone is interested in it.
     */
    AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
    AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
    CPUMSetHyperDR6(pVCpu, uDR6);

    return rc;
}


/**
 * Hacks its way around the lovely mesa driver's backdoor accesses.
 *
 * @sa hmR0SvmHandleMesaDrvGp.
 */
static int vmxHCHandleMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
{
    LogFunc(("cs:rip=%#04x:%08RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
    RT_NOREF(pCtx);

    /* For now we'll just skip the instruction. */
    return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
}


/**
 * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
 * backdoor logging w/o checking what it is running inside.
 *
 * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
 * backdoor port and magic numbers loaded in registers.
 *
 * @returns true if it is, false if it isn't.
 * @sa      hmR0SvmIsMesaDrvGp.
 */
DECLINLINE(bool) vmxHCIsMesaDrvGp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
{
    /* 0xed:  IN eAX,dx */
    uint8_t abInstr[1];
    if (pVmxTransient->cbExitInstr != sizeof(abInstr))
        return false;

    /* Check that it is #GP(0). */
    if (pVmxTransient->uExitIntErrorCode != 0)
        return false;

    /* Check magic and port. */
    Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX)));
    /*Log(("vmxHCIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));*/
    if (pCtx->rax != UINT32_C(0x564d5868))
        return false;
    if (pCtx->dx != UINT32_C(0x5658))
        return false;

    /* Flat ring-3 CS. */
    AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
    Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
    /*Log(("vmxHCIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));*/
    if (pCtx->cs.Attr.n.u2Dpl != 3)
        return false;
    if (pCtx->cs.u64Base != 0)
        return false;

    /* Check opcode. */
    AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
    Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
    int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
    /*Log(("vmxHCIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));*/
    if (RT_FAILURE(rc))
        return false;
    if (abInstr[0] != 0xed)
        return false;

    return true;
}


/**
 * VM-exit exception handler for \#GP (General-protection exception).
 *
 * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
 */
static VBOXSTRICTRC vmxHCExitXcptGP(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP);

    PCPUMCTX            pCtx            = &pVCpu->cpum.GstCtx;
    PVMXVMCSINFO        pVmcsInfo       = pVmxTransient->pVmcsInfo;
#ifndef IN_NEM_DARWIN
    PVMXVMCSINFOSHARED  pVmcsInfoShared = pVmcsInfo->pShared;
    if (pVmcsInfoShared->RealMode.fRealOnV86Active)
    { /* likely */ }
    else
#endif
    {
#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
# ifndef IN_NEM_DARWIN
        Assert(pVCpu->hmr0.s.fUsingDebugLoop || VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv || pVmxTransient->fIsNestedGuest);
# else
        Assert(/*pVCpu->hmr0.s.fUsingDebugLoop ||*/ VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv || pVmxTransient->fIsNestedGuest);
# endif
#endif
        /*
         * If the guest is not in real-mode or we have unrestricted guest execution support, or if we are
         * executing a nested-guest, reflect #GP to the guest or nested-guest.
         */
        int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
        AssertRCReturn(rc, rc);
        Log4Func(("Gst: cs:rip=%#04x:%08RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
                  pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));

        if (    pVmxTransient->fIsNestedGuest
            || !VCPU_2_VMXSTATE(pVCpu).fTrapXcptGpForLovelyMesaDrv
            || !vmxHCIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
            vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
                                   pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
        else
            rc = vmxHCHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
        return rc;
    }

#ifndef IN_NEM_DARWIN
    Assert(CPUMIsGuestInRealModeEx(pCtx));
    Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fUnrestrictedGuest);
    Assert(!pVmxTransient->fIsNestedGuest);

    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
    if (rcStrict == VINF_SUCCESS)
    {
        if (!CPUMIsGuestInRealModeEx(pCtx))
        {
            /*
             * The guest is no longer in real-mode, check if we can continue executing the
             * guest using hardware-assisted VMX. Otherwise, fall back to emulation.
             */
            pVmcsInfoShared->RealMode.fRealOnV86Active = false;
            if (HMCanExecuteVmxGuest(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx))
            {
                Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
            }
            else
            {
                Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
                rcStrict = VINF_EM_RESCHEDULE;
            }
        }
        else
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
    }
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        rcStrict = VINF_SUCCESS;
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
    }
    return VBOXSTRICTRC_VAL(rcStrict);
#endif
}


/**
 * VM-exit exception handler for \#DE (Divide Error).
 *
 * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
 */
static VBOXSTRICTRC vmxHCExitXcptDE(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE);

    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
    if (VCPU_2_VMXSTATE(pVCpu).fGCMTrapXcptDE)
    {
        uint8_t cbInstr = 0;
        VBOXSTRICTRC rc2 = GCMXcptDE(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
        if (rc2 == VINF_SUCCESS)
            rcStrict = VINF_SUCCESS;    /* Restart instruction with modified guest register context. */
        else if (rc2 == VERR_NOT_FOUND)
            rcStrict = VERR_NOT_FOUND;  /* Deliver the exception. */
        else
            Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
    }
    else
        rcStrict = VINF_SUCCESS;        /* Do nothing. */

    /* If the GCM #DE exception handler didn't succeed or wasn't needed, raise #DE. */
    if (RT_FAILURE(rcStrict))
    {
        vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
                               pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
        rcStrict = VINF_SUCCESS;
    }

    Assert(rcStrict == VINF_SUCCESS || rcStrict == VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE);
    return VBOXSTRICTRC_VAL(rcStrict);
}


/**
 * VM-exit exception handler wrapper for all other exceptions that are not handled
 * by a specific handler.
 *
 * This simply re-injects the exception back into the VM without any special
 * processing.
 *
 * @remarks Requires all fields in HMVMX_READ_XCPT_INFO to be read from the VMCS.
 */
static VBOXSTRICTRC vmxHCExitXcptOthers(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);

#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
# ifndef IN_NEM_DARWIN
    PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    AssertMsg(pVCpu->hmr0.s.fUsingDebugLoop || pVmcsInfo->pShared->RealMode.fRealOnV86Active || pVmxTransient->fIsNestedGuest,
              ("uVector=%#x u32XcptBitmap=%#X32\n",
               VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
    NOREF(pVmcsInfo);
# endif
#endif

    /*
     * Re-inject the exception into the guest. This cannot be a double-fault condition which
     * would have been handled while checking exits due to event delivery.
     */
    uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);

#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
    int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);
    Log4Func(("Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%08RX64\n", uVector, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
#endif

#ifdef VBOX_WITH_STATISTICS
    switch (uVector)
    {
        case X86_XCPT_DE:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDE);     break;
        case X86_XCPT_DB:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDB);     break;
        case X86_XCPT_BP:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBP);     break;
        case X86_XCPT_OF:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF);     break;
        case X86_XCPT_BR:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestBR);     break;
        case X86_XCPT_UD:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestUD);     break;
        case X86_XCPT_NM:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestOF);     break;
        case X86_XCPT_DF:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestDF);     break;
        case X86_XCPT_TS:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestTS);     break;
        case X86_XCPT_NP:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestNP);     break;
        case X86_XCPT_SS:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestSS);     break;
        case X86_XCPT_GP:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestGP);     break;
        case X86_XCPT_PF:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestPF);     break;
        case X86_XCPT_MF:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestMF);     break;
        case X86_XCPT_AC:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestAC);     break;
        case X86_XCPT_XF:   STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXF);     break;
        default:
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitGuestXcpUnk);
            break;
    }
#endif

    /* We should never call this function for a page-fault, we'd need to pass on the fault address below otherwise. */
    Assert(!VMX_EXIT_INT_INFO_IS_XCPT_PF(pVmxTransient->uExitIntInfo));
    NOREF(uVector);

    /* Re-inject the original exception into the guest. */
    vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
                           pVmxTransient->cbExitInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
    return VINF_SUCCESS;
}


/**
 * VM-exit exception handler for all exceptions (except NMIs!).
 *
 * @remarks This may be called for both guests and nested-guests. Take care to not
 *          make assumptions and avoid doing anything that is not relevant when
 *          executing a nested-guest (e.g., Mesa driver hacks).
 */
static VBOXSTRICTRC vmxHCExitXcpt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_ASSERT_READ(pVmxTransient, HMVMX_READ_XCPT_INFO);

    /*
     * If this VM-exit occurred while delivering an event through the guest IDT, take
     * action based on the return code and additional hints (e.g. for page-faults)
     * that will be updated in the VMX transient structure.
     */
    VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
    if (rcStrict == VINF_SUCCESS)
    {
        /*
         * If an exception caused a VM-exit due to delivery of an event, the original
         * event may have to be re-injected into the guest. We shall reinject it and
         * continue guest execution. However, page-fault is a complicated case and
         * needs additional processing done in vmxHCExitXcptPF().
         */
        Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
        uint8_t const uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
        if (   !VCPU_2_VMXSTATE(pVCpu).Event.fPending
            || uVector == X86_XCPT_PF)
        {
            switch (uVector)
            {
                case X86_XCPT_PF: return vmxHCExitXcptPF(pVCpu, pVmxTransient);
                case X86_XCPT_GP: return vmxHCExitXcptGP(pVCpu, pVmxTransient);
                case X86_XCPT_MF: return vmxHCExitXcptMF(pVCpu, pVmxTransient);
                case X86_XCPT_DB: return vmxHCExitXcptDB(pVCpu, pVmxTransient);
                case X86_XCPT_BP: return vmxHCExitXcptBP(pVCpu, pVmxTransient);
                case X86_XCPT_AC: return vmxHCExitXcptAC(pVCpu, pVmxTransient);
                case X86_XCPT_DE: return vmxHCExitXcptDE(pVCpu, pVmxTransient);
                default:
                    return vmxHCExitXcptOthers(pVCpu, pVmxTransient);
            }
        }
        /* else: inject pending event before resuming guest execution. */
    }
    else if (rcStrict == VINF_HM_DOUBLE_FAULT)
    {
        Assert(VCPU_2_VMXSTATE(pVCpu).Event.fPending);
        rcStrict = VINF_SUCCESS;
    }

    return rcStrict;
}
/** @} */


/** @name VM-exit handlers.
 * @{
 */
/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */

/**
 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
 */
HMVMX_EXIT_DECL vmxHCExitExtInt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitExtInt);

#ifndef IN_NEM_DARWIN
    /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
    if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
        return VINF_SUCCESS;
    return VINF_EM_RAW_INTERRUPT;
#else
    return VINF_SUCCESS;
#endif
}


/**
 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI). Conditional
 * VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitXcptOrNmi(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);

    vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);

    uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
    uint8_t const  uVector      = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
    Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));

    PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    Assert(   !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
           && uExitIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
    NOREF(pVmcsInfo);

    VBOXSTRICTRC rcStrict;
    switch (uExitIntType)
    {
#ifndef IN_NEM_DARWIN /* NMIs should never reach R3. */
        /*
         * Host physical NMIs:
         *     This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
         *     injected it ourselves and anything we inject is not going to cause a VM-exit directly
         *     for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
         *
         *     See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
         *     See Intel spec. 27.5.5 "Updating Non-Register State".
         */
        case VMX_EXIT_INT_INFO_TYPE_NMI:
        {
            rcStrict = hmR0VmxExitHostNmi(pVCpu, pVmcsInfo);
            break;
        }
#endif

        /*
         * Privileged software exceptions (#DB from ICEBP),
         * Software exceptions (#BP and #OF),
         * Hardware exceptions:
         *     Process the required exceptions and resume guest execution if possible.
         */
        case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
            Assert(uVector == X86_XCPT_DB);
            RT_FALL_THRU();
        case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
            Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uExitIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
            RT_FALL_THRU();
        case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
        {
            NOREF(uVector);
            vmxHCReadToTransient<  HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
                                 | HMVMX_READ_EXIT_INSTR_LEN
                                 | HMVMX_READ_IDT_VECTORING_INFO
                                 | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
            rcStrict = vmxHCExitXcpt(pVCpu, pVmxTransient);
            break;
        }

        default:
        {
            VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
            rcStrict = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
            AssertMsgFailed(("Invalid/unexpected VM-exit interruption info %#x\n", pVmxTransient->uExitIntInfo));
            break;
        }
    }

    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitXcptNmi, y3);
    return rcStrict;
}


/**
 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCClearIntWindowExitVmcs(pVCpu, pVmcsInfo);

    /* Evaluate and deliver pending events and resume guest execution. */
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIntWindow);
    return VINF_SUCCESS;
}


/**
 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindow(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
    {
        AssertMsgFailed(("Unexpected NMI-window exit.\n"));
        HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
    }

    Assert(!CPUMAreInterruptsInhibitedByNmiEx(&pVCpu->cpum.GstCtx));

    /*
     * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
     * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
     */
    uint32_t fIntrState;
    int rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
    AssertRC(rc);
    Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
    if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
    {
        CPUMClearInterruptShadow(&pVCpu->cpum.GstCtx);

        fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
        rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, fIntrState);
        AssertRC(rc);
    }

    /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
    vmxHCClearNmiWindowExitVmcs(pVCpu, pVmcsInfo);

    /* Evaluate and deliver pending events and resume guest execution. */
    return VINF_SUCCESS;
}


/**
 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
}


/**
 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitInvd(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    return vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
}


/**
 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitCpuid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /*
     * Get the state we need and update the exit history entry.
     */
    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict;
    PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
                                                            EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
                                                            pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
    if (!pExitRec)
    {
        /*
         * Regular CPUID instruction execution.
         */
        rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbExitInstr);
        if (rcStrict == VINF_SUCCESS)
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
        else if (rcStrict == VINF_IEM_RAISED_XCPT)
        {
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
            rcStrict = VINF_SUCCESS;
        }
    }
    else
    {
        /*
         * Frequent exit or something needing probing.  Get state and call EMHistoryExec.
         */
        int rc2 = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
                                        IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
        AssertRCReturn(rc2, rc2);

        Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));

        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);

        Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
    }
    return rcStrict;
}


/**
 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitGetsec(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CR4>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
        return VINF_EM_RAW_EMULATE_INSTR;

    AssertMsgFailed(("vmxHCExitGetsec: Unexpected VM-exit when CR4.SMXE is 0.\n"));
    HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
}


/**
 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitRdtsc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbExitInstr);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
    {
        /* If we get a spurious VM-exit when TSC offsetting is enabled,
           we must reset offsetting on VM-entry. See @bugref{6634}. */
        if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
            pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
    }
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitRdtscp(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbExitInstr);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
    {
        /* If we get a spurious VM-exit when TSC offsetting is enabled,
           we must reset offsetting on VM-reentry. See @bugref{6634}. */
        if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
            pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
    }
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitRdpmc(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedRdpmc(pVCpu, pVmxTransient->cbExitInstr);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmcall(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
    if (EMAreHypercallInstructionsEnabled(pVCpu))
    {
        PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
        int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RIP
                                       | CPUMCTX_EXTRN_RFLAGS
                                       | CPUMCTX_EXTRN_CR0
                                       | CPUMCTX_EXTRN_SS
                                       | CPUMCTX_EXTRN_CS
                                       | CPUMCTX_EXTRN_EFER>(pVCpu, pVmcsInfo, __FUNCTION__);
        AssertRCReturn(rc, rc);

        /* Perform the hypercall. */
        rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
        if (rcStrict == VINF_SUCCESS)
        {
            rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
            AssertRCReturn(rc, rc);
        }
        else
            Assert(   rcStrict == VINF_GIM_R3_HYPERCALL
                   || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
                   || RT_FAILURE(rcStrict));

        /* If the hypercall changes anything other than guest's general-purpose registers,
           we would need to reload the guest changed bits here before VM-entry. */
    }
    else
        Log4Func(("Hypercalls not enabled\n"));

    /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
    if (RT_FAILURE(rcStrict))
    {
        vmxHCSetPendingXcptUD(pVCpu);
        rcStrict = VINF_SUCCESS;
    }

    return rcStrict;
}


/**
 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitInvlpg(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
#ifndef IN_NEM_DARWIN
    Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging || pVCpu->hmr0.s.fUsingDebugLoop);
#endif

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->uExitQual);

    if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    else
        AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) status: %Rrc\n", pVmxTransient->uExitQual,
                         VBOXSTRICTRC_VAL(rcStrict)));
    return rcStrict;
}


/**
 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitMonitor(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedMonitor(pVCpu, pVmxTransient->cbExitInstr);
    if (rcStrict == VINF_SUCCESS)
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }

    return rcStrict;
}


/**
 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitMwait(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedMwait(pVCpu, pVmxTransient->cbExitInstr);
    if (RT_SUCCESS(rcStrict))
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
        if (EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
            rcStrict = VINF_SUCCESS;
    }

    return rcStrict;
}


/**
 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
 * VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitTripleFault(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    return VINF_EM_RESET;
}


/**
 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitHlt(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
    AssertRCReturn(rc, rc);

    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);            /* Advancing the RIP above should've imported eflags. */
    if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx))    /* Requires eflags. */
        rc = VINF_SUCCESS;
    else
        rc = VINF_EM_HALT;

    if (rc != VINF_SUCCESS)
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchHltToR3);
    return rc;
}


#ifndef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
/**
 * VM-exit handler for instructions that result in a \#UD exception delivered to
 * the guest.
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitSetPendingXcptUD(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    vmxHCSetPendingXcptUD(pVCpu);
    return VINF_SUCCESS;
}
#endif


/**
 * VM-exit handler for expiry of the VMX-preemption timer.
 */
HMVMX_EXIT_DECL vmxHCExitPreemptTimer(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
    pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
Log12(("vmxHCExitPreemptTimer:\n"));

    /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
    bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
    STAM_REL_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitPreemptTimer);
    return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
}


/**
 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitXsetbv(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbExitInstr);
    ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
                                                                                : HM_CHANGED_RAISED_XCPT_MASK);

#ifndef IN_NEM_DARWIN
    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
    bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
    if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
    {
        pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
        hmR0VmxUpdateStartVmFunction(pVCpu);
    }
#endif

    return rcStrict;
}


/**
 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitInvpcid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /** @todo Enable the new code after finding a reliably guest test-case. */
#if 1
    return VERR_EM_INTERPRETER;
#else
    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_INFO
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_SREG_MASK
                                                                    | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
    AssertRCReturn(rc, rc);

    /* Paranoia. Ensure this has a memory operand. */
    Assert(!pVmxTransient->ExitInstrInfo.Inv.u1Cleared0);

    uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
    Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
    uint64_t const uType = CPUMIsGuestIn64BitCode(pVCpu) ? pVCpu->cpum.GstCtx.aGRegs[iGReg].u64
                                                         : pVCpu->cpum.GstCtx.aGRegs[iGReg].u32;

    RTGCPTR GCPtrDesc;
    HMVMX_DECODE_MEM_OPERAND(pVCpu, pVmxTransient->ExitInstrInfo.u, pVmxTransient->uExitQual, VMXMEMACCESS_READ, &GCPtrDesc);

    VBOXSTRICTRC rcStrict = IEMExecDecodedInvpcid(pVCpu, pVmxTransient->cbExitInstr, pVmxTransient->ExitInstrInfo.Inv.iSegReg,
                                                  GCPtrDesc, uType);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
#endif
}


/**
 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error
 * VM-exit.
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitErrInvalidGuestState(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    int rc = vmxHCImportGuestStateEx(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
    AssertRCReturn(rc, rc);

    rc = vmxHCCheckCachedVmcsCtls(pVCpu, pVmcsInfo, pVmxTransient->fIsNestedGuest);
    if (RT_FAILURE(rc))
        return rc;

    uint32_t const uInvalidReason = vmxHCCheckGuestState(pVCpu, pVmcsInfo);
    NOREF(uInvalidReason);

#ifdef VBOX_STRICT
    uint32_t fIntrState;
    uint64_t u64Val;
    vmxHCReadToTransient<  HMVMX_READ_EXIT_INSTR_INFO
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    vmxHCReadEntryXcptErrorCodeVmcs(pVCpu, pVmxTransient);

    Log4(("uInvalidReason                             %u\n",     uInvalidReason));
    Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO    %#RX32\n", pVmxTransient->uEntryIntInfo));
    Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE    %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
    Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH         %#RX32\n", pVmxTransient->cbEntryInstr));

    rc = VMX_VMCS_READ_32(pVCpu, VMX_VMCS32_GUEST_INT_STATE, &fIntrState);            AssertRC(rc);
    Log4(("VMX_VMCS32_GUEST_INT_STATE                 %#RX32\n", fIntrState));
    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_GUEST_CR0, &u64Val);                        AssertRC(rc);
    Log4(("VMX_VMCS_GUEST_CR0                         %#RX64\n", u64Val));
    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, &u64Val);                    AssertRC(rc);
    Log4(("VMX_VMCS_CTRL_CR0_MASK                     %#RX64\n", u64Val));
    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Val);             AssertRC(rc);
    Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW              %#RX64\n", u64Val));
    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, &u64Val);                    AssertRC(rc);
    Log4(("VMX_VMCS_CTRL_CR4_MASK                     %#RX64\n", u64Val));
    rc = VMX_VMCS_READ_NW(pVCpu, VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Val);             AssertRC(rc);
    Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW              %#RX64\n", u64Val));
# ifndef IN_NEM_DARWIN
    if (pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
    {
        rc = VMX_VMCS_READ_64(pVCpu, VMX_VMCS64_CTRL_EPTP_FULL, &u64Val);             AssertRC(rc);
        Log4(("VMX_VMCS64_CTRL_EPTP_FULL                  %#RX64\n", u64Val));
    }

    hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
# endif
#endif

    return VERR_VMX_INVALID_GUEST_STATE;
}

/**
 * VM-exit handler for all undefined/unexpected reasons. Should never happen.
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitErrUnexpected(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    /*
     * Cumulative notes of all recognized but unexpected VM-exits.
     *
     * 1. This does -not- cover scenarios like a page-fault VM-exit occurring when
     *    nested-paging is used.
     *
     * 2. Any instruction that causes a VM-exit unconditionally (for e.g. VMXON) must be
     *    emulated or a #UD must be raised in the guest. Therefore, we should -not- be using
     *    this function (and thereby stop VM execution) for handling such instructions.
     *
     *
     * VMX_EXIT_INIT_SIGNAL:
     *    INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
     *    It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these
     *    VM-exits. However, we should not receive INIT signals VM-exit while executing a VM.
     *
     *    See Intel spec. 33.14.1 Default Treatment of SMI Delivery"
     *    See Intel spec. 29.3 "VMX Instructions" for "VMXON".
     *    See Intel spec. "23.8 Restrictions on VMX operation".
     *
     * VMX_EXIT_SIPI:
     *    SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest
     *    activity state is used. We don't make use of it as our guests don't have direct
     *    access to the host local APIC.
     *
     *    See Intel spec. 25.3 "Other Causes of VM-exits".
     *
     * VMX_EXIT_IO_SMI:
     * VMX_EXIT_SMI:
     *    This can only happen if we support dual-monitor treatment of SMI, which can be
     *    activated by executing VMCALL in VMX root operation. Only an STM (SMM transfer
     *    monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL in
     *    VMX root mode or receive an SMI. If we get here, something funny is going on.
     *
     *    See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
     *    See Intel spec. 25.3 "Other Causes of VM-Exits"
     *
     * VMX_EXIT_ERR_MSR_LOAD:
     *    Failures while loading MSRs are part of the VM-entry MSR-load area are unexpected
     *    and typically indicates a bug in the hypervisor code. We thus cannot not resume
     *    execution.
     *
     *    See Intel spec. 26.7 "VM-Entry Failures During Or After Loading Guest State".
     *
     * VMX_EXIT_ERR_MACHINE_CHECK:
     *    Machine check exceptions indicates a fatal/unrecoverable hardware condition
     *    including but not limited to system bus, ECC, parity, cache and TLB errors. A
     *    #MC exception abort class exception is raised. We thus cannot assume a
     *    reasonable chance of continuing any sort of execution and we bail.
     *
     *    See Intel spec. 15.1 "Machine-check Architecture".
     *    See Intel spec. 27.1 "Architectural State Before A VM Exit".
     *
     * VMX_EXIT_PML_FULL:
     * VMX_EXIT_VIRTUALIZED_EOI:
     * VMX_EXIT_APIC_WRITE:
     *    We do not currently support any of these features and thus they are all unexpected
     *    VM-exits.
     *
     * VMX_EXIT_GDTR_IDTR_ACCESS:
     * VMX_EXIT_LDTR_TR_ACCESS:
     * VMX_EXIT_RDRAND:
     * VMX_EXIT_RSM:
     * VMX_EXIT_VMFUNC:
     * VMX_EXIT_ENCLS:
     * VMX_EXIT_RDSEED:
     * VMX_EXIT_XSAVES:
     * VMX_EXIT_XRSTORS:
     * VMX_EXIT_UMWAIT:
     * VMX_EXIT_TPAUSE:
     * VMX_EXIT_LOADIWKEY:
     *    These VM-exits are -not- caused unconditionally by execution of the corresponding
     *    instruction. Any VM-exit for these instructions indicate a hardware problem,
     *    unsupported CPU modes (like SMM) or potentially corrupt VMCS controls.
     *
     *    See Intel spec. 25.1.3 "Instructions That Cause VM Exits Conditionally".
     */
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    AssertMsgFailed(("Unexpected VM-exit %u\n", pVmxTransient->uExitReason));
    HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
}


/**
 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
 */
HMVMX_EXIT_DECL vmxHCExitRdmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);

    /** @todo Optimize this: We currently drag in the whole MSR state
     * (CPUMCTX_EXTRN_ALL_MSRS) here.  We should optimize this to only get
     * MSRs required.  That would require changes to IEM and possibly CPUM too.
     * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
    PVMXVMCSINFO   pVmcsInfo = pVmxTransient->pVmcsInfo;
    uint32_t const idMsr     = pVCpu->cpum.GstCtx.ecx;
    int            rc;
    switch (idMsr)
    {
        default:
            rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS>(pVCpu, pVmcsInfo,
                                                                                                            __FUNCTION__);
            AssertRCReturn(rc, rc);
            break;
        case MSR_K8_FS_BASE:
            rc = vmxHCImportGuestState<  IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS
                                       | CPUMCTX_EXTRN_FS>(pVCpu, pVmcsInfo, __FUNCTION__);
            AssertRCReturn(rc, rc);
            break;
        case MSR_K8_GS_BASE:
            rc = vmxHCImportGuestState<  IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS
                                       | CPUMCTX_EXTRN_GS>(pVCpu, pVmcsInfo, __FUNCTION__);
            AssertRCReturn(rc, rc);
            break;
    }

    Log4Func(("ecx=%#RX32\n", idMsr));

#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN)
    Assert(!pVmxTransient->fIsNestedGuest);
    if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
    {
        if (   hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
            && idMsr != MSR_K6_EFER)
        {
            AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
        }
        if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
        {
            Assert(pVmcsInfo->pvMsrBitmap);
            uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
            if (fMsrpm & VMXMSRPM_ALLOW_RD)
            {
                AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
                HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
            }
        }
    }
#endif

    VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbExitInstr);
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitRdmsr);
    if (rcStrict == VINF_SUCCESS)
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    else
        AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ || rcStrict == VINF_EM_TRIPLE_FAULT,
                  ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));

    return rcStrict;
}


/**
 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
 */
HMVMX_EXIT_DECL vmxHCExitWrmsr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);

    /*
     * The FS and GS base MSRs are not part of the above all-MSRs mask.
     * Although we don't need to fetch the base as it will be overwritten shortly, while
     * loading guest-state we would also load the entire segment register including limit
     * and attributes and thus we need to load them here.
     */
    /** @todo Optimize this: We currently drag in the whole MSR state
     * (CPUMCTX_EXTRN_ALL_MSRS) here.  We should optimize this to only get
     * MSRs required.  That would require changes to IEM and possibly CPUM too.
     * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    uint32_t const idMsr   = pVCpu->cpum.GstCtx.ecx;
    int            rc;
    switch (idMsr)
    {
        default:
            rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS>(pVCpu, pVmcsInfo,
                                                                                                            __FUNCTION__);
            AssertRCReturn(rc, rc);
            break;

        case MSR_K8_FS_BASE:
            rc = vmxHCImportGuestState<  IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS
                                       | CPUMCTX_EXTRN_FS>(pVCpu, pVmcsInfo, __FUNCTION__);
            AssertRCReturn(rc, rc);
            break;
        case MSR_K8_GS_BASE:
            rc = vmxHCImportGuestState<  IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS
                                       | CPUMCTX_EXTRN_GS>(pVCpu, pVmcsInfo, __FUNCTION__);
            AssertRCReturn(rc, rc);
            break;
    }
    Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));

    VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbExitInstr);
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitWrmsr);

    if (rcStrict == VINF_SUCCESS)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);

        /* If this is an X2APIC WRMSR access, update the APIC state as well. */
        if (    idMsr == MSR_IA32_APICBASE
            || (   idMsr >= MSR_IA32_X2APIC_START
                && idMsr <= MSR_IA32_X2APIC_END))
        {
            /*
             * We've already saved the APIC related guest-state (TPR) in post-run phase.
             * When full APIC register virtualization is implemented we'll have to make
             * sure APIC state is saved from the VMCS before IEM changes it.
             */
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
        }
        else if (idMsr == MSR_IA32_TSC)        /* Windows 7 does this during bootup. See @bugref{6398}. */
            pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
        else if (idMsr == MSR_K6_EFER)
        {
            /*
             * If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
             * as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
             * We care about the other bits as well, SCE and NXE. See @bugref{7368}.
             */
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
        }

        /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not used. */
        if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
        {
            switch (idMsr)
            {
                case MSR_IA32_SYSENTER_CS:  ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR);  break;
                case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
                case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
                case MSR_K8_FS_BASE:        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_FS);               break;
                case MSR_K8_GS_BASE:        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_GS);               break;
                case MSR_K6_EFER:           /* Nothing to do, already handled above. */                                    break;
                default:
                {
#ifndef IN_NEM_DARWIN
                    if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
                        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
                    else if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
                        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
#else
                    AssertMsgFailed(("TODO\n"));
#endif
                    break;
                }
            }
        }
#if defined(VBOX_STRICT) && !defined(IN_NEM_DARWIN)
        else
        {
            /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
            switch (idMsr)
            {
                case MSR_IA32_SYSENTER_CS:
                case MSR_IA32_SYSENTER_EIP:
                case MSR_IA32_SYSENTER_ESP:
                case MSR_K8_FS_BASE:
                case MSR_K8_GS_BASE:
                {
                    AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
                    HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
                }

                /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
                default:
                {
                    if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
                    {
                        /* EFER MSR writes are always intercepted. */
                        if (idMsr != MSR_K6_EFER)
                        {
                            AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
                                             idMsr));
                            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
                        }
                    }

                    if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
                    {
                        Assert(pVmcsInfo->pvMsrBitmap);
                        uint32_t fMsrpm = CPUMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
                        if (fMsrpm & VMXMSRPM_ALLOW_WR)
                        {
                            AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
                            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, idMsr);
                        }
                    }
                    break;
                }
            }
        }
#endif  /* VBOX_STRICT */
    }
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    else
        AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE || rcStrict == VINF_EM_TRIPLE_FAULT,
                  ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));

    return rcStrict;
}


/**
 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitPause(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /** @todo The guest has likely hit a contended spinlock. We might want to
     *        poke a schedule different guest VCPU. */
    int rc = vmxHCAdvanceGuestRip(pVCpu, pVmxTransient);
    if (RT_SUCCESS(rc))
        return VINF_EM_RAW_INTERRUPT;

    AssertMsgFailed(("vmxHCExitPause: Failed to increment RIP. rc=%Rrc\n", rc));
    return rc;
}


/**
 * VM-exit handler for when the TPR value is lowered below the specified
 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThreshold(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);

    /*
     * The TPR shadow would've been synced with the APIC TPR in the post-run phase.
     * We'll re-evaluate pending interrupts and inject them before the next VM
     * entry so we can just continue execution here.
     */
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTprBelowThreshold);
    return VINF_SUCCESS;
}


/**
 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
 * VM-exit.
 *
 * @retval VINF_SUCCESS when guest execution can continue.
 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
 * @retval VERR_EM_RESCHEDULE_REM when we need to return to ring-3 due to
 *         incompatible guest state for VMX execution (real-on-v86 case).
 */
HMVMX_EXIT_DECL vmxHCExitMovCRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);

    VBOXSTRICTRC rcStrict;
    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
    uint64_t const uExitQual   = pVmxTransient->uExitQual;
    uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
    switch (uAccessType)
    {
        /*
         * MOV to CRx.
         */
        case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
        {
            /*
             * When PAE paging is used, the CPU will reload PAE PDPTEs from CR3 when the guest
             * changes certain bits even in CR0, CR4 (and not just CR3). We are currently fine
             * since IEM_CPUMCTX_EXTRN_MUST_MASK (used below) includes CR3 which will import
             * PAE PDPTEs as well.
             */
            int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
            AssertRCReturn(rc, rc);

            HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
#ifndef IN_NEM_DARWIN
            uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
#endif
            uint8_t const  iGReg   = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
            uint8_t const  iCrReg  = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);

            /*
             * MOV to CR3 only cause a VM-exit when one or more of the following are true:
             *   - When nested paging isn't used.
             *   - If the guest doesn't have paging enabled (intercept CR3 to update shadow page tables).
             *   - We are executing in the VM debug loop.
             */
#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
# ifndef IN_NEM_DARWIN
            Assert(   iCrReg != 3
                   || !VM_IS_VMX_NESTED_PAGING(pVM)
                   || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
                   || pVCpu->hmr0.s.fUsingDebugLoop);
# else
            Assert(   iCrReg != 3
                   || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
# endif
#endif

            /* MOV to CR8 writes only cause VM-exits when TPR shadow is not used. */
            Assert(   iCrReg != 8
                   || !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));

            rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
            AssertMsg(   rcStrict == VINF_SUCCESS
                      || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));

#ifndef IN_NEM_DARWIN
            /*
             * This is a kludge for handling switches back to real mode when we try to use
             * V86 mode to run real mode code directly.  Problem is that V86 mode cannot
             * deal with special selector values, so we have to return to ring-3 and run
             * there till the selector values are V86 mode compatible.
             *
             * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
             *       latter is an alias for VINF_IEM_RAISED_XCPT which is asserted at the end of
             *       this function.
             */
            if (   iCrReg == 0
                && rcStrict == VINF_SUCCESS
                && !VM_IS_VMX_UNRESTRICTED_GUEST(pVM)
                && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
                && (uOldCr0 & X86_CR0_PE)
                && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
            {
                /** @todo Check selectors rather than returning all the time.  */
                Assert(!pVmxTransient->fIsNestedGuest);
                Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
                rcStrict = VINF_EM_RESCHEDULE_REM;
            }
#endif

            break;
        }

        /*
         * MOV from CRx.
         */
        case VMX_EXIT_QUAL_CRX_ACCESS_READ:
        {
            uint8_t const iGReg  = VMX_EXIT_QUAL_CRX_GENREG(uExitQual);
            uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(uExitQual);

            /*
             * MOV from CR3 only cause a VM-exit when one or more of the following are true:
             *   - When nested paging isn't used.
             *   - If the guest doesn't have paging enabled (pass guest's CR3 rather than our identity mapped CR3).
             *   - We are executing in the VM debug loop.
             */
#ifndef HMVMX_ALWAYS_INTERCEPT_CR3_ACCESS
# ifndef IN_NEM_DARWIN
            Assert(   iCrReg != 3
                   || !VM_IS_VMX_NESTED_PAGING(pVM)
                   || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
                   || pVCpu->hmr0.s.fLeaveDone);
# else
            Assert(   iCrReg != 3
                   || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx));
# endif
#endif

            /* MOV from CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
            Assert(   iCrReg != 8
                   || !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));

            rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
            break;
        }

        /*
         * CLTS (Clear Task-Switch Flag in CR0).
         */
        case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
        {
            rcStrict = vmxHCExitClts(pVCpu, pVmcsInfo, pVmxTransient->cbExitInstr);
            break;
        }

        /*
         * LMSW (Load Machine-Status Word into CR0).
         * LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here.
         */
        case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:
        {
            RTGCPTR        GCPtrEffDst;
            uint8_t const  cbInstr     = pVmxTransient->cbExitInstr;
            uint16_t const uMsw        = VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual);
            bool const     fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(uExitQual);
            if (fMemOperand)
            {
                vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
                GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
            }
            else
                GCPtrEffDst = NIL_RTGCPTR;
            rcStrict = vmxHCExitLmsw(pVCpu, pVmcsInfo, cbInstr, uMsw, GCPtrEffDst);
            break;
        }

        default:
        {
            AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
        }
    }

    Assert((VCPU_2_VMXSTATE(pVCpu).fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
                                   == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
    Assert(rcStrict != VINF_IEM_RAISED_XCPT);

    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitMovCRx, y2);
    NOREF(pVM);
    return rcStrict;
}


/**
 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
 * VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitIoInstr(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);

    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
#define VMX_HC_EXIT_IO_INSTR_INITIAL_REGS   (IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER)
    /* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
    int rc = vmxHCImportGuestState<VMX_HC_EXIT_IO_INSTR_INITIAL_REGS>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
    uint32_t const uIOPort      = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
    uint8_t  const uIOSize      = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
    bool     const fIOWrite     = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
    bool     const fIOString    = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
    bool     const fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
    bool     const fDbgStepping = VCPU_2_VMXSTATE(pVCpu).fSingleInstruction;
    AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);

    /*
     * Update exit history to see if this exit can be optimized.
     */
    VBOXSTRICTRC rcStrict;
    PCEMEXITREC  pExitRec = NULL;
    if (   !fGstStepping
        && !fDbgStepping)
        pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
                                                    !fIOString
                                                    ? !fIOWrite
                                                    ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
                                                    : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
                                                    : !fIOWrite
                                                    ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
                                                    : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
                                                    pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
    if (!pExitRec)
    {
        static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 };                    /* Size of the I/O accesses in bytes. */
        static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff };   /* AND masks for saving result in AL/AX/EAX. */

        uint32_t const cbValue  = s_aIOSizes[uIOSize];
        uint32_t const cbInstr  = pVmxTransient->cbExitInstr;
        bool  fUpdateRipAlready = false; /* ugly hack, should be temporary. */
        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
        if (fIOString)
        {
            /*
             * INS/OUTS - I/O String instruction.
             *
             * Use instruction-information if available, otherwise fall back on
             * interpreting the instruction.
             */
            Log4Func(("cs:rip=%#04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
            AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
            bool const fInsOutsInfo = RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
            if (fInsOutsInfo)
            {
                vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
                AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
                AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
                IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
                bool const fRep           = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
                if (fIOWrite)
                    rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
                                                    pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
                else
                {
                    /*
                     * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
                     * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
                     * See Intel Instruction spec. for "INS".
                     * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
                     */
                    rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
                }
            }
            else
                rcStrict = IEMExecOne(pVCpu);

            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
            fUpdateRipAlready = true;
        }
        else
        {
            /*
             * IN/OUT - I/O instruction.
             */
            Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
            uint32_t const uAndVal = s_aIOOpAnd[uIOSize];
            Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
            if (fIOWrite)
            {
                rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
                STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite);
#ifndef IN_NEM_DARWIN
                if (    rcStrict == VINF_IOM_R3_IOPORT_WRITE
                    && !pCtx->eflags.Bits.u1TF)
                    rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
#endif
            }
            else
            {
                uint32_t u32Result = 0;
                rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
                if (IOM_SUCCESS(rcStrict))
                {
                    /* Save result of I/O IN instr. in AL/AX/EAX. */
                    pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
                }
#ifndef IN_NEM_DARWIN
                if (    rcStrict == VINF_IOM_R3_IOPORT_READ
                    && !pCtx->eflags.Bits.u1TF)
                    rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
#endif
                STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitIORead);
            }
        }

        if (IOM_SUCCESS(rcStrict))
        {
            if (!fUpdateRipAlready)
            {
                vmxHCAdvanceGuestRipBy(pVCpu, cbInstr);
                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP);
            }

            /*
             * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
             * while booting Fedora 17 64-bit guest.
             *
             * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
             */
            if (fIOString)
                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RFLAGS);

            /*
             * If any I/O breakpoints are armed, we need to check if one triggered
             * and take appropriate action.
             * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
             */
#if 1
            AssertCompile(VMX_HC_EXIT_IO_INSTR_INITIAL_REGS & CPUMCTX_EXTRN_DR7);
#else
            AssertCompile(!(VMX_HC_EXIT_IO_INSTR_INITIAL_REGS & CPUMCTX_EXTRN_DR7));
            rc = vmxHCImportGuestState<CPUMCTX_EXTRN_DR7>(pVCpu, pVmcsInfo);
            AssertRCReturn(rc, rc);
#endif

            /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
             *  execution engines about whether hyper BPs and such are pending. */
            uint32_t const uDr7 = pCtx->dr[7];
            if (RT_UNLIKELY(   (   (uDr7 & X86_DR7_ENABLED_MASK)
                                && X86_DR7_ANY_RW_IO(uDr7)
                                && (pCtx->cr4 & X86_CR4_DE))
                            || DBGFBpIsHwIoArmed(pVM)))
            {
                STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxIoCheck);

#ifndef IN_NEM_DARWIN
                /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
                VMMRZCallRing3Disable(pVCpu);
                HM_DISABLE_PREEMPT(pVCpu);

                bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);

                VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
                if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
                {
                    /* Raise #DB. */
                    if (fIsGuestDbgActive)
                        ASMSetDR6(pCtx->dr[6]);
                    if (pCtx->dr[7] != uDr7)
                        VCPU_2_VMXSTATE(pVCpu).fCtxChanged |= HM_CHANGED_GUEST_DR7;

                    vmxHCSetPendingXcptDB(pVCpu);
                }
                /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
                   however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
                else if (   rcStrict2 != VINF_SUCCESS
                         && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
                    rcStrict = rcStrict2;
                AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);

                HM_RESTORE_PREEMPT();
                VMMRZCallRing3Enable(pVCpu);
#else
                /** @todo */
#endif
            }
        }

#ifdef VBOX_STRICT
        if (   rcStrict == VINF_IOM_R3_IOPORT_READ
            || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
            Assert(!fIOWrite);
        else if (   rcStrict == VINF_IOM_R3_IOPORT_WRITE
                 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
                 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
            Assert(fIOWrite);
        else
        {
# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
           *        statuses, that the VMM device and some others may return. See
           *        IOM_SUCCESS() for guidance. */
            AssertMsg(   RT_FAILURE(rcStrict)
                      || rcStrict == VINF_SUCCESS
                      || rcStrict == VINF_EM_RAW_EMULATE_INSTR
                      || rcStrict == VINF_EM_DBG_BREAKPOINT
                      || rcStrict == VINF_EM_RAW_GUEST_TRAP
                      || rcStrict == VINF_EM_RAW_TO_R3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
# endif
        }
#endif
        STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitIO, y1);
    }
    else
    {
        /*
         * Frequent exit or something needing probing.  Get state and call EMHistoryExec.
         */
        int rc2 = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL,
                                        VMX_HC_EXIT_IO_INSTR_INITIAL_REGS>(pVCpu, pVmcsInfo, __FUNCTION__);
        AssertRCReturn(rc2, rc2);
        STAM_COUNTER_INC(!fIOString ? fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIORead
                         : fIOWrite ? &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringWrite : &VCPU_2_VMXSTATS(pVCpu).StatExitIOStringRead);
        Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
              VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
              fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOSize));

        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);

        Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
    }
    return rcStrict;
}


/**
 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
 * VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitTaskSwitch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /* Check if this task-switch occurred while delivery an event through the guest IDT. */
    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
    if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
    {
        vmxHCReadToTransient<HMVMX_READ_IDT_VECTORING_INFO>(pVCpu, pVmxTransient);
        if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
        {
            uint32_t uErrCode;
            if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
            {
                vmxHCReadToTransient<HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);
                uErrCode = pVmxTransient->uIdtVectoringErrorCode;
            }
            else
                uErrCode = 0;

            RTGCUINTPTR GCPtrFaultAddress;
            if (VMX_IDT_VECTORING_INFO_IS_XCPT_PF(pVmxTransient->uIdtVectoringInfo))
                GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
            else
                GCPtrFaultAddress = 0;

            vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);

            vmxHCSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
                                 pVmxTransient->cbExitInstr, uErrCode, GCPtrFaultAddress);

            Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo),
                      VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
            return VINF_EM_RAW_INJECT_TRPM_EVENT;
        }
    }

    /* Fall back to the interpreter to emulate the task-switch. */
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitTaskSwitch);
    return VERR_EM_INTERPRETER;
}


/**
 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitMtf(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
    int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
    AssertRC(rc);
    return VINF_EM_DBG_STEPPED;
}


/**
 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitApicAccess(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitApicAccess);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_LEN
                         | HMVMX_READ_EXIT_INTERRUPTION_INFO
                         | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
                         | HMVMX_READ_IDT_VECTORING_INFO
                         | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);

    /*
     * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
     */
    VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
    {
        /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
        if (RT_UNLIKELY(VCPU_2_VMXSTATE(pVCpu).Event.fPending))
        {
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
            return VINF_EM_RAW_INJECT_TRPM_EVENT;
        }
    }
    else
    {
        Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
        return rcStrict;
    }

    /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
    uint32_t const uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
    switch (uAccessType)
    {
#ifndef IN_NEM_DARWIN
        case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
        case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
        {
            AssertMsg(   !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
                      || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
                      ("vmxHCExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));

            RTGCPHYS GCPhys = VCPU_2_VMXSTATE(pVCpu).vmx.u64GstMsrApicBase;    /* Always up-to-date, as it is not part of the VMCS. */
            GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
            GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
            Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
                 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));

            rcStrict = IOMR0MmioPhysHandler(pVCpu->CTX_SUFF(pVM), pVCpu,
                                            uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW, GCPhys);
            Log4Func(("IOMR0MmioPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
            if (   rcStrict == VINF_SUCCESS
                || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
                || rcStrict == VERR_PAGE_NOT_PRESENT)
            {
                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
                                                         | HM_CHANGED_GUEST_APIC_TPR);
                rcStrict = VINF_SUCCESS;
            }
            break;
        }
#else
        /** @todo */
#endif

        default:
        {
            Log4Func(("uAccessType=%#x\n", uAccessType));
            rcStrict = VINF_EM_RAW_EMULATE_INSTR;
            break;
        }
    }

    if (rcStrict != VINF_SUCCESS)
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatSwitchApicAccessToR3);
    return rcStrict;
}


/**
 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
 * VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitMovDRx(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;

    /*
     * We might also get this VM-exit if the nested-guest isn't intercepting MOV DRx accesses.
     * In such a case, rather than disabling MOV DRx intercepts and resuming execution, we
     * must emulate the MOV DRx access.
     */
    if (!pVmxTransient->fIsNestedGuest)
    {
        /* We should -not- get this VM-exit if the guest's debug registers were active. */
        if (   pVmxTransient->fWasGuestDebugStateActive
#ifdef VMX_WITH_MAYBE_ALWAYS_INTERCEPT_MOV_DRX
            && !pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fAlwaysInterceptMovDRx
#endif
           )
        {
            AssertMsgFailed(("Unexpected MOV DRx exit\n"));
            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient->uExitReason);
        }

        if (   !VCPU_2_VMXSTATE(pVCpu).fSingleInstruction
            && !pVmxTransient->fWasHyperDebugStateActive)
        {
            Assert(!DBGFIsStepping(pVCpu));
            Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));

            /* Whether we disable intercepting MOV DRx instructions and resume
               the current one, or emulate it and keep intercepting them is
               configurable.  Though it usually comes down to whether there are
               any new DR6 & DR7 bits (RTM) we want to hide from the guest. */
#ifdef VMX_WITH_MAYBE_ALWAYS_INTERCEPT_MOV_DRX
            bool const fResumeInstruction = !pVCpu->CTX_SUFF(pVM)->hmr0.s.vmx.fAlwaysInterceptMovDRx;
#else
            bool const fResumeInstruction = true;
#endif
            if (fResumeInstruction)
            {
                pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
                int rc = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
                AssertRC(rc);
            }

#ifndef IN_NEM_DARWIN
            /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
            VMMRZCallRing3Disable(pVCpu);
            HM_DISABLE_PREEMPT(pVCpu);

            /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
            CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
            Assert(CPUMIsGuestDebugStateActive(pVCpu));

            HM_RESTORE_PREEMPT();
            VMMRZCallRing3Enable(pVCpu);
#else
            CPUMR3NemActivateGuestDebugState(pVCpu);
            Assert(CPUMIsGuestDebugStateActive(pVCpu));
            Assert(!CPUMIsHyperDebugStateActive(pVCpu));
#endif

            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatDRxContextSwitch);
            if (fResumeInstruction)
            {
#ifdef VBOX_WITH_STATISTICS
                vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
                if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
                    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
                else
                    STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
#endif
                return VINF_SUCCESS;
            }
        }
    }

    /*
     * Import state.  We must have DR7 loaded here as it's always consulted,
     * both for reading and writing.  The other debug registers are never
     * exported as such.
     */
    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<  IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK
                                   | CPUMCTX_EXTRN_GPRS_MASK
                                   | CPUMCTX_EXTRN_DR7>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    uint8_t const iGReg  = VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual);
    uint8_t const iDrReg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
    Log4Func(("cs:rip=%#04x:%08RX64 r%d %s dr%d\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, iGReg,
              VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE ? "->" : "<-", iDrReg));

    VBOXSTRICTRC  rcStrict;
    if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
    {
        /*
         * Write DRx register.
         */
        rcStrict = IEMExecDecodedMovDRxWrite(pVCpu, pVmxTransient->cbExitInstr, iDrReg, iGReg);
        AssertMsg(   rcStrict == VINF_SUCCESS
                  || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));

        if (rcStrict == VINF_SUCCESS)
       {
            /** @todo r=bird: Not sure why we always flag DR7 as modified here, but I've
             * kept it for now to avoid breaking something non-obvious. */
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
                                                                | HM_CHANGED_GUEST_DR7);
            /* Update the DR6 register if guest debug state is active, otherwise we'll
               trash it when calling CPUMR0DebugStateMaybeSaveGuestAndRestoreHost. */
            if (iDrReg == 6 && CPUMIsGuestDebugStateActive(pVCpu))
                ASMSetDR6(pVCpu->cpum.GstCtx.dr[6]);
            Log4Func(("r%d=%#RX64 => dr%d=%#RX64\n", iGReg, pVCpu->cpum.GstCtx.aGRegs[iGReg].u,
                      iDrReg, pVCpu->cpum.GstCtx.dr[iDrReg]));
        }
        else if (rcStrict == VINF_IEM_RAISED_XCPT)
        {
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
            rcStrict = VINF_SUCCESS;
        }

        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxWrite);
    }
    else
    {
        /*
         * Read DRx register into a general purpose register.
         */
        rcStrict = IEMExecDecodedMovDRxRead(pVCpu, pVmxTransient->cbExitInstr, iGReg, iDrReg);
        AssertMsg(   rcStrict == VINF_SUCCESS
                  || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));

        if (rcStrict == VINF_SUCCESS)
        {
            if (iGReg == X86_GREG_xSP)
                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
                                                                    | HM_CHANGED_GUEST_RSP);
            else
                ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
        }
        else if (rcStrict == VINF_IEM_RAISED_XCPT)
        {
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
            rcStrict = VINF_SUCCESS;
        }

        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitDRxRead);
    }

    return rcStrict;
}


/**
 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
 * Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitEptMisconfig(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

#ifndef IN_NEM_DARWIN
    Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_INSTR_LEN
                         | HMVMX_READ_EXIT_INTERRUPTION_INFO
                         | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
                         | HMVMX_READ_IDT_VECTORING_INFO
                         | HMVMX_READ_IDT_VECTORING_ERROR_CODE
                         | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);

    /*
     * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
     */
    VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
    {
        /*
         * In the unlikely case where delivering an event causes an EPT misconfig (MMIO), go back to
         * instruction emulation to inject the original event. Otherwise, injecting the original event
         * using hardware-assisted VMX would trigger the same EPT misconfig VM-exit again.
         */
        if (!VCPU_2_VMXSTATE(pVCpu).Event.fPending)
        { /* likely */ }
        else
        {
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectInterpret);
# ifdef VBOX_WITH_NESTED_HWVIRT_VMX
            /** @todo NSTVMX: Think about how this should be handled. */
            if (pVmxTransient->fIsNestedGuest)
                return VERR_VMX_IPE_3;
# endif
            return VINF_EM_RAW_INJECT_TRPM_EVENT;
        }
    }
    else
    {
        Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
        return rcStrict;
    }

    /*
     * Get sufficient state and update the exit history entry.
     */
    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    RTGCPHYS const GCPhys = pVmxTransient->uGuestPhysicalAddr;
    PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
                                                            EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
                                                            pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
    if (!pExitRec)
    {
        /*
         * If we succeed, resume guest execution.
         * If we fail in interpreting the instruction because we couldn't get the guest physical address
         * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
         * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
         * weird case. See @bugref{6043}.
         */
        PVMCC    pVM  = pVCpu->CTX_SUFF(pVM);
/** @todo bird: We can probably just go straight to IOM here and assume that
 *        it's MMIO, then fall back on PGM if that hunch didn't work out so
 *        well.  However, we need to address that aliasing workarounds that
 *        PGMR0Trap0eHandlerNPMisconfig implements.  So, some care is needed.
 *
 *        Might also be interesting to see if we can get this done more or
 *        less locklessly inside IOM.  Need to consider the lookup table
 *        updating and use a bit more carefully first (or do all updates via
 *        rendezvous) */
        rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, &pVCpu->cpum.GstCtx, GCPhys, UINT32_MAX);
        Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pVCpu->cpum.GstCtx.rip, VBOXSTRICTRC_VAL(rcStrict)));
        if (   rcStrict == VINF_SUCCESS
            || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
            || rcStrict == VERR_PAGE_NOT_PRESENT)
        {
            /* Successfully handled MMIO operation. */
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
                                                     | HM_CHANGED_GUEST_APIC_TPR);
            rcStrict = VINF_SUCCESS;
        }
    }
    else
    {
        /*
         * Frequent exit or something needing probing. Call EMHistoryExec.
         */
        int rc2 = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL, IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
        AssertRCReturn(rc2, rc2);
        Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));

        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);

        Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
    }
    return rcStrict;
#else
    AssertFailed();
    return VERR_VMX_IPE_3; /* Should never happen with Apple HV in R3. */
#endif
}


/**
 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
 * VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitEptViolation(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
#ifndef IN_NEM_DARWIN
    Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_LEN
                         | HMVMX_READ_EXIT_INTERRUPTION_INFO
                         | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
                         | HMVMX_READ_IDT_VECTORING_INFO
                         | HMVMX_READ_IDT_VECTORING_ERROR_CODE
                         | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);

    /*
     * If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly.
     */
    VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
    {
        /*
         * If delivery of an event causes an EPT violation (true nested #PF and not MMIO),
         * we shall resolve the nested #PF and re-inject the original event.
         */
        if (VCPU_2_VMXSTATE(pVCpu).Event.fPending)
            STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatInjectReflectNPF);
    }
    else
    {
        Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
        return rcStrict;
    }

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    int rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    RTGCPHYS const GCPhys    = pVmxTransient->uGuestPhysicalAddr;
    uint64_t const uExitQual = pVmxTransient->uExitQual;
    AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", uExitQual));

    RTGCUINT uErrorCode = 0;
    if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH)
        uErrorCode |= X86_TRAP_PF_ID;
    if (uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
        uErrorCode |= X86_TRAP_PF_RW;
    if (uExitQual & (VMX_EXIT_QUAL_EPT_ENTRY_READ | VMX_EXIT_QUAL_EPT_ENTRY_WRITE | VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE))
        uErrorCode |= X86_TRAP_PF_P;

    PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
    Log4Func(("at %#RX64 (%#RX64 errcode=%#x) cs:rip=%#04x:%08RX64\n", GCPhys, uExitQual, uErrorCode, pCtx->cs.Sel, pCtx->rip));

    PVMCC    pVM  = pVCpu->CTX_SUFF(pVM);

    /*
     * Handle the pagefault trap for the nested shadow table.
     */
    TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
    rcStrict = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, pCtx, GCPhys);
    TRPMResetTrap(pVCpu);

    /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
    if (   rcStrict == VINF_SUCCESS
        || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
        || rcStrict == VERR_PAGE_NOT_PRESENT)
    {
        /* Successfully synced our nested page tables. */
        STAM_COUNTER_INC(&VCPU_2_VMXSTATS(pVCpu).StatExitReasonNpf);
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
        return VINF_SUCCESS;
    }
    Log4Func(("EPT return to ring-3 rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
    return rcStrict;

#else /* IN_NEM_DARWIN */
    PVM pVM = pVCpu->CTX_SUFF(pVM);
    uint64_t const uHostTsc = ASMReadTSC(); RT_NOREF(uHostTsc);
    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
    vmxHCImportGuestRip(pVCpu);
    vmxHCImportGuestSegReg<X86_SREG_CS>(pVCpu);

    /*
     * Ask PGM for information about the given GCPhys.  We need to check if we're
     * out of sync first.
     */
    NEMHCDARWINHMACPCCSTATE State = { RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE),
                                      false,
                                      false };
    PGMPHYSNEMPAGEINFO      Info;
    int rc = PGMPhysNemPageInfoChecker(pVM, pVCpu, pVmxTransient->uGuestPhysicalAddr, State.fWriteAccess, &Info,
                                       nemR3DarwinHandleMemoryAccessPageCheckerCallback, &State);
    if (RT_SUCCESS(rc))
    {
        if (Info.fNemProt & (  RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
                             ? NEM_PAGE_PROT_WRITE : NEM_PAGE_PROT_READ))
        {
            if (State.fCanResume)
            {
                Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; restarting\n",
                      pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
                      pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
                      Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
                      State.fDidSomething ? "" : " no-change"));
                EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_MEMORY_ACCESS),
                                 pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);
                return VINF_SUCCESS;
            }
        }

        Log4(("MemExit/%u: %04x:%08RX64: %RGp (=>%RHp) %s fProt=%u%s%s%s; emulating\n",
              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
              pVmxTransient->uGuestPhysicalAddr, Info.HCPhys, g_apszPageStates[Info.u2NemState], Info.fNemProt,
              Info.fHasHandlers ? " handlers" : "", Info.fZeroPage    ? " zero-pg" : "",
              State.fDidSomething ? "" : " no-change"));
    }
    else
        Log4(("MemExit/%u: %04x:%08RX64: %RGp rc=%Rrc%s; emulating\n",
              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
              pVmxTransient->uGuestPhysicalAddr, rc, State.fDidSomething ? " modified-backing" : ""));

    /*
     * Emulate the memory access, either access handler or special memory.
     */
    PCEMEXITREC pExitRec = EMHistoryAddExit(pVCpu,
                                              RT_BOOL(pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)
                                            ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
                                            : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
                                            pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, uHostTsc);

    rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    VBOXSTRICTRC rcStrict;
    if (!pExitRec)
        rcStrict = IEMExecOne(pVCpu);
    else
    {
        /* Frequent access or probing. */
        rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
        Log4(("MemExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
              pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
              VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
    }

    ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);

    Log4Func(("EPT return rcStrict=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
    return rcStrict;
#endif /* IN_NEM_DARWIN */
}

#ifdef VBOX_WITH_NESTED_HWVIRT_VMX

/**
 * VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmclear(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_INFO
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
                                   | CPUMCTX_EXTRN_SREG_MASK
                                   | CPUMCTX_EXTRN_HWVIRT
                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);

    VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmlaunch(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
       otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
    VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMLAUNCH);
    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
        if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
            rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
    }
    Assert(rcStrict != VINF_IEM_RAISED_XCPT);
    return rcStrict;
}


/**
 * VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmptrld(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_INFO
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
                                   | CPUMCTX_EXTRN_SREG_MASK
                                   | CPUMCTX_EXTRN_HWVIRT
                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);

    VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmptrst(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_INFO
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
                                   | CPUMCTX_EXTRN_SREG_MASK
                                   | CPUMCTX_EXTRN_HWVIRT
                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);

    VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmread(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /*
     * Strictly speaking we should not get VMREAD VM-exits for shadow VMCS fields and
     * thus might not need to import the shadow VMCS state, it's safer just in case
     * code elsewhere dares look at unsynced VMCS fields.
     */
    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_INFO
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
                                   | CPUMCTX_EXTRN_SREG_MASK
                                   | CPUMCTX_EXTRN_HWVIRT
                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
    if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
        HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);

    VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);

# if 0 //ndef IN_NEM_DARWIN /** @todo this needs serious tuning still, slows down things enormously. */
        /* Try for exit optimization.  This is on the following instruction
           because it would be a waste of time to have to reinterpret the
           already decoded vmwrite instruction. */
        PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndType(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_VMREAD));
        if (pExitRec)
        {
            /* Frequent access or probing. */
            rc = vmxHCImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
            AssertRCReturn(rc, rc);

            rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
            Log4(("vmread/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
                  pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
                  VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
        }
# endif
    }
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmresume(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
       otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    STAM_PROFILE_ADV_START(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
    VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbExitInstr, VMXINSTRID_VMRESUME);
    STAM_PROFILE_ADV_STOP(&VCPU_2_VMXSTATS(pVCpu).StatExitVmentry, z);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_ALL_GUEST);
        if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
            rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
    }
    Assert(rcStrict != VINF_IEM_RAISED_XCPT);
    return rcStrict;
}


/**
 * VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmwrite(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /*
     * Although we should not get VMWRITE VM-exits for shadow VMCS fields, since our HM hook
     * gets invoked when IEM's VMWRITE instruction emulation modifies the current VMCS and it
     * flags re-loading the entire shadow VMCS, we should save the entire shadow VMCS here.
     */
    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_INFO
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
                                   | CPUMCTX_EXTRN_SREG_MASK
                                   | CPUMCTX_EXTRN_HWVIRT
                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
    if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
        HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);

    VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmxoff(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_CR4
                                   | CPUMCTX_EXTRN_HWVIRT
                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbExitInstr);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_HWVIRT);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmxon(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_INFO
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
                                   | CPUMCTX_EXTRN_SREG_MASK
                                   | CPUMCTX_EXTRN_HWVIRT
                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);

    VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_HWVIRT);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * VM-exit handler for INVVPID (VMX_EXIT_INVVPID). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitInvvpid(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_INFO
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
                                   | CPUMCTX_EXTRN_SREG_MASK
                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);

    VBOXSTRICTRC rcStrict = IEMExecDecodedInvvpid(pVCpu, &ExitInfo);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT
/**
 * VM-exit handler for INVEPT (VMX_EXIT_INVEPT). Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitInvept(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_INFO
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    int rc = vmxHCImportGuestState<  CPUMCTX_EXTRN_RSP
                                   | CPUMCTX_EXTRN_SREG_MASK
                                   | IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
    AssertRCReturn(rc, rc);

    HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);

    VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
    HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);

    VBOXSTRICTRC rcStrict = IEMExecDecodedInvept(pVCpu, &ExitInfo);
    if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
    else if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}
# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */
#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
/** @} */


#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
/** @name Nested-guest VM-exit handlers.
 * @{
 */
/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- Nested-guest VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */

/**
 * Nested-guest VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
 * Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitXcptOrNmiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);

    uint64_t const uExitIntInfo = pVmxTransient->uExitIntInfo;
    uint32_t const uExitIntType = VMX_EXIT_INT_INFO_TYPE(uExitIntInfo);
    Assert(VMX_EXIT_INT_INFO_IS_VALID(uExitIntInfo));

    switch (uExitIntType)
    {
# ifndef IN_NEM_DARWIN
        /*
         * Physical NMIs:
         *     We shouldn't direct host physical NMIs to the nested-guest. Dispatch it to the host.
         */
        case VMX_EXIT_INT_INFO_TYPE_NMI:
            return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
# endif

        /*
         * Hardware exceptions,
         * Software exceptions,
         * Privileged software exceptions:
         *     Figure out if the exception must be delivered to the guest or the nested-guest.
         */
        case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
        case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
        case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
        {
            vmxHCReadToTransient<  HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
                                 | HMVMX_READ_EXIT_INSTR_LEN
                                 | HMVMX_READ_IDT_VECTORING_INFO
                                 | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);

            PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
            if (CPUMIsGuestVmxXcptInterceptSet(pCtx, VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo), pVmxTransient->uExitIntErrorCode))
            {
                /* Exit qualification is required for debug and page-fault exceptions. */
                vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);

                /*
                 * For VM-exits due to software exceptions (those generated by INT3 or INTO) and privileged
                 * software exceptions (those generated by INT1/ICEBP) we need to supply the VM-exit instruction
                 * length. However, if delivery of a software interrupt, software exception or privileged
                 * software exception causes a VM-exit, that too provides the VM-exit instruction length.
                 */
                VMXVEXITINFO const      ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
                VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT(pVmxTransient->uExitIntInfo,
                                                                               pVmxTransient->uExitIntErrorCode,
                                                                               pVmxTransient->uIdtVectoringInfo,
                                                                               pVmxTransient->uIdtVectoringErrorCode);
#ifdef DEBUG_ramshankar
                vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
                Log4Func(("exit_int_info=%#RX32 err_code=%#RX32 exit_qual=%#RX64\n",
                          pVmxTransient->uExitIntInfo, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual));
                if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
                    Log4Func(("idt_info=%#RX32 idt_errcode=%#RX32 cr2=%#RX64\n",
                              pVmxTransient->uIdtVectoringInfo, pVmxTransient->uIdtVectoringErrorCode, pCtx->cr2));
#endif
                return IEMExecVmxVmexitXcpt(pVCpu, &ExitInfo, &ExitEventInfo);
            }

            /* Nested paging is currently a requirement, otherwise we would need to handle shadow #PFs in vmxHCExitXcptPF. */
            Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
            return vmxHCExitXcpt(pVCpu, pVmxTransient);
        }

        /*
         * Software interrupts:
         *    VM-exits cannot be caused by software interrupts.
         *
         * External interrupts:
         *    This should only happen when "acknowledge external interrupts on VM-exit"
         *    control is set. However, we never set this when executing a guest or
         *    nested-guest. For nested-guests it is emulated while injecting interrupts into
         *    the guest.
         */
        case VMX_EXIT_INT_INFO_TYPE_SW_INT:
        case VMX_EXIT_INT_INFO_TYPE_EXT_INT:
        default:
        {
            VCPU_2_VMXSTATE(pVCpu).u32HMError = pVmxTransient->uExitIntInfo;
            return VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
        }
    }
}


/**
 * Nested-guest VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT).
 * Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitTripleFaultNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    return IEMExecVmxVmexitTripleFault(pVCpu);
}


/**
 * Nested-guest VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitIntWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INT_WINDOW_EXIT))
        return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
    return vmxHCExitIntWindow(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitNmiWindowNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_NMI_WINDOW_EXIT))
        return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, 0 /* uExitQual */);
    return vmxHCExitIntWindow(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH).
 * Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitTaskSwitchNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_LEN
                         | HMVMX_READ_IDT_VECTORING_INFO
                         | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);

    VMXVEXITINFO const      ExitInfo      = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
    VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
                                                                            pVmxTransient->uIdtVectoringErrorCode);
    return IEMExecVmxVmexitTaskSwitch(pVCpu, &ExitInfo, &ExitEventInfo);
}


/**
 * Nested-guest VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitHltNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_HLT_EXIT))
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
    }
    return vmxHCExitHlt(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitInvlpgNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
    {
        vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                             | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
    }
    return vmxHCExitInvlpg(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitRdpmcNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDPMC_EXIT))
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
    }
    return vmxHCExitRdpmc(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for VMREAD (VMX_EXIT_VMREAD) and VMWRITE
 * (VMX_EXIT_VMWRITE). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVmreadVmwriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    Assert(   pVmxTransient->uExitReason == VMX_EXIT_VMREAD
           || pVmxTransient->uExitReason == VMX_EXIT_VMWRITE);

    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);

    uint8_t const iGReg = pVmxTransient->ExitInstrInfo.VmreadVmwrite.iReg2;
    Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
    uint64_t u64VmcsField = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;

    HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
    if (!CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
        u64VmcsField &= UINT64_C(0xffffffff);

    if (CPUMIsGuestVmxVmreadVmwriteInterceptSet(pVCpu, pVmxTransient->uExitReason, u64VmcsField))
    {
        vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                             | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
    }

    if (pVmxTransient->uExitReason == VMX_EXIT_VMREAD)
        return vmxHCExitVmread(pVCpu, pVmxTransient);
    return vmxHCExitVmwrite(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitRdtscNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
    }

    return vmxHCExitRdtsc(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX).
 * Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitMovCRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);

    VBOXSTRICTRC rcStrict;
    uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual);
    switch (uAccessType)
    {
        case VMX_EXIT_QUAL_CRX_ACCESS_WRITE:
        {
            uint8_t const iCrReg   = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
            uint8_t const iGReg    = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
            Assert(iGReg < RT_ELEMENTS(pVCpu->cpum.GstCtx.aGRegs));
            uint64_t const uNewCrX = pVCpu->cpum.GstCtx.aGRegs[iGReg].u64;

            bool fIntercept;
            switch (iCrReg)
            {
                case 0:
                case 4:
                    fIntercept = CPUMIsGuestVmxMovToCr0Cr4InterceptSet(&pVCpu->cpum.GstCtx, iCrReg, uNewCrX);
                    break;

                case 3:
                    fIntercept = CPUMIsGuestVmxMovToCr3InterceptSet(pVCpu, uNewCrX);
                    break;

                case 8:
                    fIntercept = CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_CR8_LOAD_EXIT);
                    break;

                default:
                    fIntercept = false;
                    break;
            }
            if (fIntercept)
            {
                VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
                rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
            }
            else
            {
                int const rc = vmxHCImportGuestState<IEM_CPUMCTX_EXTRN_MUST_MASK>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
                AssertRCReturn(rc, rc);
                rcStrict = vmxHCExitMovToCrX(pVCpu, pVmxTransient->cbExitInstr, iGReg, iCrReg);
            }
            break;
        }

        case VMX_EXIT_QUAL_CRX_ACCESS_READ:
        {
            /*
             * CR0/CR4 reads do not cause VM-exits, the read-shadow is used (subject to masking).
             * CR2 reads do not cause a VM-exit.
             * CR3 reads cause a VM-exit depending on the "CR3 store exiting" control.
             * CR8 reads cause a VM-exit depending on the "CR8 store exiting" control.
             */
            uint8_t const iCrReg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
            if (   iCrReg == 3
                || iCrReg == 8)
            {
                static const uint32_t s_auCrXReadIntercepts[] = { 0, 0, 0, VMX_PROC_CTLS_CR3_STORE_EXIT, 0,
                                                                  0, 0, 0, VMX_PROC_CTLS_CR8_STORE_EXIT };
                uint32_t const uIntercept = s_auCrXReadIntercepts[iCrReg];
                if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, uIntercept))
                {
                    VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
                    rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
                }
                else
                {
                    uint8_t const iGReg = VMX_EXIT_QUAL_CRX_GENREG(pVmxTransient->uExitQual);
                    rcStrict = vmxHCExitMovFromCrX(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, iGReg, iCrReg);
                }
            }
            else
            {
                AssertMsgFailed(("MOV from CR%d VM-exit must not happen\n", iCrReg));
                HMVMX_UNEXPECTED_EXIT_RET(pVCpu, iCrReg);
            }
            break;
        }

        case VMX_EXIT_QUAL_CRX_ACCESS_CLTS:
        {
            PCVMXVVMCS const pVmcsNstGst  = &pVCpu->cpum.GstCtx.hwvirt.vmx.Vmcs;
            uint64_t const   uGstHostMask = pVmcsNstGst->u64Cr0Mask.u;
            uint64_t const   uReadShadow  = pVmcsNstGst->u64Cr0ReadShadow.u;
            if (   (uGstHostMask & X86_CR0_TS)
                && (uReadShadow  & X86_CR0_TS))
            {
                VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
                rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
            }
            else
                rcStrict = vmxHCExitClts(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr);
            break;
        }

        case VMX_EXIT_QUAL_CRX_ACCESS_LMSW:        /* LMSW (Load Machine-Status Word into CR0) */
        {
            RTGCPTR        GCPtrEffDst;
            uint16_t const uNewMsw     = VMX_EXIT_QUAL_CRX_LMSW_DATA(pVmxTransient->uExitQual);
            bool const     fMemOperand = VMX_EXIT_QUAL_CRX_LMSW_OP_MEM(pVmxTransient->uExitQual);
            if (fMemOperand)
            {
                vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
                GCPtrEffDst = pVmxTransient->uGuestLinearAddr;
            }
            else
                GCPtrEffDst = NIL_RTGCPTR;

            if (CPUMIsGuestVmxLmswInterceptSet(&pVCpu->cpum.GstCtx, uNewMsw))
            {
                VMXVEXITINFO ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
                ExitInfo.u64GuestLinearAddr = GCPtrEffDst;
                rcStrict = IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
            }
            else
                rcStrict = vmxHCExitLmsw(pVCpu, pVmxTransient->pVmcsInfo, pVmxTransient->cbExitInstr, uNewMsw, GCPtrEffDst);
            break;
        }

        default:
        {
            AssertMsgFailed(("Unrecognized Mov CRX access type %#x\n", uAccessType));
            HMVMX_UNEXPECTED_EXIT_RET(pVCpu, uAccessType);
        }
    }

    if (rcStrict == VINF_IEM_RAISED_XCPT)
    {
        ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
        rcStrict = VINF_SUCCESS;
    }
    return rcStrict;
}


/**
 * Nested-guest VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX).
 * Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitMovDRxNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MOV_DR_EXIT))
    {
        vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                             | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
    }
    return vmxHCExitMovDRx(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR).
 * Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitIoInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);

    uint32_t const uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
    uint8_t  const uIOSize = VMX_EXIT_QUAL_IO_SIZE(pVmxTransient->uExitQual);
    AssertReturn(uIOSize <= 3 && uIOSize != 2, VERR_VMX_IPE_1);

    static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 };   /* Size of the I/O accesses in bytes. */
    uint8_t const cbAccess = s_aIOSizes[uIOSize];
    if (CPUMIsGuestVmxIoInterceptSet(pVCpu, uIOPort, cbAccess))
    {
        /*
         * IN/OUT instruction:
         *   - Provides VM-exit instruction length.
         *
         * INS/OUTS instruction:
         *   - Provides VM-exit instruction length.
         *   - Provides Guest-linear address.
         *   - Optionally provides VM-exit instruction info (depends on CPU feature).
         */
        PVMCC pVM = pVCpu->CTX_SUFF(pVM);
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);

        /* Make sure we don't use stale/uninitialized VMX-transient info. below. */
        pVmxTransient->ExitInstrInfo.u  = 0;
        pVmxTransient->uGuestLinearAddr = 0;

        bool const fVmxInsOutsInfo = pVM->cpum.ro.GuestFeatures.fVmxInsOutInfo;
        bool const fIOString       = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
        if (fIOString)
        {
            vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
            if (fVmxInsOutsInfo)
            {
                Assert(RT_BF_GET(g_HmMsrs.u.vmx.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS)); /* Paranoia. */
                vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
            }
        }

        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_AND_LIN_ADDR_FROM_TRANSIENT(pVmxTransient);
        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
    }
    return vmxHCExitIoInstr(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
 */
HMVMX_EXIT_DECL vmxHCExitRdmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    uint32_t fMsrpm;
    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
        fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
    else
        fMsrpm = VMXMSRPM_EXIT_RD;

    if (fMsrpm & VMXMSRPM_EXIT_RD)
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
    }
    return vmxHCExitRdmsr(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
 */
HMVMX_EXIT_DECL vmxHCExitWrmsrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    uint32_t fMsrpm;
    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_MSR_BITMAPS))
        fMsrpm = CPUMGetVmxMsrPermission(pVCpu->cpum.GstCtx.hwvirt.vmx.abMsrBitmap, pVCpu->cpum.GstCtx.ecx);
    else
        fMsrpm = VMXMSRPM_EXIT_WR;

    if (fMsrpm & VMXMSRPM_EXIT_WR)
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
    }
    return vmxHCExitWrmsr(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitMwaitNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MWAIT_EXIT))
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
    }
    return vmxHCExitMwait(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional
 * VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitMtfNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /** @todo NSTVMX: Should consider debugging nested-guests using VM debugger. */
    vmxHCReadToTransient<HMVMX_READ_GUEST_PENDING_DBG_XCPTS>(pVCpu, pVmxTransient);
    VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_DBG_XCPTS_FROM_TRANSIENT(pVmxTransient);
    return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
}


/**
 * Nested-guest VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitMonitorNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_MONITOR_EXIT))
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
    }
    return vmxHCExitMonitor(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitPauseNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /** @todo NSTVMX: Think about this more. Does the outer guest need to intercept
     *        PAUSE when executing a nested-guest? If it does not, we would not need
     *        to check for the intercepts here. Just call VM-exit... */

    /* The CPU would have already performed the necessary CPL checks for PAUSE-loop exiting. */
    if (   CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_PAUSE_EXIT)
        || CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
    }
    return vmxHCExitPause(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for when the TPR value is lowered below the
 * specified threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitTprBelowThresholdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_USE_TPR_SHADOW))
    {
        vmxHCReadToTransient<HMVMX_READ_GUEST_PENDING_DBG_XCPTS>(pVCpu, pVmxTransient);
        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_DBG_XCPTS_FROM_TRANSIENT(pVmxTransient);
        return IEMExecVmxVmexitTrapLike(pVCpu, &ExitInfo);
    }
    return vmxHCExitTprBelowThreshold(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional
 * VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitApicAccessNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_LEN
                         | HMVMX_READ_IDT_VECTORING_INFO
                         | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);

    Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_APIC_ACCESS));

    Log4Func(("at offset %#x type=%u\n", VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual),
              VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual)));

    VMXVEXITINFO const      ExitInfo      = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_FROM_TRANSIENT(pVmxTransient);
    VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
                                                                            pVmxTransient->uIdtVectoringErrorCode);
    return IEMExecVmxVmexitApicAccess(pVCpu, &ExitInfo, &ExitEventInfo);
}


/**
 * Nested-guest VM-exit handler for APIC write emulation (VMX_EXIT_APIC_WRITE).
 * Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitApicWriteNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_APIC_REG_VIRT));
    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
    return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
}


/**
 * Nested-guest VM-exit handler for virtualized EOI (VMX_EXIT_VIRTUALIZED_EOI).
 * Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitVirtEoiNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_VIRT_INT_DELIVERY));
    vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
    return IEMExecVmxVmexit(pVCpu, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
}


/**
 * Nested-guest VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitRdtscpNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_RDTSC_EXIT))
    {
        Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_RDTSCP));
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
    }
    return vmxHCExitRdtscp(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
 */
HMVMX_EXIT_NSRC_DECL vmxHCExitWbinvdNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_WBINVD_EXIT))
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
    }
    return vmxHCExitWbinvd(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitInvpcidNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    if (CPUMIsGuestVmxProcCtlsSet(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS_INVLPG_EXIT))
    {
        Assert(CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_INVPCID));
        vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                             | HMVMX_READ_EXIT_INSTR_INFO
                             | HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
        VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
        return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
    }
    return vmxHCExitInvpcid(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for invalid-guest state
 * (VMX_EXIT_ERR_INVALID_GUEST_STATE). Error VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitErrInvalidGuestStateNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

    /*
     * Currently this should never happen because we fully emulate VMLAUNCH/VMRESUME in IEM.
     * So if it does happen, it indicates a bug possibly in the hardware-assisted VMX code.
     * Handle it like it's in an invalid guest state of the outer guest.
     *
     * When the fast path is implemented, this should be changed to cause the corresponding
     * nested-guest VM-exit.
     */
    return vmxHCExitErrInvalidGuestState(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for instructions that cause VM-exits unconditionally
 * and only provide the instruction length.
 *
 * Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitInstrNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

#ifdef VBOX_STRICT
    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
    switch (pVmxTransient->uExitReason)
    {
        case VMX_EXIT_ENCLS:
            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_ENCLS_EXIT));
            break;

        case VMX_EXIT_VMFUNC:
            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_VMFUNC));
            break;
    }
#endif

    vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_LEN>(pVCpu, pVmxTransient);
    return IEMExecVmxVmexitInstr(pVCpu, pVmxTransient->uExitReason, pVmxTransient->cbExitInstr);
}


/**
 * Nested-guest VM-exit handler for instructions that provide instruction length as
 * well as more information.
 *
 * Unconditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitInstrWithInfoNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_NESTED_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);

# ifdef VBOX_STRICT
    PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
    switch (pVmxTransient->uExitReason)
    {
        case VMX_EXIT_GDTR_IDTR_ACCESS:
        case VMX_EXIT_LDTR_TR_ACCESS:
            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_DESC_TABLE_EXIT));
            break;

        case VMX_EXIT_RDRAND:
            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDRAND_EXIT));
            break;

        case VMX_EXIT_RDSEED:
            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_RDSEED_EXIT));
            break;

        case VMX_EXIT_XSAVES:
        case VMX_EXIT_XRSTORS:
            /** @todo NSTVMX: Verify XSS-bitmap. */
            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_XSAVES_XRSTORS));
            break;

        case VMX_EXIT_UMWAIT:
        case VMX_EXIT_TPAUSE:
            Assert(CPUMIsGuestVmxProcCtlsSet(pCtx, VMX_PROC_CTLS_RDTSC_EXIT));
            Assert(CPUMIsGuestVmxProcCtls2Set(pCtx, VMX_PROC_CTLS2_USER_WAIT_PAUSE));
            break;

        case VMX_EXIT_LOADIWKEY:
            Assert(CPUMIsGuestVmxProcCtls3Set(pCtx, VMX_PROC_CTLS3_LOADIWKEY_EXIT));
            break;
    }
# endif

    vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                         | HMVMX_READ_EXIT_INSTR_LEN
                         | HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
    VMXVEXITINFO const ExitInfo = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_INFO_FROM_TRANSIENT(pVmxTransient);
    return IEMExecVmxVmexitInstrWithInfo(pVCpu, &ExitInfo);
}

# ifdef VBOX_WITH_NESTED_HWVIRT_VMX_EPT

/**
 * Nested-guest VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION).
 * Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitEptViolationNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_EPT))
    {
        vmxHCReadToTransient<  HMVMX_READ_EXIT_QUALIFICATION
                             | HMVMX_READ_EXIT_INSTR_LEN
                             | HMVMX_READ_EXIT_INTERRUPTION_INFO
                             | HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE
                             | HMVMX_READ_IDT_VECTORING_INFO
                             | HMVMX_READ_IDT_VECTORING_ERROR_CODE
                             | HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
        int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
        AssertRCReturn(rc, rc);

        /*
         * If it's our VMEXIT, we're responsible for re-injecting any event which delivery
         * might have triggered this VMEXIT.  If we forward the problem to the inner VMM,
         * it's its problem to deal with that issue and we'll clear the recovered event.
         */
        VBOXSTRICTRC rcStrict = vmxHCCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
        if (RT_LIKELY(rcStrict == VINF_SUCCESS))
        { /*likely*/ }
        else
        {
            Assert(rcStrict != VINF_HM_DOUBLE_FAULT);
            return rcStrict;
        }
        uint32_t const fClearEventOnForward = VCPU_2_VMXSTATE(pVCpu).Event.fPending; /* paranoia. should not inject events below.  */

        RTGCPHYS const GCPhysNestedFault = pVmxTransient->uGuestPhysicalAddr;
        uint64_t const uExitQual         = pVmxTransient->uExitQual;

        RTGCPTR GCPtrNestedFault;
        bool const fIsLinearAddrValid = RT_BOOL(uExitQual & VMX_EXIT_QUAL_EPT_LINEAR_ADDR_VALID);
        if (fIsLinearAddrValid)
        {
            vmxHCReadToTransient<HMVMX_READ_GUEST_LINEAR_ADDR>(pVCpu, pVmxTransient);
            GCPtrNestedFault = pVmxTransient->uGuestLinearAddr;
        }
        else
            GCPtrNestedFault = 0;

        RTGCUINT const uErr = ((uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_INSTR_FETCH) ? X86_TRAP_PF_ID : 0)
                            | ((uExitQual & VMX_EXIT_QUAL_EPT_ACCESS_WRITE)       ? X86_TRAP_PF_RW : 0)
                            | ((uExitQual & (  VMX_EXIT_QUAL_EPT_ENTRY_READ
                                             | VMX_EXIT_QUAL_EPT_ENTRY_WRITE
                                             | VMX_EXIT_QUAL_EPT_ENTRY_EXECUTE))  ? X86_TRAP_PF_P  : 0);

        PGMPTWALK Walk;
        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
        rcStrict = PGMR0NestedTrap0eHandlerNestedPaging(pVCpu, PGMMODE_EPT, uErr, pCtx, GCPhysNestedFault,
                                                        fIsLinearAddrValid, GCPtrNestedFault, &Walk);
        Log7Func(("PGM (uExitQual=%#RX64, %RGp, %RGv) -> %Rrc (fFailed=%d)\n",
                  uExitQual, GCPhysNestedFault, GCPtrNestedFault, VBOXSTRICTRC_VAL(rcStrict), Walk.fFailed));
        if (RT_SUCCESS(rcStrict))
            return rcStrict;

        if (fClearEventOnForward)
            VCPU_2_VMXSTATE(pVCpu).Event.fPending = false;

        VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
                                                                                pVmxTransient->uIdtVectoringErrorCode);
        if (Walk.fFailed & PGM_WALKFAIL_EPT_VIOLATION)
        {
            VMXVEXITINFO const ExitInfo
                = VMXVEXITINFO_INIT_WITH_QUAL_AND_INSTR_LEN_AND_GST_ADDRESSES(VMX_EXIT_EPT_VIOLATION,
                                                                              pVmxTransient->uExitQual,
                                                                              pVmxTransient->cbExitInstr,
                                                                              pVmxTransient->uGuestLinearAddr,
                                                                              pVmxTransient->uGuestPhysicalAddr);
            return IEMExecVmxVmexitEptViolation(pVCpu, &ExitInfo, &ExitEventInfo);
        }

        AssertMsgReturn(Walk.fFailed & PGM_WALKFAIL_EPT_MISCONFIG,
                        ("uErr=%#RX32 uExitQual=%#RX64 GCPhysNestedFault=%#RGp GCPtrNestedFault=%#RGv\n",
                         (uint32_t)uErr, uExitQual, GCPhysNestedFault, GCPtrNestedFault),
                        rcStrict);
        return IEMExecVmxVmexitEptMisconfig(pVCpu, pVmxTransient->uGuestPhysicalAddr, &ExitEventInfo);
    }

    return vmxHCExitEptViolation(pVCpu, pVmxTransient);
}


/**
 * Nested-guest VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
 * Conditional VM-exit.
 */
HMVMX_EXIT_DECL vmxHCExitEptMisconfigNested(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient)
{
    HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
    Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);

    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    if (CPUMIsGuestVmxProcCtls2Set(&pVCpu->cpum.GstCtx, VMX_PROC_CTLS2_EPT))
    {
        vmxHCReadToTransient<HMVMX_READ_GUEST_PHYSICAL_ADDR>(pVCpu, pVmxTransient);
        int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_ALL>(pVCpu, pVmcsInfo, __FUNCTION__);
        AssertRCReturn(rc, rc);

        PGMPTWALK Walk;
        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
        RTGCPHYS const GCPhysNestedFault = pVmxTransient->uGuestPhysicalAddr;
        VBOXSTRICTRC rcStrict = PGMR0NestedTrap0eHandlerNestedPaging(pVCpu, PGMMODE_EPT, X86_TRAP_PF_RSVD, pCtx,
                                                                     GCPhysNestedFault, false /* fIsLinearAddrValid */,
                                                                     0 /* GCPtrNestedFault */, &Walk);
        if (RT_SUCCESS(rcStrict))
        {
            AssertMsgFailed(("Shouldn't happen with the way we have programmed the EPT shadow tables\n"));
            return rcStrict;
        }

        AssertMsg(Walk.fFailed & PGM_WALKFAIL_EPT_MISCONFIG, ("GCPhysNestedFault=%#RGp\n", GCPhysNestedFault));
        vmxHCReadToTransient<  HMVMX_READ_IDT_VECTORING_INFO
                             | HMVMX_READ_IDT_VECTORING_ERROR_CODE>(pVCpu, pVmxTransient);

        VMXVEXITEVENTINFO const ExitEventInfo = VMXVEXITEVENTINFO_INIT_ONLY_IDT(pVmxTransient->uIdtVectoringInfo,
                                                                                pVmxTransient->uIdtVectoringErrorCode);
        return IEMExecVmxVmexitEptMisconfig(pVCpu, pVmxTransient->uGuestPhysicalAddr, &ExitEventInfo);
    }

    return vmxHCExitEptMisconfig(pVCpu, pVmxTransient);
}

# endif /* VBOX_WITH_NESTED_HWVIRT_VMX_EPT */

/** @} */
#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */


/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
 *  probes.
 *
 * The following few functions and associated structure contains the bloat
 * necessary for providing detailed debug events and dtrace probes as well as
 * reliable host side single stepping.  This works on the principle of
 * "subclassing" the normal execution loop and workers.  We replace the loop
 * method completely and override selected helpers to add necessary adjustments
 * to their core operation.
 *
 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
 * any performance for debug and analysis features.
 *
 * @{
 */

/**
 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
 * the debug run loop.
 */
typedef struct VMXRUNDBGSTATE
{
    /** The RIP we started executing at.  This is for detecting that we stepped.  */
    uint64_t    uRipStart;
    /** The CS we started executing with.  */
    uint16_t    uCsStart;

    /** Whether we've actually modified the 1st execution control field. */
    bool        fModifiedProcCtls : 1;
    /** Whether we've actually modified the 2nd execution control field. */
    bool        fModifiedProcCtls2 : 1;
    /** Whether we've actually modified the exception bitmap. */
    bool        fModifiedXcptBitmap : 1;

    /** We desire the modified the CR0 mask to be cleared. */
    bool        fClearCr0Mask : 1;
    /** We desire the modified the CR4 mask to be cleared. */
    bool        fClearCr4Mask : 1;
    /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
    uint32_t    fCpe1Extra;
    /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
    uint32_t    fCpe1Unwanted;
    /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
    uint32_t    fCpe2Extra;
    /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
    uint32_t    bmXcptExtra;
    /** The sequence number of the Dtrace provider settings the state was
     *  configured against. */
    uint32_t    uDtraceSettingsSeqNo;
    /** VM-exits to check (one bit per VM-exit). */
    uint32_t    bmExitsToCheck[3];

    /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
    uint32_t    fProcCtlsInitial;
    /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
    uint32_t    fProcCtls2Initial;
    /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
    uint32_t    bmXcptInitial;
} VMXRUNDBGSTATE;
AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;


/**
 * Initializes the VMXRUNDBGSTATE structure.
 *
 * @param   pVCpu           The cross context virtual CPU structure of the
 *                          calling EMT.
 * @param   pVmxTransient   The VMX-transient structure.
 * @param   pDbgState       The debug state to initialize.
 */
static void vmxHCRunDebugStateInit(PVMCPUCC pVCpu, PCVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
{
    pDbgState->uRipStart            = pVCpu->cpum.GstCtx.rip;
    pDbgState->uCsStart             = pVCpu->cpum.GstCtx.cs.Sel;

    pDbgState->fModifiedProcCtls    = false;
    pDbgState->fModifiedProcCtls2   = false;
    pDbgState->fModifiedXcptBitmap  = false;
    pDbgState->fClearCr0Mask        = false;
    pDbgState->fClearCr4Mask        = false;
    pDbgState->fCpe1Extra           = 0;
    pDbgState->fCpe1Unwanted        = 0;
    pDbgState->fCpe2Extra           = 0;
    pDbgState->bmXcptExtra          = 0;
    pDbgState->fProcCtlsInitial     = pVmxTransient->pVmcsInfo->u32ProcCtls;
    pDbgState->fProcCtls2Initial    = pVmxTransient->pVmcsInfo->u32ProcCtls2;
    pDbgState->bmXcptInitial        = pVmxTransient->pVmcsInfo->u32XcptBitmap;
}


/**
 * Updates the VMSC fields with changes requested by @a pDbgState.
 *
 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
 * immediately before executing guest code, i.e. when interrupts are disabled.
 * We don't check status codes here as we cannot easily assert or return in the
 * latter case.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 * @param   pDbgState       The debug state.
 */
static void vmxHCPreRunGuestDebugStateApply(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
{
    /*
     * Ensure desired flags in VMCS control fields are set.
     * (Ignoring write failure here, as we're committed and it's just debug extras.)
     *
     * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
     *       there should be no stale data in pCtx at this point.
     */
    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
    if (   (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
        || (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
    {
        pVmcsInfo->u32ProcCtls |= pDbgState->fCpe1Extra;
        pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
        VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
        Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
        pDbgState->fModifiedProcCtls   = true;
    }

    if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
    {
        pVmcsInfo->u32ProcCtls2  |= pDbgState->fCpe2Extra;
        VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
        Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
        pDbgState->fModifiedProcCtls2  = true;
    }

    if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
    {
        pVmcsInfo->u32XcptBitmap |= pDbgState->bmXcptExtra;
        VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
        Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
        pDbgState->fModifiedXcptBitmap = true;
    }

    if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
    {
        pVmcsInfo->u64Cr0Mask = 0;
        VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR0_MASK, 0);
        Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
    }

    if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
    {
        pVmcsInfo->u64Cr4Mask = 0;
        VMX_VMCS_WRITE_NW(pVCpu, VMX_VMCS_CTRL_CR4_MASK, 0);
        Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
    }

    NOREF(pVCpu);
}


/**
 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
 * re-entry next time around.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 * @param   pDbgState       The debug state.
 * @param   rcStrict        The return code from executing the guest using single
 *                          stepping.
 */
static VBOXSTRICTRC vmxHCRunDebugStateRevert(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
                                             VBOXSTRICTRC rcStrict)
{
    /*
     * Restore VM-exit control settings as we may not reenter this function the
     * next time around.
     */
    PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;

    /* We reload the initial value, trigger what we can of recalculations the
       next time around.  From the looks of things, that's all that's required atm. */
    if (pDbgState->fModifiedProcCtls)
    {
        if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
            pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
        int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
        AssertRC(rc2);
        pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
    }

    /* We're currently the only ones messing with this one, so just restore the
       cached value and reload the field. */
    if (   pDbgState->fModifiedProcCtls2
        && pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
    {
        int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
        AssertRC(rc2);
        pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
    }

    /* If we've modified the exception bitmap, we restore it and trigger
       reloading and partial recalculation the next time around. */
    if (pDbgState->fModifiedXcptBitmap)
    {
        int rc2 = VMX_VMCS_WRITE_32(pVCpu, VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pDbgState->bmXcptInitial);
        AssertRC(rc2);
        pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
    }

    return rcStrict;
}


/**
 * Configures VM-exit controls for current DBGF and DTrace settings.
 *
 * This updates @a pDbgState and the VMCS execution control fields to reflect
 * the necessary VM-exits demanded by DBGF and DTrace.
 *
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure. May update
 *                          fUpdatedTscOffsettingAndPreemptTimer.
 * @param   pDbgState       The debug state.
 */
static void vmxHCPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
{
#ifndef IN_NEM_DARWIN
    /*
     * Take down the dtrace serial number so we can spot changes.
     */
    pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
    ASMCompilerBarrier();
#endif

    /*
     * We'll rebuild most of the middle block of data members (holding the
     * current settings) as we go along here, so start by clearing it all.
     */
    pDbgState->bmXcptExtra      = 0;
    pDbgState->fCpe1Extra       = 0;
    pDbgState->fCpe1Unwanted    = 0;
    pDbgState->fCpe2Extra       = 0;
    for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
        pDbgState->bmExitsToCheck[i] = 0;

    /*
     * Software interrupts (INT XXh) - no idea how to trigger these...
     */
    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
    if (   DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
        || VBOXVMM_INT_SOFTWARE_ENABLED())
    {
        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
    }

    /*
     * INT3 breakpoints - triggered by #BP exceptions.
     */
    if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
        pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);

    /*
     * Exception bitmap and XCPT events+probes.
     */
    for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
        if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
            pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);

    if (VBOXVMM_XCPT_DE_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
    if (VBOXVMM_XCPT_DB_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
    if (VBOXVMM_XCPT_BP_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
    if (VBOXVMM_XCPT_OF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
    if (VBOXVMM_XCPT_BR_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
    if (VBOXVMM_XCPT_UD_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
    if (VBOXVMM_XCPT_NM_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
    if (VBOXVMM_XCPT_DF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
    if (VBOXVMM_XCPT_TS_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
    if (VBOXVMM_XCPT_NP_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
    if (VBOXVMM_XCPT_SS_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
    if (VBOXVMM_XCPT_GP_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
    if (VBOXVMM_XCPT_PF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
    if (VBOXVMM_XCPT_MF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
    if (VBOXVMM_XCPT_AC_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
    if (VBOXVMM_XCPT_XF_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
    if (VBOXVMM_XCPT_VE_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
    if (VBOXVMM_XCPT_SX_ENABLED())  pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);

    if (pDbgState->bmXcptExtra)
        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);

    /*
     * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
     *
     * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
     *       So, when adding/changing/removing please don't forget to update it.
     *
     * Some of the macros are picking up local variables to save horizontal space,
     * (being able to see it in a table is the lesser evil here).
     */
#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
        (    DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
         ||  RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
        {   AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
        } else do { } while (0)
#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
        { \
            (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
            AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
        } else do { } while (0)
#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
        { \
            (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
            AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
        } else do { } while (0)
#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
        if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
        { \
            (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
            AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
            ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
        } else do { } while (0)

    SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH,         VMX_EXIT_TASK_SWITCH);   /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION,   VMX_EXIT_EPT_VIOLATION); /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG,   VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS,    VMX_EXIT_APIC_ACCESS);   /* feature dependent, nothing to enable here */
    SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE,     VMX_EXIT_APIC_WRITE);    /* feature dependent, nothing to enable here */

    SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID,              VMX_EXIT_CPUID);         /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID,              VMX_EXIT_CPUID);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC,             VMX_EXIT_GETSEC);        /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC,             VMX_EXIT_GETSEC);
    SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT,               VMX_EXIT_HLT,      VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT,               VMX_EXIT_HLT);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD,               VMX_EXIT_INVD);          /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD,               VMX_EXIT_INVD);
    SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG,             VMX_EXIT_INVLPG,   VMX_PROC_CTLS_INVLPG_EXIT);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG,             VMX_EXIT_INVLPG);
    SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC,              VMX_EXIT_RDPMC,    VMX_PROC_CTLS_RDPMC_EXIT);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC,              VMX_EXIT_RDPMC);
    SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC,              VMX_EXIT_RDTSC,    VMX_PROC_CTLS_RDTSC_EXIT);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC,              VMX_EXIT_RDTSC);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM,                VMX_EXIT_RSM);           /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM,                VMX_EXIT_RSM);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL,           VMX_EXIT_VMCALL);        /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL,           VMX_EXIT_VMCALL);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR,        VMX_EXIT_VMCLEAR);       /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR,        VMX_EXIT_VMCLEAR);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH,       VMX_EXIT_VMLAUNCH);      /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH,       VMX_EXIT_VMLAUNCH);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD,        VMX_EXIT_VMPTRLD);       /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD,        VMX_EXIT_VMPTRLD);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST,        VMX_EXIT_VMPTRST);       /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST,        VMX_EXIT_VMPTRST);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD,         VMX_EXIT_VMREAD);        /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD,         VMX_EXIT_VMREAD);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME,       VMX_EXIT_VMRESUME);      /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME,       VMX_EXIT_VMRESUME);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE,        VMX_EXIT_VMWRITE);       /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE,        VMX_EXIT_VMWRITE);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF,         VMX_EXIT_VMXOFF);        /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF,         VMX_EXIT_VMXOFF);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON,          VMX_EXIT_VMXON);         /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON,          VMX_EXIT_VMXON);

    if (   IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
        || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
    {
        int rc = vmxHCImportGuestStateEx(pVCpu, pVmxTransient->pVmcsInfo,
                                         CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
        AssertRC(rc);

#if 0 /** @todo fix me */
        pDbgState->fClearCr0Mask = true;
        pDbgState->fClearCr4Mask = true;
#endif
        if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
            pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
        if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
            pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
        pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
        /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT.  It would
                 require clearing here and in the loop if we start using it. */
        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
    }
    else
    {
        if (pDbgState->fClearCr0Mask)
        {
            pDbgState->fClearCr0Mask = false;
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR0);
        }
        if (pDbgState->fClearCr4Mask)
        {
            pDbgState->fClearCr4Mask = false;
            ASMAtomicUoOrU64(&VCPU_2_VMXSTATE(pVCpu).fCtxChanged, HM_CHANGED_GUEST_CR4);
        }
    }
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ,           VMX_EXIT_MOV_CRX);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE,          VMX_EXIT_MOV_CRX);

    if (   IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
        || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
    {
        /** @todo later, need to fix handler as it assumes this won't usually happen. */
        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
    }
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ,           VMX_EXIT_MOV_DRX);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE,          VMX_EXIT_MOV_DRX);

    SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR,              VMX_EXIT_RDMSR,    VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR,              VMX_EXIT_RDMSR);
    SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR,              VMX_EXIT_WRMSR,    VMX_PROC_CTLS_USE_MSR_BITMAPS);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR,              VMX_EXIT_WRMSR);
    SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT,              VMX_EXIT_MWAIT,    VMX_PROC_CTLS_MWAIT_EXIT);   /* paranoia */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT,              VMX_EXIT_MWAIT);
    SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR,            VMX_EXIT_MONITOR,  VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR,            VMX_EXIT_MONITOR);
#if 0 /** @todo too slow, fix handler. */
    SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE,              VMX_EXIT_PAUSE,    VMX_PROC_CTLS_PAUSE_EXIT);
#endif
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE,              VMX_EXIT_PAUSE);

    if (   IS_EITHER_ENABLED(pVM, INSTR_SGDT)
        || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
        || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
        || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
    {
        pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
    }
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT,               VMX_EXIT_GDTR_IDTR_ACCESS);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT,               VMX_EXIT_GDTR_IDTR_ACCESS);

    if (   IS_EITHER_ENABLED(pVM, INSTR_SLDT)
        || IS_EITHER_ENABLED(pVM, INSTR_STR)
        || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
        || IS_EITHER_ENABLED(pVM, INSTR_LTR))
    {
        pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
        ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
    }
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT,               VMX_EXIT_LDTR_TR_ACCESS);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR,                VMX_EXIT_LDTR_TR_ACCESS);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT,               VMX_EXIT_LDTR_TR_ACCESS);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR,                VMX_EXIT_LDTR_TR_ACCESS);

    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT,         VMX_EXIT_INVEPT);        /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT,         VMX_EXIT_INVEPT);
    SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP,             VMX_EXIT_RDTSCP,   VMX_PROC_CTLS_RDTSC_EXIT);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP,             VMX_EXIT_RDTSCP);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID,        VMX_EXIT_INVVPID);       /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID,        VMX_EXIT_INVVPID);
    SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD,             VMX_EXIT_WBINVD,   VMX_PROC_CTLS2_WBINVD_EXIT);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD,             VMX_EXIT_WBINVD);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV,             VMX_EXIT_XSETBV);        /* unconditional */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV,             VMX_EXIT_XSETBV);
    SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND,             VMX_EXIT_RDRAND,   VMX_PROC_CTLS2_RDRAND_EXIT);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND,             VMX_EXIT_RDRAND);
    SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID,        VMX_EXIT_INVPCID,  VMX_PROC_CTLS_INVLPG_EXIT);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID,        VMX_EXIT_INVPCID);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC,         VMX_EXIT_VMFUNC);        /* unconditional for the current setup */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC,         VMX_EXIT_VMFUNC);
    SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED,             VMX_EXIT_RDSEED,   VMX_PROC_CTLS2_RDSEED_EXIT);
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED,             VMX_EXIT_RDSEED);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES,             VMX_EXIT_XSAVES);        /* unconditional (enabled by host, guest cfg) */
    SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES,              VMX_EXIT_XSAVES);
    SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS,            VMX_EXIT_XRSTORS);       /* unconditional (enabled by host, guest cfg) */
    SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS,            VMX_EXIT_XRSTORS);

#undef IS_EITHER_ENABLED
#undef SET_ONLY_XBM_IF_EITHER_EN
#undef SET_CPE1_XBM_IF_EITHER_EN
#undef SET_CPEU_XBM_IF_EITHER_EN
#undef SET_CPE2_XBM_IF_EITHER_EN

    /*
     * Sanitize the control stuff.
     */
    pDbgState->fCpe2Extra       &= g_HmMsrs.u.vmx.ProcCtls2.n.allowed1;
    if (pDbgState->fCpe2Extra)
        pDbgState->fCpe1Extra   |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
    pDbgState->fCpe1Extra       &= g_HmMsrs.u.vmx.ProcCtls.n.allowed1;
    pDbgState->fCpe1Unwanted    &= ~g_HmMsrs.u.vmx.ProcCtls.n.allowed0;
#ifndef IN_NEM_DARWIN
    if (pVCpu->hmr0.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
    {
        pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
        pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
    }
#else
    if (pVCpu->nem.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
    {
        pVCpu->nem.s.fDebugWantRdTscExit ^= true;
        pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
    }
#endif

    Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
          pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
          pDbgState->fClearCr0Mask ? " clr-cr0" : "",
          pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
}


/**
 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
 * appropriate.
 *
 * The caller has checked the VM-exit against the
 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
 * already, so we don't have to do that either.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @param   pVCpu           The cross context virtual CPU structure.
 * @param   pVmxTransient   The VMX-transient structure.
 * @param   uExitReason     The VM-exit reason.
 *
 * @remarks The name of this function is displayed by dtrace, so keep it short
 *          and to the point. No longer than 33 chars long, please.
 */
static VBOXSTRICTRC vmxHCHandleExitDtraceEvents(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
{
    /*
     * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
     * same time check whether any corresponding Dtrace event is enabled (fDtrace).
     *
     * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
     *       does.  Must add/change/remove both places.  Same ordering, please.
     *
     *       Added/removed events must also be reflected in the next section
     *       where we dispatch dtrace events.
     */
    bool            fDtrace1   = false;
    bool            fDtrace2   = false;
    DBGFEVENTTYPE   enmEvent1  = DBGFEVENT_END;
    DBGFEVENTTYPE   enmEvent2  = DBGFEVENT_END;
    uint32_t        uEventArg  = 0;
#define SET_EXIT(a_EventSubName) \
        do { \
            enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_,  a_EventSubName); \
            fDtrace2  = RT_CONCAT3(VBOXVMM_EXIT_,   a_EventSubName, _ENABLED)(); \
        } while (0)
#define SET_BOTH(a_EventSubName) \
        do { \
            enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
            enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_,  a_EventSubName); \
            fDtrace1  = RT_CONCAT3(VBOXVMM_INSTR_,  a_EventSubName, _ENABLED)(); \
            fDtrace2  = RT_CONCAT3(VBOXVMM_EXIT_,   a_EventSubName, _ENABLED)(); \
        } while (0)
    switch (uExitReason)
    {
        case VMX_EXIT_MTF:
            return vmxHCExitMtf(pVCpu, pVmxTransient);

        case VMX_EXIT_XCPT_OR_NMI:
        {
            uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
            switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
            {
                case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
                case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
                case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
                    if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
                    {
                        if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
                        {
                            vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE>(pVCpu, pVmxTransient);
                            uEventArg = pVmxTransient->uExitIntErrorCode;
                        }
                        enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
                        switch (enmEvent1)
                        {
                            case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
                            case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
                            case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
                            case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
                            case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
                            case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
                            case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
                            case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
                            case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
                            case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
                            case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
                            case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
                            case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
                            case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
                            case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
                            case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
                            case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
                            case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
                            default:                                                      break;
                        }
                    }
                    else
                        AssertFailed();
                    break;

                case VMX_EXIT_INT_INFO_TYPE_SW_INT:
                    uEventArg = idxVector;
                    enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
                    fDtrace1  = VBOXVMM_INT_SOFTWARE_ENABLED();
                    break;
            }
            break;
        }

        case VMX_EXIT_TRIPLE_FAULT:
            enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
            //fDtrace1  = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
            break;
        case VMX_EXIT_TASK_SWITCH:      SET_EXIT(TASK_SWITCH); break;
        case VMX_EXIT_EPT_VIOLATION:    SET_EXIT(VMX_EPT_VIOLATION); break;
        case VMX_EXIT_EPT_MISCONFIG:    SET_EXIT(VMX_EPT_MISCONFIG); break;
        case VMX_EXIT_APIC_ACCESS:      SET_EXIT(VMX_VAPIC_ACCESS); break;
        case VMX_EXIT_APIC_WRITE:       SET_EXIT(VMX_VAPIC_WRITE); break;

        /* Instruction specific VM-exits: */
        case VMX_EXIT_CPUID:            SET_BOTH(CPUID); break;
        case VMX_EXIT_GETSEC:           SET_BOTH(GETSEC); break;
        case VMX_EXIT_HLT:              SET_BOTH(HALT); break;
        case VMX_EXIT_INVD:             SET_BOTH(INVD); break;
        case VMX_EXIT_INVLPG:           SET_BOTH(INVLPG); break;
        case VMX_EXIT_RDPMC:            SET_BOTH(RDPMC); break;
        case VMX_EXIT_RDTSC:            SET_BOTH(RDTSC); break;
        case VMX_EXIT_RSM:              SET_BOTH(RSM); break;
        case VMX_EXIT_VMCALL:           SET_BOTH(VMM_CALL); break;
        case VMX_EXIT_VMCLEAR:          SET_BOTH(VMX_VMCLEAR); break;
        case VMX_EXIT_VMLAUNCH:         SET_BOTH(VMX_VMLAUNCH); break;
        case VMX_EXIT_VMPTRLD:          SET_BOTH(VMX_VMPTRLD); break;
        case VMX_EXIT_VMPTRST:          SET_BOTH(VMX_VMPTRST); break;
        case VMX_EXIT_VMREAD:           SET_BOTH(VMX_VMREAD); break;
        case VMX_EXIT_VMRESUME:         SET_BOTH(VMX_VMRESUME); break;
        case VMX_EXIT_VMWRITE:          SET_BOTH(VMX_VMWRITE); break;
        case VMX_EXIT_VMXOFF:           SET_BOTH(VMX_VMXOFF); break;
        case VMX_EXIT_VMXON:            SET_BOTH(VMX_VMXON); break;
        case VMX_EXIT_MOV_CRX:
            vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
            if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
                SET_BOTH(CRX_READ);
            else
                SET_BOTH(CRX_WRITE);
            uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
            break;
        case VMX_EXIT_MOV_DRX:
            vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
            if (   VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
                == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
                SET_BOTH(DRX_READ);
            else
                SET_BOTH(DRX_WRITE);
            uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
            break;
        case VMX_EXIT_RDMSR:            SET_BOTH(RDMSR); break;
        case VMX_EXIT_WRMSR:            SET_BOTH(WRMSR); break;
        case VMX_EXIT_MWAIT:            SET_BOTH(MWAIT); break;
        case VMX_EXIT_MONITOR:          SET_BOTH(MONITOR); break;
        case VMX_EXIT_PAUSE:            SET_BOTH(PAUSE); break;
        case VMX_EXIT_GDTR_IDTR_ACCESS:
            vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
            switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
            {
                case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
                case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
                case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
                case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
            }
            break;

        case VMX_EXIT_LDTR_TR_ACCESS:
            vmxHCReadToTransient<HMVMX_READ_EXIT_INSTR_INFO>(pVCpu, pVmxTransient);
            switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
            {
                case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
                case VMX_YYTR_INSINFO_II_STR:  SET_BOTH(STR); break;
                case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
                case VMX_YYTR_INSINFO_II_LTR:  SET_BOTH(LTR); break;
            }
            break;

        case VMX_EXIT_INVEPT:           SET_BOTH(VMX_INVEPT); break;
        case VMX_EXIT_RDTSCP:           SET_BOTH(RDTSCP); break;
        case VMX_EXIT_INVVPID:          SET_BOTH(VMX_INVVPID); break;
        case VMX_EXIT_WBINVD:           SET_BOTH(WBINVD); break;
        case VMX_EXIT_XSETBV:           SET_BOTH(XSETBV); break;
        case VMX_EXIT_RDRAND:           SET_BOTH(RDRAND); break;
        case VMX_EXIT_INVPCID:          SET_BOTH(VMX_INVPCID); break;
        case VMX_EXIT_VMFUNC:           SET_BOTH(VMX_VMFUNC); break;
        case VMX_EXIT_RDSEED:           SET_BOTH(RDSEED); break;
        case VMX_EXIT_XSAVES:           SET_BOTH(XSAVES); break;
        case VMX_EXIT_XRSTORS:          SET_BOTH(XRSTORS); break;

        /* Events that aren't relevant at this point. */
        case VMX_EXIT_EXT_INT:
        case VMX_EXIT_INT_WINDOW:
        case VMX_EXIT_NMI_WINDOW:
        case VMX_EXIT_TPR_BELOW_THRESHOLD:
        case VMX_EXIT_PREEMPT_TIMER:
        case VMX_EXIT_IO_INSTR:
            break;

        /* Errors and unexpected events. */
        case VMX_EXIT_INIT_SIGNAL:
        case VMX_EXIT_SIPI:
        case VMX_EXIT_IO_SMI:
        case VMX_EXIT_SMI:
        case VMX_EXIT_ERR_INVALID_GUEST_STATE:
        case VMX_EXIT_ERR_MSR_LOAD:
        case VMX_EXIT_ERR_MACHINE_CHECK:
        case VMX_EXIT_PML_FULL:
        case VMX_EXIT_VIRTUALIZED_EOI:
            break;

        default:
            AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
            break;
    }
#undef SET_BOTH
#undef SET_EXIT

    /*
     * Dtrace tracepoints go first.   We do them here at once so we don't
     * have to copy the guest state saving and stuff a few dozen times.
     * Down side is that we've got to repeat the switch, though this time
     * we use enmEvent since the probes are a subset of what DBGF does.
     */
    if (fDtrace1 || fDtrace2)
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
        vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
        PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
        switch (enmEvent1)
        {
            /** @todo consider which extra parameters would be helpful for each probe.   */
            case DBGFEVENT_END: break;
            case DBGFEVENT_XCPT_DE:                 VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_DB:                 VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
            case DBGFEVENT_XCPT_BP:                 VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_OF:                 VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_BR:                 VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_UD:                 VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_NM:                 VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_DF:                 VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_TS:                 VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
            case DBGFEVENT_XCPT_NP:                 VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
            case DBGFEVENT_XCPT_SS:                 VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
            case DBGFEVENT_XCPT_GP:                 VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
            case DBGFEVENT_XCPT_PF:                 VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
            case DBGFEVENT_XCPT_MF:                 VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_AC:                 VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_XF:                 VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_VE:                 VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
            case DBGFEVENT_XCPT_SX:                 VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
            case DBGFEVENT_INTERRUPT_SOFTWARE:      VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
            case DBGFEVENT_INSTR_CPUID:             VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
            case DBGFEVENT_INSTR_GETSEC:            VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_HALT:              VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_INVD:              VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_INVLPG:            VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_RDPMC:             VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_RDTSC:             VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_RSM:               VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_CRX_READ:          VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
            case DBGFEVENT_INSTR_CRX_WRITE:         VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
            case DBGFEVENT_INSTR_DRX_READ:          VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
            case DBGFEVENT_INSTR_DRX_WRITE:         VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
            case DBGFEVENT_INSTR_RDMSR:             VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
            case DBGFEVENT_INSTR_WRMSR:             VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
                                                                        RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
            case DBGFEVENT_INSTR_MWAIT:             VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_MONITOR:           VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_PAUSE:             VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_SGDT:              VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_SIDT:              VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_LGDT:              VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_LIDT:              VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_SLDT:              VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_STR:               VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_LLDT:              VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_LTR:               VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_RDTSCP:            VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_WBINVD:            VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_XSETBV:            VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_RDRAND:            VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_RDSEED:            VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_XSAVES:            VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_XRSTORS:           VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMM_CALL:          VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_VMCLEAR:       VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_VMLAUNCH:      VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_VMPTRLD:       VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_VMPTRST:       VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_VMREAD:        VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_VMRESUME:      VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_VMWRITE:       VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_VMXOFF:        VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_VMXON:         VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_INVEPT:        VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_INVVPID:       VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_INVPCID:       VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
            case DBGFEVENT_INSTR_VMX_VMFUNC:        VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
            default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
        }
        switch (enmEvent2)
        {
            /** @todo consider which extra parameters would be helpful for each probe. */
            case DBGFEVENT_END: break;
            case DBGFEVENT_EXIT_TASK_SWITCH:        VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_CPUID:              VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
            case DBGFEVENT_EXIT_GETSEC:             VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_HALT:               VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_INVD:               VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_INVLPG:             VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_RDPMC:              VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_RDTSC:              VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_RSM:                VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_CRX_READ:           VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
            case DBGFEVENT_EXIT_CRX_WRITE:          VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
            case DBGFEVENT_EXIT_DRX_READ:           VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
            case DBGFEVENT_EXIT_DRX_WRITE:          VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
            case DBGFEVENT_EXIT_RDMSR:              VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
            case DBGFEVENT_EXIT_WRMSR:              VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
                                                                       RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
            case DBGFEVENT_EXIT_MWAIT:              VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_MONITOR:            VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_PAUSE:              VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_SGDT:               VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_SIDT:               VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_LGDT:               VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_LIDT:               VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_SLDT:               VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_STR:                VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_LLDT:               VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_LTR:                VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_RDTSCP:             VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_WBINVD:             VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_XSETBV:             VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_RDRAND:             VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_RDSEED:             VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_XSAVES:             VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_XRSTORS:            VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMM_CALL:           VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VMCLEAR:        VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VMLAUNCH:       VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VMPTRLD:        VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VMPTRST:        VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VMREAD:         VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VMRESUME:       VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VMWRITE:        VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VMXOFF:         VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VMXON:          VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_INVEPT:         VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_INVVPID:        VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_INVPCID:        VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VMFUNC:         VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG:  VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_EPT_VIOLATION:  VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS:   VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
            case DBGFEVENT_EXIT_VMX_VAPIC_WRITE:    VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
            default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
        }
    }

    /*
     * Fire of the DBGF event, if enabled (our check here is just a quick one,
     * the DBGF call will do a full check).
     *
     * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
     * Note! If we have to events, we prioritize the first, i.e. the instruction
     *       one, in order to avoid event nesting.
     */
    PVMCC pVM = pVCpu->CTX_SUFF(pVM);
    if (   enmEvent1 != DBGFEVENT_END
        && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
    {
        vmxHCImportGuestState<CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
        VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
        if (rcStrict != VINF_SUCCESS)
            return rcStrict;
    }
    else if (   enmEvent2 != DBGFEVENT_END
             && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
    {
        vmxHCImportGuestState<CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
        VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
        if (rcStrict != VINF_SUCCESS)
            return rcStrict;
    }

    return VINF_SUCCESS;
}


/**
 * Single-stepping VM-exit filtering.
 *
 * This is preprocessing the VM-exits and deciding whether we've gotten far
 * enough to return VINF_EM_DBG_STEPPED already.  If not, normal VM-exit
 * handling is performed.
 *
 * @returns Strict VBox status code (i.e. informational status codes too).
 * @param   pVCpu           The cross context virtual CPU structure of the calling EMT.
 * @param   pVmxTransient   The VMX-transient structure.
 * @param   pDbgState       The debug state.
 */
DECLINLINE(VBOXSTRICTRC) vmxHCRunDebugHandleExit(PVMCPUCC pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
{
    /*
     * Expensive (saves context) generic dtrace VM-exit probe.
     */
    uint32_t const uExitReason = pVmxTransient->uExitReason;
    if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
    { /* more likely */ }
    else
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_QUALIFICATION>(pVCpu, pVmxTransient);
        int rc = vmxHCImportGuestState<HMVMX_CPUMCTX_EXTRN_ALL>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
        AssertRC(rc);
        VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
    }

#ifndef IN_NEM_DARWIN
    /*
     * Check for host NMI, just to get that out of the way.
     */
    if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
    { /* normally likely */ }
    else
    {
        vmxHCReadToTransient<HMVMX_READ_EXIT_INTERRUPTION_INFO>(pVCpu, pVmxTransient);
        uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
        if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
            return hmR0VmxExitHostNmi(pVCpu, pVmxTransient->pVmcsInfo);
    }
#endif

    /*
     * Check for single stepping event if we're stepping.
     */
    if (VCPU_2_VMXSTATE(pVCpu).fSingleInstruction)
    {
        switch (uExitReason)
        {
            case VMX_EXIT_MTF:
                return vmxHCExitMtf(pVCpu, pVmxTransient);

            /* Various events: */
            case VMX_EXIT_XCPT_OR_NMI:
            case VMX_EXIT_EXT_INT:
            case VMX_EXIT_TRIPLE_FAULT:
            case VMX_EXIT_INT_WINDOW:
            case VMX_EXIT_NMI_WINDOW:
            case VMX_EXIT_TASK_SWITCH:
            case VMX_EXIT_TPR_BELOW_THRESHOLD:
            case VMX_EXIT_APIC_ACCESS:
            case VMX_EXIT_EPT_VIOLATION:
            case VMX_EXIT_EPT_MISCONFIG:
            case VMX_EXIT_PREEMPT_TIMER:

            /* Instruction specific VM-exits: */
            case VMX_EXIT_CPUID:
            case VMX_EXIT_GETSEC:
            case VMX_EXIT_HLT:
            case VMX_EXIT_INVD:
            case VMX_EXIT_INVLPG:
            case VMX_EXIT_RDPMC:
            case VMX_EXIT_RDTSC:
            case VMX_EXIT_RSM:
            case VMX_EXIT_VMCALL:
            case VMX_EXIT_VMCLEAR:
            case VMX_EXIT_VMLAUNCH:
            case VMX_EXIT_VMPTRLD:
            case VMX_EXIT_VMPTRST:
            case VMX_EXIT_VMREAD:
            case VMX_EXIT_VMRESUME:
            case VMX_EXIT_VMWRITE:
            case VMX_EXIT_VMXOFF:
            case VMX_EXIT_VMXON:
            case VMX_EXIT_MOV_CRX:
            case VMX_EXIT_MOV_DRX:
            case VMX_EXIT_IO_INSTR:
            case VMX_EXIT_RDMSR:
            case VMX_EXIT_WRMSR:
            case VMX_EXIT_MWAIT:
            case VMX_EXIT_MONITOR:
            case VMX_EXIT_PAUSE:
            case VMX_EXIT_GDTR_IDTR_ACCESS:
            case VMX_EXIT_LDTR_TR_ACCESS:
            case VMX_EXIT_INVEPT:
            case VMX_EXIT_RDTSCP:
            case VMX_EXIT_INVVPID:
            case VMX_EXIT_WBINVD:
            case VMX_EXIT_XSETBV:
            case VMX_EXIT_RDRAND:
            case VMX_EXIT_INVPCID:
            case VMX_EXIT_VMFUNC:
            case VMX_EXIT_RDSEED:
            case VMX_EXIT_XSAVES:
            case VMX_EXIT_XRSTORS:
            {
                int rc = vmxHCImportGuestState<CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP>(pVCpu, pVmxTransient->pVmcsInfo, __FUNCTION__);
                AssertRCReturn(rc, rc);
                if (   pVCpu->cpum.GstCtx.rip    != pDbgState->uRipStart
                    || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
                    return VINF_EM_DBG_STEPPED;
                break;
            }

            /* Errors and unexpected events: */
            case VMX_EXIT_INIT_SIGNAL:
            case VMX_EXIT_SIPI:
            case VMX_EXIT_IO_SMI:
            case VMX_EXIT_SMI:
            case VMX_EXIT_ERR_INVALID_GUEST_STATE:
            case VMX_EXIT_ERR_MSR_LOAD:
            case VMX_EXIT_ERR_MACHINE_CHECK:
            case VMX_EXIT_PML_FULL:
            case VMX_EXIT_VIRTUALIZED_EOI:
            case VMX_EXIT_APIC_WRITE:  /* Some talk about this being fault like, so I guess we must process it? */
                break;

            default:
                AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
                break;
        }
    }

    /*
     * Check for debugger event breakpoints and dtrace probes.
     */
    if (   uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
        && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
    {
        VBOXSTRICTRC rcStrict = vmxHCHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
        if (rcStrict != VINF_SUCCESS)
            return rcStrict;
    }

    /*
     * Normal processing.
     */
#ifdef HMVMX_USE_FUNCTION_TABLE
    return g_aVMExitHandlers[uExitReason].pfn(pVCpu, pVmxTransient);
#else
    return vmxHCHandleExit(pVCpu, pVmxTransient, uExitReason);
#endif
}

/** @} */